improvement: update models and spider for saving less data

Author: leafcoder

ncovapi/admin.py

@@ -18,67 +18,11 @@ class BaseAdmin(admin.ModelAdmin):
     preserve_filters = True
 
 
-@admin.register(models.Crawler)
-class CrawlerAdmin(BaseAdmin):
-
-    list_display = (
-        'id', 'crawlTime', 'createTime', 'modifyTime'
-    )
-
-
 @admin.register(models.Statistics)
 class StatisticsAdmin(BaseAdmin):
 
-    list_display = (
-        'id', 'countryType', 'seriousCount', 'currentConfirmedCount',
-        'confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'
-    )
-    search_fields = ('crawler_id', )
-    list_filter = ('countryType', )
-
-
-@admin.register(models.Notice)
-class NoticeAdmin(BaseAdmin):
-
-    list_display = (
-        'id', 'crawler', 'remarks', 'notes', 'generalRemark'
-    )
-
-@admin.register(models.WHOArticle)
-class WHOArticleAdmin(BaseAdmin):
+    pass
 
-    list_display = ('id', 'title', 'linkUrl', 'imgUrl')
-
-@admin.register(models.Recommend)
-class RecommendAdmin(BaseAdmin):
-
-    list_display = (
-        'id', 'title', 'linkUrl', 'imgUrl', 'contentType',
-        'countryType', 'recordStatus')
-
-@admin.register(models.Timeline)
-class TimelineAdmin(BaseAdmin):
-
-    list_display = (
-        'id', 'title', 'summary', 'pubDateStr', 'infoSource', 'sourceUrl')
-
-@admin.register(models.Wiki)
-class WikiAdmin(BaseAdmin):
-
-    list_display = ('id', 'title', 'linkUrl', 'imgUrl', 'description')
-
-@admin.register(models.GoodsGuide)
-class GoodsGuideAdmin(BaseAdmin):
-
-    list_display = (
-        'categoryName', 'title', 'recordStatus', 'contentImgUrls')
-
-@admin.register(models.Rumor)
-class RumorAdmin(BaseAdmin):
-
-    list_display = (
-        'title', 'mainSummary', 'summary', 'body',
-        'sourceUrl', 'score', 'rumorType')
 
 @admin.register(models.City)
 class CityAdmin(BaseAdmin):

ncovapi/filters.py

@@ -11,12 +11,21 @@ class CityFilter(django_filters.rest_framework.FilterSet):
 
     provinceShortNames = CharInFilter(
         field_name='province__provinceShortName', lookup_expr='in')
+    provinceNames = CharInFilter(
+        field_name='province__provinceName', lookup_expr='in')
     cityNames = CharInFilter(
         field_name='cityName', lookup_expr='in')
+ 
+    provinceShortName = django_filters.CharFilter(
+        field_name='province__provinceShortName', lookup_expr='exact')
+    provinceName = django_filters.CharFilter(
+        field_name='province__provinceName', lookup_expr='exact')
+    cityName = django_filters.CharFilter(
+        field_name='cityName', lookup_expr='exact')
 
     class Meta:
         model = City
-        fields = ['provinceShortNames', 'cityName']
+        fields = ['provinceShortName', 'provinceName', 'cityName']
 
 
 class ProvinceFilter(django_filters.rest_framework.FilterSet):
@@ -26,6 +35,11 @@ class ProvinceFilter(django_filters.rest_framework.FilterSet):
     provinceNames = CharInFilter(
         field_name='provinceName', lookup_expr='in')
 
+    provinceShortName = django_filters.CharFilter(
+        field_name='provinceShortName', lookup_expr='exact')
+    provinceName = django_filters.CharFilter(
+        field_name='provinceName', lookup_expr='exact')
+
     class Meta:
         model = Province
         fields = ['provinceName', 'provinceShortName']

ncovapi/models.py

@@ -1,192 +1,39 @@
 from django.db import models
 from django.utils import timezone
-from django_mysql.models import ListCharField
+from django_mysql.models import JSONField
 
 
-class Crawler(models.Model):
+class Statistics(models.Model):
+
 
+    JSON_FIELDS = (
+        'globalStatistics', 'domesticStatistics', 'internationalStatistics',
+        'remarks', 'notes', 'WHOArticle', 'recommends', 'timelines',
+        'wikis', 'goodsGuides', 'rumors'
+    )
+
+    globalStatistics = models.TextField(default='{}')
+    domesticStatistics = models.TextField(default='{}')
+    internationalStatistics = models.TextField(default='{}')
+    remarks = models.TextField(default='[]')
+    notes = models.TextField(default='[]')
+    generalRemark = models.TextField(default='')
+    WHOArticle = models.TextField(verbose_name='WHO 文章', default='{}')
+    recommends = models.TextField(verbose_name='防护知识', default='[]')
+    timelines = models.TextField(verbose_name='时间线事件', default='[]')
+    wikis = models.TextField(verbose_name='Wiki', default='[]')
+    goodsGuides = models.TextField(verbose_name='购物指南', default='[]')
+    rumors = models.TextField(verbose_name='辟谣与防护', default='[]')
     modifyTime = models.DateTimeField(null=True)
     createTime = models.DateTimeField(null=True)
     crawlTime = models.DateTimeField(
         "抓取时间", default=timezone.now, editable=False)
 
-    class Meta:
-        verbose_name = "抓取版本"
-        verbose_name_plural = "抓取版本"
-
-
-class Statistics(models.Model):
-
-    GLOBAL = 0
-    DOMESTIC = 1
-    INTERNATIONAL = 2
-    COUNTRY_TYPES = (
-        (GLOBAL, '全球'),
-        (DOMESTIC, '国内'),
-        (INTERNATIONAL, '国外')
-    )
-
-    countryType = models.IntegerField(choices=COUNTRY_TYPES)
-    currentConfirmedCount = models.IntegerField(default=0)
-    confirmedCount = models.IntegerField(default=0)
-    suspectedCount = models.IntegerField(default=0)
-    seriousCount = models.IntegerField('现存无症状', default=0)
-    curedCount = models.IntegerField(default=0)
-    deadCount = models.IntegerField(default=0)
-    currentConfirmedIncr = models.IntegerField(default=0)
-    confirmedIncr = models.IntegerField(default=0)
-    suspectedIncr = models.IntegerField(default=0)
-    curedIncr = models.IntegerField(default=0)
-    deadIncr = models.IntegerField(default=0)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="statistics",
-        db_column="crawlerId"
-    )
-
     class Meta:
         verbose_name = '统计数据'
         verbose_name_plural = '统计数据'
 
 
-class Notice(models.Model):
-
-    remarks = ListCharField(
-        models.CharField(max_length=100), size=10, max_length=100*11)
-    notes = ListCharField(
-        models.CharField(max_length=100), size=10, max_length=100*11)
-    generalRemark = models.TextField(null=True)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="notices",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = '注意信息'
-        verbose_name_plural = '注意信息'
-
-
-class WHOArticle(models.Model):
-
-    title = models.CharField(max_length=100)
-    linkUrl = models.URLField(max_length=200)
-    imgUrl = models.URLField(max_length=200)
-    crawler = models.OneToOneField(
-        "Crawler", on_delete=models.CASCADE, related_name="WHO_article",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = 'WHO 文章'
-        verbose_name_plural = 'WHO 文章'
-
-
-class Recommend(models.Model):
-
-    CONTENT_TYPES = [
-        (1, '我要出行'),
-        (2, '家有小孩'),
-        (3, '未知'),
-        (4, '我宅在家'),
-        (5, '未知'),
-        (6, '未知'),
-        (7, '未知')
-    ]
-    GLOBAL = 0
-    DOMESTIC = 1
-    INTERNATIONAL = 2
-    COUNTRY_TYPES = (
-        (GLOBAL, '全球'),
-        (DOMESTIC, '国内'),
-        (INTERNATIONAL, '国外')
-    )
-
-    title = models.CharField(max_length=100)
-    linkUrl = models.URLField(max_length=200)
-    imgUrl = models.URLField(max_length=200)
-    contentType = models.IntegerField(choices=CONTENT_TYPES)
-    recordStatus = models.IntegerField()
-    countryType = models.IntegerField(choices=COUNTRY_TYPES)
-    sort = models.IntegerField()
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="recommends",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = '防护知识'
-        verbose_name_plural = '防护知识'
-
-
-class Timeline(models.Model):
-
-    pubDate = models.IntegerField()
-    pubDateStr = models.CharField(max_length=50)
-    title = models.CharField(max_length=100)
-    summary = models.TextField()
-    infoSource = models.CharField(max_length=50)
-    sourceUrl = models.URLField(max_length=200)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="timelines",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = '时间线事件'
-        verbose_name_plural = '时间线事件'
-
-
-class Wiki(models.Model):
-
-    title = models.CharField(max_length=100)
-    linkUrl = models.URLField(max_length=200)
-    imgUrl = models.URLField(max_length=200)
-    description = models.TextField()
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="wikis",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = 'Wiki'
-        verbose_name_plural = 'Wiki'
-
-
-class GoodsGuide(models.Model):
-
-    title = models.CharField(max_length=100)
-    categoryName = models.CharField(max_length=50)
-    recordStatus = models.IntegerField()
-    contentImgUrls = ListCharField(
-        models.CharField(max_length=100), size=10, max_length=100*11)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="goods_guides",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = '购物指南'
-        verbose_name_plural = '购物指南'
-
-
-class Rumor(models.Model):
-
-    title = models.CharField(max_length=100)
-    mainSummary = models.TextField()
-    summary = models.TextField()
-    body = models.TextField()
-    sourceUrl = models.URLField(max_length=200)
-    score = models.IntegerField()
-    rumorType = models.IntegerField()
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="rumors",
-        db_column="crawlerId"
-    )
-
-    class Meta:
-        verbose_name = '辟谣与防护'
-        verbose_name_plural = '辟谣与防护'
-
-
 class Province(models.Model):
 
     locationId = models.IntegerField()
@@ -204,10 +51,6 @@ class Province(models.Model):
         '创建时间', auto_now_add=True, editable=False)
     updated = models.DateTimeField(
         '更新时间', auto_now=True, editable=False)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="provinces",
-        db_column="crawlerId"
-    )
 
     class Meta:
         verbose_name = '国内省份'
@@ -231,9 +74,6 @@ class City(models.Model):
         "Province", on_delete=models.CASCADE, related_name="cities",
         db_column="provinceId"
     )
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, db_column="crawlerId"
-    )
 
     @property
     def provinceName(self):
@@ -273,10 +113,6 @@ class Country(models.Model):
         '创建时间', auto_now_add=True, editable=False)
     updated = models.DateTimeField(
         '更新时间', auto_now=True, editable=False)
-    crawler = models.ForeignKey(
-        "Crawler", on_delete=models.CASCADE, related_name="countries",
-        db_column="countryId"
-    )
 
     class Meta:
         verbose_name = "国家地区"