Merge branch 'develop'

Author: leafcoder

MANIFEST.in

@@ -0,0 +1,13 @@
+include LICENSE
+include README.md
+include requirements.txt
+
+recursive-include django_covid19/spider *
+recursive-include docs *
+recursive-include demo *
+recursive-include demo_proj manage.py
+recursive-include demo_proj requirements.txt
+recursive-include demo_proj/ncov *
+
+recursive-exclude demo_proj db.sqlite3
+recursive-exclude django_covid19/migrations *

README.md

@@ -20,7 +20,7 @@
     <img src="https://img.shields.io/github/languages/top/leafcoder/django-covid19" data-origin="https://img.shields.io/github/languages/top/leafcoder/django-covid19" alt="GitHub top language">
     <img src="https://img.shields.io/github/languages/code-size/leafcoder/django-covid19" data-origin="https://img.shields.io/github/languages/code-size/leafcoder/django-covid19" alt="GitHub code size in bytes">
     <img src="https://img.shields.io/github/commit-activity/w/leafcoder/django-covid19" data-origin="https://img.shields.io/github/commit-activity/w/leafcoder/django-covid19" alt="GitHub commit activity">
-    <img src="https://img.shields.io/github/downloads/leafcoder/django-covid19/total" data-origin="https://img.shields.io/github/downloads/leafcoder/django-covid19/total" alt="GitHub All Releases">
+    <img src="https://img.shields.io/pypi/dm/django_covid19" data-origin="https://img.shields.io/pypi/dm/django_covid19" alt="PyPI - Downloads">
 </p>
 
 </div>
@@ -36,14 +36,14 @@
 如果开发者本身没有个人的云服务器用来部署本项目，也可以直接调用本项目已部署好
 的实时接口，可用于科研、娱乐、教学等各方面。
 
-[![在线文档](docs/images/docs.png)](http://111.231.75.86:8000/docs/)
+[![在线文档](https://raw.githubusercontent.com/leafcoder/django-covid19/master/docs/images/docs.png)](http://111.231.75.86:8000/docs/)
 
 # 在线大屏
 
 根据已部署的疫情在线接口，并结合使用开源数据大屏项目中的示例代码，本项目提
 供了一个使用本项目接口的数据大屏示例。
 
-[![在线数据大屏](docs/images/dashboard.png)](http://111.231.75.86/dashboard)
+[![在线数据大屏](https://raw.githubusercontent.com/leafcoder/django-covid19/master/docs/images/dashboard.png)](http://111.231.75.86/dashboard)
 
 # 问题相关
 

manage.py renamed to demo_proj/manage.py

@@ -5,7 +5,7 @@
 
 
 def main():
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'covid19.settings')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ncov.settings')
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

covid19/__init__.py renamed to demo_proj/ncov/__init__.py

@@ -1,5 +1,5 @@
 """
-Django settings for covid19 project.
+Django settings for ncov project.
 
 Generated by 'django-admin startproject' using Django 2.2.10.
 
@@ -37,26 +37,26 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
-
+    # set by user
     'corsheaders',
     'django_crontab',
     'rest_framework',
     'django_filters',
-    'ncovapi.apps.NcovapiConfig'
+    'django_covid19'
 ]
 
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
-    'corsheaders.middleware.CorsMiddleware',
+    'corsheaders.middleware.CorsMiddleware',  # 新增跨域部分
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
-ROOT_URLCONF = 'covid19.urls'
+ROOT_URLCONF = 'ncov.urls'
 
 TEMPLATES = [
     {
@@ -74,7 +74,7 @@
     },
 ]
 
-WSGI_APPLICATION = 'covid19.wsgi.application'
+WSGI_APPLICATION = 'ncov.wsgi.application'
 
 
 # Database
@@ -139,15 +139,15 @@
 CACHES = {
     'default': {
         'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-        'LOCATION': '/var/tmp/covid19_cache',
+        'LOCATION': '/var/tmp/ncov_cache',
         'TIMEOUT': 3600,
         'OPTIONS': {
             'MAX_ENTRIES': 20000
         }
     }
 }
 
-#跨域增加忽略
+# 跨域增加忽略
 CORS_ALLOW_CREDENTIALS = True
 CORS_ORIGIN_ALLOW_ALL = True
 
@@ -175,22 +175,12 @@
     'Pragma',
 )
 
-CRONTAB_LOCK_JOBS = True
-
 # 静态文件目录
 STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
-# 日志文件目录
-LOGS_DIR = os.path.join(BASE_DIR, 'var', 'logs')
-if not os.path.exists(LOGS_DIR):
-    os.makedirs(LOGS_DIR)
-
-# 配置 Scrapy 命令完整路径
-SCRAPY_CMD = '~/.virtualenvs/django-covid19/bin/scrapy'
-
-# Setting of Crontab
+CRONTAB_LOCK_JOBS = True
 CRONJOBS = (
     # 每分钟抓取一次
-    ('*/1 * * * *', 'ncovapi.cron.crawl_dxy', [], {}, '>> %s/crontab.log' % LOGS_DIR),
+    ('*/1 * * * *', 'django.core.management.call_command', ['crawl']),
 )
 

covid19/settings.py renamed to demo_proj/ncov/settings.py

@@ -1,4 +1,4 @@
-"""covid19 URL Configuration
+"""ncov URL Configuration
 
 The `urlpatterns` list routes URLs to views. For more information please see:
     https://docs.djangoproject.com/en/2.2/topics/http/urls/
@@ -19,17 +19,9 @@
 from django.conf import settings
 from django.conf.urls import url
 
-import os
 
 urlpatterns = [
     path('admin/', admin.site.urls),
-    path('api/', include('ncovapi.urls', namespace='ncovapi')),
-    url(r'^static/(?P<path>.*)$', serve, {'document_root': settings.STATIC_ROOT}),
-    url(r'^docs/$', serve, {
-        'document_root': os.path.join(settings.BASE_DIR, 'docs'),
-        'path': 'index.html'
-    }),
-    url(r'^docs/(?P<path>.*)$', serve, {
-        'document_root': os.path.join(settings.BASE_DIR, 'docs')
-    })
+    path('api/', include('django_covid19.urls', namespace='django_covid19')),
+    url(r'^static/(?P<path>.*)$', serve, {'document_root': settings.STATIC_ROOT})
 ]

covid19/urls.py renamed to demo_proj/ncov/urls.py

@@ -1,5 +1,5 @@
 """
-WSGI config for covid19 project.
+WSGI config for ncov project.
 
 It exposes the WSGI callable as a module-level variable named ``application``.
 
@@ -11,6 +11,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'covid19.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ncov.settings')
 
 application = get_wsgi_application()

covid19/wsgi.py renamed to demo_proj/ncov/wsgi.py

@@ -0,0 +1,40 @@
+attrs==19.3.0
+Automat==20.2.0
+certifi==2020.4.5.1
+cffi==1.14.0
+chardet==3.0.4
+constantly==15.1.0
+cryptography==2.9
+cssselect==1.1.0
+Django==2.2.12
+django-cors-headers==3.2.1
+django-covid19 @ file:///home/zhanglei3/Desktop/django-covid19/dist/django_covid19-0.3.tar.gz
+django-crontab==0.7.1
+django-filter==2.2.0
+django-mysql==3.4.0
+djangorestframework==3.11.0
+drf-extensions==0.6.0
+hyperlink==19.0.0
+idna==2.9
+incremental==17.5.0
+lxml==4.5.0
+parsel==1.5.2
+Protego==0.1.16
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycparser==2.20
+PyDispatcher==2.0.5
+PyHamcrest==2.0.2
+pyOpenSSL==19.1.0
+pytz==2019.3
+queuelib==1.5.0
+schedule==0.6.0
+Scrapy==2.0.1
+scrapy-djangoitem==1.1.1
+service-identity==18.1.0
+six==1.14.0
+sqlparse==0.3.1
+Twisted==20.3.0
+urllib3==1.25.9
+w3lib==1.21.0
+zope.interface==5.1.0

demo_proj/requirements.txt

@@ -0,0 +1 @@
+default_app_config = 'django_covid19.apps.DjangoCovid19Config'

django_covid19/__init__.py

@@ -1,9 +1,10 @@
 from django.contrib import admin
-from django.conf import settings
 from django.urls import reverse
 from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 
 from . import models
+import json
 # Register your models here.
 
 
@@ -22,11 +23,37 @@ class BaseAdmin(admin.ModelAdmin):
 class StatisticsAdmin(BaseAdmin):
 
     list_display = (
-        'globalStatistics', 'domesticStatistics', 'internationalStatistics',
-        'modifyTime', 'createTime', 'crawlTime'
+        'id', 'jsonGlobalStatistics', 'jsonDomesticStatistics',
+        'jsonInternationalStatistics', 'modifyTime', 'crawlTime'
     )
     search_fields = ('crawlTime', 'modifyTime')
 
+    def jsonGlobalStatistics(self, obj):
+        return self.to_json(obj.globalStatistics)
+    jsonGlobalStatistics.short_description = '全球疫情'
+    jsonGlobalStatistics.admin_order_field = 'globalStatistics'
+
+    def jsonDomesticStatistics(self, obj):
+        return self.to_json(obj.domesticStatistics)
+    jsonDomesticStatistics.short_description = '国内疫情'
+    jsonDomesticStatistics.admin_order_field = 'domesticStatistics'
+
+    def jsonInternationalStatistics(self, obj):
+        return self.to_json(obj.internationalStatistics)
+    jsonInternationalStatistics.short_description = '国际疫情'
+    jsonInternationalStatistics.admin_order_field = 'internationalStatistics'
+
+    def to_json(self, data):
+        try:
+            data = json.loads(data)
+        except:
+            return
+        result = []
+        for k, v in sorted(data.items()):
+            result.append(format_html('{}: {}', k, v))
+        return mark_safe(format_html(
+            '<pre>{}</pre>', format_html('<br>'.join(result))))
+
 @admin.register(models.City)
 class CityAdmin(BaseAdmin):
 

ncovapi/admin.py renamed to django_covid19/admin.py

@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class DjangoCovid19Config(AppConfig):
+    name = 'django_covid19'
+    verbose_name = '新冠肺炎疫情'
+
+    def ready(self):
+        import django_covid19.signals

django_covid19/apps.py

@@ -0,0 +1,30 @@
+import os
+import sys
+import django_covid19
+
+app_dir = os.path.dirname(django_covid19.__file__)
+sys.path.insert(0, os.path.join(app_dir, 'spider'))
+
+from nCoV.spiders.dxy import DXYSpider
+from scrapy.crawler import CrawlerProcess
+from scrapy.utils.project import get_project_settings
+from django.core.management.base import BaseCommand
+
+class Scraper:
+    def __init__(self):
+        settings_file_path = 'nCoV.settings'
+        os.environ.setdefault('SCRAPY_SETTINGS_MODULE', settings_file_path)
+        self.process = CrawlerProcess(get_project_settings())
+        self.spider = DXYSpider
+
+    def run_spiders(self):
+        self.process.crawl(self.spider)
+        self.process.start()
+
+class Command(BaseCommand):
+
+    help = 'Crawl data from DingXiangYuan.'
+
+    def handle(self, *args, **options):
+        scraper = Scraper()
+        scraper.run_spiders()

ncovapi/filters.py renamed to django_covid19/filters.py

@@ -0,0 +1,3 @@
+from django.conf import settings
+
+CACHE_PAGE_TIMEOUT = getattr(settings, 'CACHE_PAGE_TIMEOUT', 24*60*60)

ncovapi/__init__.py renamed to django_covid19/management/__init__.py

@@ -1,8 +1,8 @@
 from django.dispatch import receiver
 from django.core.cache import cache
 from django.core.signals import request_finished
- 
+
 @receiver(request_finished)
-def my_callback(sender, **kwargs):
+def spider_callback(sender, **kwargs):
     if cache.get('crawled'):
         cache.clear()

ncovapi/migrations/__init__.py renamed to django_covid19/management/commands/__init__.py

@@ -8,7 +8,7 @@
 import scrapy
 
 from scrapy_djangoitem import DjangoItem
-from ncovapi import models
+from covid19 import models
 
 
 class StatisticsItem(DjangoItem):

django_covid19/management/commands/crawl.py

@@ -8,6 +8,7 @@
 import os
 import json
 import sqlite3
+from uuid import uuid4
 
 from django.core.cache import cache
 
@@ -17,6 +18,8 @@
 class NcovPipeline(object):
 
     def open_spider(self, spider):
+        spider.object_id = uuid4().hex
+        cache.set('running_spider_id', spider.object_id)
         spider.crawled = 0
 
     def process_item(self, item, spider):
@@ -46,3 +49,4 @@ def process_item(self, item, spider):
 
     def close_spider(self, spider):
         cache.set('crawled', spider.crawled)
+        cache.delete('running_spider_id')

spider/nCoV/__init__.py renamed to django_covid19/migrations/__init__.py

@@ -1,16 +1,5 @@
 # -*- coding: utf-8 -*-
 
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath('.')))
-
-print(sys.path)
-os.environ['DJANGO_SETTINGS_MODULE'] = 'covid19.settings'
-
-import django
-django.setup()
-
 # Scrapy settings for nCoV project
 #
 # For simplicity, this file contains only settings considered important or