Updated parsing and fixed views

Fixed admin
Removed pyppeteer mentions
Added crochet and celery

Author: dhvcc

.envs.example/docker.env.example

@@ -1,8 +1,12 @@
 # Used in docker compose
 
-# ##########
-# DATABASE #
-# ##########
+# ######
+# DATA #
+########
+
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=sora
 
 DATABASE_HOST=db
 DATABASE_USER=postgres
@@ -11,6 +15,10 @@ DATABASE_PASSWORD=postgres
 DATABASE_PORT=5432
 DATABASE_NAME=sora
 
+REDIS_URL=redis://redis:6379
+
+ELASTICSEARCH_HOST=elasticsearch:9200
+
 # ########
 # DJANGO #
 # ########
@@ -20,22 +28,6 @@ SECRET_KEY=DevServer
 # Comma-separated list
 ALLOWED_HOSTS=*
 PAGE_SIZE=20
-DJANGO_COLORS="success=green,bold;notice=cyan,bold;error=red,bold"
-
-# ########
-# DOCKER #
-# ########
-
-# Postgres
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=postgres
-POSTGRES_DB=sora
-
-# #######
-# REDIS #
-# #######
-
-REDIS_URL=redis://redis:6379
 
 # #######
 # Other #

.envs.example/local.env.example

@@ -1,8 +1,8 @@
 # Your local config to develop without docker
 
-# ##########
-# DATABASE #
-# ##########
+# ######
+# DATA #
+# ######
 
 DATABASE_HOST=localhost
 DATABASE_USER=postgres
@@ -12,6 +12,12 @@ DATABASE_PASSWORD=postgres
 DATABASE_PORT=8882
 DATABASE_NAME=sora
 
+# Port from docker-compose
+REDIS_URL=redis://localhost:8883
+
+# Port from docker-compose
+ELASTICSEARCH_HOST=localhost:9200
+
 # ########
 # DJANGO #
 # ########
@@ -21,14 +27,6 @@ SECRET_KEY=DevServer
 # Comma-separated list
 ALLOWED_HOSTS=*
 PAGE_SIZE=20
-DJANGO_COLORS="success=green,bold;notice=cyan,bold;error=red,bold"
-
-# #######
-# REDIS #
-# #######
-
-# Port from docker-compose
-REDIS_URL=redis://localhost:8883
 
 # #######
 # Other #

apps/parse/admin.py

@@ -54,7 +54,7 @@ class MangaAdmin(BaseAdmin, ImagePreviewMixin, admin.ModelAdmin):
         "status",
         "genre_list",
     )
-    list_filter = ("genres", SourceFilter)
+    # list_filter = ("genres", SourceFilter)
 
     def custom_title(self, obj: Manga):
         concat = f"{obj.title}{', ' + obj.year if obj.year else ''}"
@@ -98,4 +98,4 @@ class ChapterAdmin(BaseAdmin, admin.ModelAdmin):
     }
 
     def manga_name(self, obj):
-        return obj.manga_set.first().title
+        return obj.manga.title

apps/parse/api/views.py

@@ -32,9 +32,9 @@ def retrieve(self, _, pk, *args, **kwargs):
         manga = self.get_fast_manga(pk)
         try:
             if needs_update(manga["updated_detail"]):
-                run_parser(DETAIL_PARSER, manga.source, manga["source_url"])
+                run_parser(DETAIL_PARSER, manga["source"], manga["source_url"])
         except Exception as e:
-            return format_error_response("Errors occured during parsing" + str(e))
+            return format_error_response("Errors occured during parsing " + str(e))
         return get_fast_response(manga)
 
     def list(self, request):
@@ -67,11 +67,11 @@ def chapters_list(self, _, pk):
         manga: Manga = Manga.objects.prefetch_related("chapters").get(pk=pk)
 
         try:
-            if needs_update(manga["updated_detail"]):
-                run_parser(DETAIL_PARSER, manga.source, manga["source_url"])
-                run_parser(CHAPTER_PARSER, manga.source, manga["source_url"])
+            if needs_update(manga.updated_detail.isoformat()):
+                run_parser(DETAIL_PARSER, manga.source, manga.source_url)
+                run_parser(CHAPTER_PARSER, manga.source, manga.source_url)
         except Exception as e:
-            return format_error_response("Errors occured during parsing" + str(e))
+            return format_error_response("Errors occured during parsing " + str(e))
         return get_fast_response(
             list(manga.chapters.order_by("-volume", "-number").values(*CHAPTER_FIELDS))
         )

apps/parse/management/commands/crawl.py

@@ -0,0 +1,51 @@
+import logging
+import sys
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from apps.parse.const import (
+    CATALOGUE_NAMES,
+    CHAPTER_PARSER,
+    DETAIL_PARSER,
+    IMAGE_PARSER,
+    PARSER_TYPES,
+)
+from apps.parse.scrapy.utils import run_parser
+from apps.parse.utils import mute_logger_stdout
+
+logger = logging.getLogger("management")
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "type",
+            type=str,
+            choices=PARSER_TYPES,
+            help="which type of data to parse",
+        )
+        parser.add_argument(
+            "catalogue",
+            type=str,
+            default="readmanga",
+            choices=CATALOGUE_NAMES,
+            help="parser to use which respresents a website source",
+        )
+        parser.add_argument(
+            "--url",
+            type=str,
+            required=sys.argv[2] in [DETAIL_PARSER, CHAPTER_PARSER, IMAGE_PARSER],
+            help="A link which to parse (detail/chapter/rss url)",
+        )
+
+    def handle(self, *args, **options):
+        mute_logger_stdout("scrapy", "elasticsearch", "asyncio", "protego", "urllib3", "requests")
+        try:
+            catalogue_name: str = options["catalogue"]
+            logger.info("Running parser")
+            run_parser(options["type"], catalogue_name, url=options["url"])
+            logger.info("Finished parsing")
+        except (AttributeError, KeyError):
+            logger.error(f"Can't find Catalogue [{catalogue_name}]")
+        except Exception:
+            logger.error(f"Some errors occured in the parser {catalogue_name}")

apps/parse/management/commands/parse.py

@@ -47,5 +47,5 @@ def handle(self, *args, **options):
             logger.info("Finished parsing")
         except (AttributeError, KeyError):
             logger.error(f"Can't find Catalogue [{catalogue_name}]")
-        except Exception:
-            logger.error(f"Some errors occured in the parser {catalogue_name}")
+        except Exception as e:
+            logger.error(f"Some errors occured in the parser {catalogue_name} {e}")

apps/parse/migrations/0026_auto_20211112_1448.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.1 on 2021-11-12 14:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('parse', '0025_chapters_to_fk'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='chapter',
+            name='manga',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chapters', to='parse.manga'),
+        ),
+    ]

apps/parse/readmanga/chapter.py

@@ -17,8 +17,8 @@ class ReadmangaChapterSpider(scrapy.Spider):
         "ITEM_PIPELINES": {"apps.parse.readmanga.pipelines.ReadmangaChapterPipeline": 300}
     }
 
-    def __init__(self, *, url: str):
-        self.start_urls = [url]
+    def __init__(self, *args, url: str):
+        super().__init__(*args, start_urls=[url])
 
     def parse(self, response: XmlResponse) -> List[MangaChapterItem]:
         chapters = []

apps/parse/readmanga/detail.py

@@ -17,8 +17,8 @@
 class ReadmangaDetailSpider(scrapy.Spider):
     name = "readmanga_detail"
 
-    def __init__(self, *, url: str):
-        self.start_urls = [url]
+    def __init__(self, *args, url: str):
+        super().__init__(*args, start_urls=[url])
 
     def parse(self, response: HtmlResponse):
         year = response.xpath(YEAR_TAG).extract_first("")

apps/parse/readmanga/images.py

@@ -15,12 +15,12 @@ class ReadmangaImageSpider(scrapy.Spider):
         "ITEM_PIPELINES": {"apps.parse.readmanga.pipelines.ReadmangaImagePipeline": 300}
     }
 
-    def __init__(self, *, url: str):
-        self.start_urls = [url]
-        self.redis_client = init_redis_client()
+    def __init__(self, *args, url: str, **kwargs):
+        super().__init__(*args, **kwargs, start_urls=[url], redis_client=init_redis_client())
 
     def parse(self, response: HtmlResponse):
         images = re.search(r"rm_h.initReader\(.*(\[{2}.*\]{2}).*\)", response.text)
+        image_links = []
         if images:
             image_links = [
                 "".join(image[:COUNT_LINK_ELEMENTS])

apps/parse/readmanga/pipelines.py

@@ -28,7 +28,7 @@ def bulk_get_or_create(cls: Type[BaseModel], names: List[str]) -> Tuple:
 class ReadmangaImagePipeline:
     @staticmethod
     def process_item(item: Dict[str, List[str]], spider: ReadmangaImageSpider):
-        url, images = item.items()[0]
+        url, images = next(iter(item.items()))
         spider.redis_client.delete(url)
         spider.redis_client.expire(url, IMAGE_UPDATE_FREQUENCY)
         spider.redis_client.rpush(url, *images)

apps/parse/scrapy/utils.py

@@ -1,17 +1,14 @@
-import os
-
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.project import get_project_settings
 
 from apps.parse.const import CATALOGUES
 
 
-def run_parser(parser_type: str, catalogue_name: str = "readmanga", *args, **kwargs):
+def run_parser(parser_type: str, catalogue_name: str = "readmanga", url: str = None):
     catalogue = CATALOGUES[catalogue_name]
     spider = catalogue["parsers"][parser_type]
 
-    os.environ.setdefault("SCRAPY_SETTINGS_MODULE", catalogue["settings"])
     process = CrawlerProcess(get_project_settings())
 
-    process.crawl(spider, *args, **kwargs)
+    process.crawl(spider, url=url)
     process.start()

docker-compose.yml

@@ -48,6 +48,7 @@ services:
 
   elasticsearch:
     image: elasticsearch:7.14.2
+    restart: unless-stopped
     environment:
       - discovery.type=single-node
     volumes:

docker-entrypoint.sh

@@ -22,6 +22,14 @@ echo
 echo "Running migrations"
 ./manage.py migrate --no-input
 
+until curl --output /dev/null --silent --head --fail "http://$ELASTICSEARCH_HOST"; do
+    echo >&2 "Postgres is unavailable - sleeping"
+    sleep 1
+done
+echo
+echo "Rebuilding index"
+./manage.py search_index --rebuild
+
 echo "Running the server on port $PORT"
 
 core_count=$(grep 'cpu[0-9]' /proc/stat | wc -l)

manga_reader/settings.py

@@ -19,6 +19,7 @@
 BASE_DIR = Path(__file__).resolve().parent.parent
 ROOT_URLCONF = "manga_reader.urls"
 WSGI_APPLICATION = "manga_reader.wsgi.application"
+WEBDRIVER_PATH = os.getenv("WEBDRIVER_PATH", None)
 
 ############
 # Security #
@@ -50,10 +51,13 @@
     "apps.core.apps.CoreConfig",
     "django_extensions",
     "django.contrib.postgres",
-    "silk",
     "django_elasticsearch_dsl",
 ]
 
+ELASTICSEARCH_DSL = {
+    "default": {"hosts": os.getenv("ELASTICSEARCH_HOST", "localhost:92000")},
+}
+
 #########
 # ADMIN #
 #########
@@ -278,29 +282,14 @@
 )
 ignore_logger("django.security.DisallowedHost")
 
-#############
-# Pyppeteer #
-#############
-
-WEBDRIVER_PATH = os.getenv("WEBDRIVER_PATH", None)
-DEFAULT_LAUNCH_ARGS = {
-    "headless": True,
-    "args": ["--no-sandbox", "--disable-setuid-sandbox"],
-    "executablePath": WEBDRIVER_PATH,
-}
-
-
 ########
 # Silk #
 ########
 
 if DEBUG:
+    INSTALLED_APPS.append("silk")
     MIDDLEWARE.append("silk.middleware.SilkyMiddleware")
 
-ELASTICSEARCH_DSL = {
-    "default": {"hosts": "localhost:9200"},
-}
-
 ##########
 # Logging #
 ##########

manga_reader/urls.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.conf.urls import include
 from django.contrib import admin
 from django.urls import path, re_path
@@ -10,6 +11,8 @@
 
 urlpatterns = [
     path("api/", include(apipatterns)),
-    path("silk/", include("silk.urls", namespace="silk")),
     re_path(r"^(?!api)\w*?", admin.site.urls),
 ]
+
+if settings.DEBUG:
+    urlpatterns.append(path("silk/", include("silk.urls", namespace="silk")))