add celery

Author: mominur-helios

.gitignore

@@ -15,3 +15,5 @@ env
 .env
 node_modules
 yarn.lock
+
+tasks_logs/

apps/common/migrations/0001_initial.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.5 on 2023-10-08 08:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Product',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=100)),
+                ('info', models.CharField(default='', max_length=100)),
+                ('price', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+    ]

celerybeat-schedule

@@ -52,6 +52,8 @@
     "apps.charts", 
     "apps.tables",
     "apps.tasks",
+
+    "django_celery_results",
 ]
 
 MIDDLEWARE = [
@@ -144,3 +146,23 @@
 # https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+
+
+CELERY_SCRIPTS_DIR        = os.path.join(BASE_DIR, "tasks_scripts" )
+
+CELERY_LOGS_URL           = "/tasks_logs/"
+CELERY_LOGS_DIR           = os.path.join(BASE_DIR, "tasks_logs"    )
+
+CELERY_BROKER_URL         = os.environ.get("CELERY_BROKER", "redis://localhost:6379")
+CELERY_RESULT_BACKEND     = os.environ.get("CELERY_BROKER", "redis://localhost:6379")
+
+CELERY_TASK_TRACK_STARTED = True
+CELERY_TASK_TIME_LIMIT    = 30 * 60
+CELERY_CACHE_BACKEND      = "django-cache"
+CELERY_RESULT_BACKEND     = "django-db"
+CELERY_RESULT_EXTENDED    = True
+CELERY_RESULT_EXPIRES     = 60*60*24*30 # Results expire after 1 month
+CELERY_ACCEPT_CONTENT     = ["json"]
+CELERY_TASK_SERIALIZER    = 'json'
+CELERY_RESULT_SERIALIZER  = 'json'

core/settings.py

@@ -16,6 +16,8 @@
 """
 from django.contrib import admin
 from django.urls import include, path
+from django.conf import settings
+from django.conf.urls.static import static
 
 urlpatterns = [
     path("", include("home.urls")),
@@ -26,3 +28,4 @@
     path("tables/", include("apps.tables.urls")),
     path("tasks/", include("apps.tasks.urls")),
 ]
+urlpatterns += static(settings.CELERY_LOGS_URL, document_root=settings.CELERY_LOGS_DIR)

core/urls.py

@@ -0,0 +1,18 @@
+## Celery docs
+
+- Install the dependencies
+```bash
+$ pip install -r requirements.txt
+```
+
+- In the base directory inside `tasks_scripts` folder you need to write your scripts file.
+- Run the celery command from the CLI.
+```bash
+$ celery -A home worker -l info -B
+```
+- Run the django server
+```bash
+$ python manage.py runserver
+```
+- You will see a new route `Tasks` in the sidebar.
+- You can start and cancel any task from the UI.

docs/CELERY.md

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+"""
+Copyright (c) 2019 - present AppSeed.us
+"""
+
+import os
+
+from celery import Celery
+
+if os.environ.get('DJANGO_SETTINGS_MODULE'):
+
+    app = Celery('core')
+
+    # Using a string here means the worker doesn't have to serialize
+    # the configuration object to child processes.
+    # - namespace='CELERY' means all celery-related configuration keys
+    #   should have a `CELERY_` prefix.
+    app.config_from_object('django.conf:settings', namespace='CELERY')
+
+    # Load task modules from all registered Django apps.
+    app.autodiscover_tasks()
+
+else:
+    print(' ')
+    print('Celery Configuration ERROR: ') 
+    print('  > "DJANGO_SETTINGS_MODULE" not set in environment (value in manage.py)')
+    print('  Hint: export DJANGO_SETTINGS_MODULE=project.settings ') 
+    print(' ')

home/celery.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2023-10-08 08:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='FileInfo',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.URLField()),
+                ('info', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

home/migrations/0001_initial.py

@@ -1,3 +1,10 @@
 from django.db import models
 
 # Create your models here.
+
+class FileInfo(models.Model):
+    path = models.URLField()
+    info = models.CharField(max_length=255)
+
+    def __str__(self):
+        return self.path

home/models.py

@@ -0,0 +1,85 @@
+import os, time, subprocess
+import datetime
+from os import listdir
+from os.path import isfile, join
+
+from .celery import app
+from celery.contrib.abortable import AbortableTask
+from django_celery_results.models import TaskResult
+
+from django.contrib.auth.models import User
+from django.conf import settings
+from celery.exceptions import Ignore, TaskError
+
+
+def get_scripts():
+    """
+    Returns all scripts from 'ROOT_DIR/celery_scripts'
+    """
+    raw_scripts = []
+    scripts     = []
+    ignored_ext = ['db', 'txt']
+
+    try:
+        raw_scripts = [f for f in listdir(settings.CELERY_SCRIPTS_DIR) if isfile(join(settings.CELERY_SCRIPTS_DIR, f))]
+    except Exception as e:
+        return None, 'Error CELERY_SCRIPTS_DIR: ' + str( e ) 
+
+    for filename in raw_scripts:
+
+        ext = filename.split(".")[-1]
+        if ext not in ignored_ext:
+           scripts.append( filename )
+
+    return scripts, None           
+
+def write_to_log_file(logs, script_name):
+    """
+    Writes logs to a log file with formatted name in the CELERY_LOGS_DIR directory.
+    """
+    script_base_name = os.path.splitext(script_name)[0]  # Remove the .py extension
+    current_time = datetime.datetime.now().strftime("%y%m%d-%H%M%S")
+    log_file_name = f"{script_base_name}-{current_time}.log"
+    log_file_path = os.path.join(settings.CELERY_LOGS_DIR, log_file_name)
+    
+    with open(log_file_path, 'w') as log_file:
+        log_file.write(logs)
+    
+    return log_file_path
+
+@app.task(bind=True, base=AbortableTask)
+def execute_script(self, data: dict):
+    """
+    This task executes scripts found in settings.CELERY_SCRIPTS_DIR and logs are later generated and stored in settings.CELERY_LOGS_DIR
+    :param data dict: contains data needed for task execution. Example `input` which is the script to be executed.
+    :rtype: None
+    """
+    script = data.get("script")
+    args   = data.get("args")
+
+    print( '> EXEC [' + script + '] -> ('+args+')' ) 
+
+    scripts, ErrInfo = get_scripts()
+
+    if script and script in scripts:
+        # Executing related script
+        script_path = os.path.join(settings.CELERY_SCRIPTS_DIR, script)
+        process = subprocess.Popen(
+            f"python {script_path} {args}", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        time.sleep(8)
+
+        exit_code = process.wait()
+        error = False
+        status = "STARTED"
+        if exit_code == 0:  # If script execution successfull
+            logs = process.stdout.read().decode()
+            status = "SUCCESS"
+        else:
+            logs = process.stderr.read().decode()
+            error = True
+            status = "FAILURE"
+
+
+        log_file = write_to_log_file(logs, script)
+
+        return {"logs": logs, "input": script, "error": error, "output": "", "status": status, "log_file": log_file}

home/tasks.py

@@ -0,0 +1,19 @@
+from django import template
+import os
+from urllib.parse import quote
+
+register = template.Library()
+
+@register.filter
+def file_extension(value):
+    _, extension = os.path.splitext(value)
+    return extension.lower()
+
+
+@register.filter
+def encoded_file_path(path):
+    return path.replace('/', '%slash%')
+
+@register.filter
+def encoded_path(path):
+    return path.replace('\\', '/')

home/templatetags/__init__.py

@@ -0,0 +1,65 @@
+# -*- encoding: utf-8 -*-
+"""
+Copyright (c) 2019 - present AppSeed.us
+"""
+
+import json
+import os
+
+from django import template
+from django.conf import settings
+
+register = template.Library()
+
+def date_format(date):
+    """
+    Returns a formatted date string
+    Format:  `Year-Month-Day-Hour-Minute-Second`
+    Example: `2022-10-10-00-20-33`
+    :param date datetime: Date object to be formatted
+    :rtype: str
+    """
+    try:
+        return date.strftime(r'%Y-%m-%d-%H-%M-%S')
+    except:
+        return date
+
+register.filter("date_format", date_format)
+
+def get_result_field(result, field: str):
+    """
+    Returns a field from the content of the result attibute in result 
+    Example: `result.result['field']`
+    :param result AbortableAsyncResult: Result object to get field from
+    :param field str: Field to return from result object
+    :rtype: str
+    """
+    result = json.loads(result.result)
+    if result:
+        return result.get(field)
+
+register.filter("get_result_field", get_result_field)
+
+
+
+def log_file_path(path):
+    file_path = path.split("tasks_logs")[1]
+    return file_path
+
+register.filter("log_file_path", log_file_path)
+
+
+def log_to_text(path):
+    path = path.lstrip('/')
+
+    full_path = os.path.join(settings.CELERY_LOGS_DIR, path)
+
+    try:
+        with open(full_path, 'r') as file:
+            text = file.read()
+        
+        return text
+    except:
+        return 'NO LOGS'
+
+register.filter("log_to_text", log_to_text)

home/templatetags/file_extension.py

@@ -0,0 +1,12 @@
+from django import template
+from home.models import FileInfo
+
+register = template.Library()
+
+@register.filter
+def info_value(path):
+    file_info = FileInfo.objects.filter(path=path)
+    if file_info.exists():
+        return file_info.first().info
+    else:
+        return ""

home/templatetags/formats.py

@@ -5,4 +5,12 @@
 urlpatterns = [
     path("", views.index, name="index"),
     path("starter/", views.starter, name="starter"),
+
+    # Celery
+    path('tasks/', views.tasks, name="tasks"),
+    path('tasks/run/<str:task_name>'  , views.run_task,    name="run-task"    ),
+    path('tasks/cancel/<str:task_id>' , views.cancel_task, name="cancel-task" ),
+    path('tasks/output/'              , views.task_output, name="task-output" ),
+    path('tasks/log/'                 , views.task_log,    name="task-log"    ), 
+    path('download-log-file/<str:file_path>/', views.download_log_file, name='download_log_file'),
 ]