Use Redis for Celery Result Backend (#348)

* Disable Celery worker mingling

* Enable Redis result backend

* Setup Redis in dev container

* Ensure tasks are invoked correctly in tests

* Add Redis to commit tests

* Remove unnecessary Redis memory optimization

* Use REDIS_URL environment variable

* Pin to Redis 7.0

* Pin to PostgreSQL 14.10

* Pin to Redis 7

Author: aaron-lane

.devcontainer/devcontainer.json

@@ -9,7 +9,12 @@
 
   // Features to add to the dev container. More info: https://containers.dev/features.
   "features": {
-    "ghcr.io/devcontainers-contrib/features/postgres-asdf:1": {}
+    "ghcr.io/devcontainers-contrib/features/postgres-asdf:1": {
+      "version": "14.10"
+    },
+    "ghcr.io/itsmechlark/features/redis-server:1": {
+      "version": "7"
+    }
   },
 
   // Use portAttributes to map port numbers to default attributes.

.env.sample

@@ -1,6 +1,5 @@
 ADMIN=NAME,[email protected]
 ALLOWED_HOSTS=rebootcanadadb.herokuapp.com,rebootcanada.herokuapp.com,localhost,0.0.0.0,127.0.0.1
-CELERY_RESULT_BACKEND="rpc://"
 CLOUDAMQP_APIKEY=
 CLOUDAMQP_URL=
 CSRF_TRUSTED_ORIGINS=.preview.app.github.dev
@@ -14,6 +13,7 @@ EMAIL_HOST=smtp.gmail.com
 EMAIL_HOST_DISPLAY_NAME=
 EMAIL_HOST_USER=
 EMAIL_HOST_PASSWORD=
+REDIS_URL="redis://localhost:6379/0"
 # SECRET_KEY: Check out https://www.miniwebtool.com/django-secret-key-generator/ for generating new key
 SECRET_KEY=abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)
 SECURE_SSL_REDIRECT=False

.github/workflows/deployment.yml

@@ -52,7 +52,6 @@ jobs:
     env:
       ADMIN: NAME,[email protected]
       ALLOWED_HOSTS: localhost,0.0.0.0,127.0.0.1
-      CELERY_RESULT_BACKEND: "rpc://"
       CSRF_TRUSTED_ORIGINS: ""
       DB_NAME: reboot
       DB_USER: root
@@ -62,12 +61,13 @@ jobs:
       EMAIL_HOST: smtp.gmail.com
       EMAIL_HOST_USER: ""
       EMAIL_HOST_PASSWORD: ""
+      REDIS_URL: "redis://localhost:6379/0"
       SECRET_KEY: ${{ github.run_id }}-${{ github.run_attempt }}
       SECURE_SSL_REDIRECT: False
 
     services:
       postgres:
-        image: postgres:13
+        image: postgres:14.10
         env:
           POSTGRES_USER: ${{ env.DB_USER }}
           POSTGRES_PASSWORD: ${{ env.DB_PASSWORD }}
@@ -76,6 +76,12 @@ jobs:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: --entrypoint redis-server --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
     - name: Checkout the Git repository
       uses: actions/checkout@v3

Makefile

@@ -68,7 +68,7 @@ migrate:
 
 .PHONY: celery
 celery:
-	celery worker -A reboot --without-gossip --without-heartbeat
+	celery worker -A reboot --without-heartbeat --without-gossip --without-mingle
 
 .PHONY: clean
 clean:

Procfile

@@ -1,2 +1,2 @@
-worker: celery worker -A reboot --without-gossip --without-heartbeat
+worker: celery worker -A reboot --without-heartbeat --without-gossip --without-mingle
 web: gunicorn reboot.wsgi --log-level info

app/worker/tasks/importers/test__init__.py

@@ -95,8 +95,7 @@ def test_historical_data_importer(self) -> None:
         })
         csvvalue = csvfile.getvalue().splitlines()
 
-        importers.historical_data_importer(
-            csvpath=csvvalue)
+        importers.historical_data_importer.apply(args=[csvvalue])
 
         got_donor = Donor.objects.get(donor_name="Example Danger Donor")
 

app/worker/tasks/test__init__.py

@@ -20,8 +20,8 @@ def test_receiptor_single_file(self) -> None:
         queryset_json = serializers.serialize(format="json", queryset=queryset)
         total_count = Donation.objects.count()
 
-        response = tasks.receiptor(
-            queryset=queryset_json, total_count=total_count)
+        result = tasks.receiptor.apply(args=[queryset_json, total_count])
+        response = result.get()
 
         self.assertEqual(first=response.status_code, second=200)
         self.assertEqual(
@@ -34,8 +34,8 @@ def test_receiptor_multiple_files(self) -> None:
         queryset_json = serializers.serialize(format="json", queryset=queryset)
         total_count = Donation.objects.count()
 
-        response = tasks.receiptor(
-            queryset=queryset_json, total_count=total_count)
+        result = tasks.receiptor.apply(args=[queryset_json, total_count])
+        response = result.get()
 
         self.assertEqual(first=response.status_code, second=200)
         self.assertEqual(

app/worker/tasks/test_exporter.py

@@ -28,8 +28,8 @@ def test_exporter(self) -> None:
         qs = serialize(format="json", queryset=queryset)
         total_count = len(queryset)
 
-        response = exporter(file_name=file_name, qs=qs,
-                            total_count=total_count)
+        result = exporter.apply(args=[file_name, qs, total_count])
+        response = result.get()
         content_type = response["Content-Type"]
 
         self.assertEqual(first=response.status_code, second=200)

reboot/celeryconfig.py

@@ -1,6 +1,6 @@
 import ssl
-from decouple import config
 
+from decouple import config
 
 broker_url = config('CLOUDAMQP_URL', default='amqp://guest@localhost//')
 broker_connection_timeout = 30
@@ -10,7 +10,7 @@
 worker_prefetch_multiplier = 1
 worker_concurrency = 10
 accept_content = ['json', 'pickle']
-result_backend = config("CELERY_RESULT_BACKEND", default=broker_url)
+result_backend = config("REDIS_URL")
 task_serializer = 'pickle'
 result_serializer = 'pickle'
 

requirements.txt

@@ -1,8 +1,8 @@
 # Commented requirements indicate the latest patches of the earliest releases that support Python 3.9.
 
 # https://pypi.org/project/celery/5.1.2/
-# celery==5.1.2
-celery==4.4.7
+# celery[redis]==5.1.2
+celery[redis]==4.4.7
 # https://pypi.org/project/Django/2.2.28/
 Django==2.2.28
 # https://pypi.org/project/dj-database-url/1.0.0/