Setup Celery

apps/common/management/commands/run_celery_dev.py

@@ -0,0 +1,29 @@
+import shlex
+import os
+import subprocess
+
+from django.core.management.base import BaseCommand
+from django.utils import autoreload
+from main.celery import CeleryQueue
+
+
+WORKER_STATE_DIR = '/var/run/celery'
+
+CMD = (
+    f"celery -A main worker -Q {','.join(CeleryQueue.ALL_QUEUES)} -E --concurrency=2 -l info"
+)
+
+
+def restart_celery(*args, **kwargs):
+    kill_worker_cmd = 'pkill -9 celery'
+    subprocess.call(shlex.split(kill_worker_cmd))
+    subprocess.call(shlex.split(CMD))
+
+
+class Command(BaseCommand):
+
+    def handle(self, *args, **options):
+        self.stdout.write('Starting celery worker with autoreload...')
+        if not os.path.exists(WORKER_STATE_DIR):
+            os.makedirs(WORKER_STATE_DIR)
+        autoreload.run_with_reloader(restart_celery, args=None, kwargs=None)

gh-docker-compose.yml

@@ -8,6 +8,14 @@ services:
       POSTGRES_PASSWORD: postgres
       POSTGRES_DB: qb
 
+  redis:
+    image: redis:latest
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   backend:
     image: $DOCKER_IMAGE_BACKEND
     environment:
@@ -25,16 +33,22 @@ services:
       SESSION_COOKIE_DOMAIN: localhost
       CSRF_COOKIE_DOMAIN: localhost
       DJANGO_ALLOWED_HOST: '*'
+      # Database
       DJANGO_DB_NAME: qb
       DJANGO_DB_USER: postgres
       DJANGO_DB_PASSWORD: postgres
       DJANGO_DB_PORT: 5432
       DJANGO_DB_HOST: db
       DJANGO_CORS_ORIGIN_REGEX_WHITELIST: localhost
+      # Redis
+      CELERY_REDIS_URL: redis://redis:6379/1
+      DJANGO_CACHE_REDIS_URL: redis://redis:6379/2
+      TEST_DJANGO_CACHE_REDIS_URL: redis://redis:6379/12
       # EMAIL
       EMAIL_FROM: [email protected]
     volumes:
       - ./coverage/:/code/coverage/
       - ./ci-share/:/ci-share/
     depends_on:
       - db
+      - redis

main/celery.py

@@ -0,0 +1,47 @@
+import os
+import celery
+
+from django.conf import settings
+
+from main import sentry
+
+
+class Celery(celery.Celery):
+    def on_configure(self):
+        if settings.SENTRY_ENABLED:
+            sentry.init_sentry(**settings.SENTRY_CONFIG)
+
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'main.settings')
+
+app = Celery('main')
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+app.autodiscover_tasks(['main'])
+
+
+# This is used for ECS Cluster (Each queue needs it's own clusters)
+class CeleryQueue():
+    DEFAULT = 'CELERY-DEFAULT-QUEUE'
+    EXPORT_HEAVY = 'CELERY-EXPORT-HEAVY-QUEUE'
+
+    ALL_QUEUES = (
+        DEFAULT,
+        EXPORT_HEAVY,
+    )
+
+
+app.conf.task_default_queue = CeleryQueue.DEFAULT
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

main/sentry.py

@@ -2,6 +2,8 @@
 from django.conf import settings
 from strawberry.permission import BasePermission
 from sentry_sdk.integrations.django import DjangoIntegration
+from sentry_sdk.integrations.celery import CeleryIntegration
+from sentry_sdk.integrations.redis import RedisIntegration
 from sentry_sdk.integrations.logging import ignore_logger
 
 IGNORED_ERRORS = [
@@ -18,6 +20,8 @@
 def init_sentry(app_type, tags={}, **config):
     integrations = [
         DjangoIntegration(),
+        CeleryIntegration(),
+        RedisIntegration(),
     ]
     sentry_sdk.init(
         **config,

main/settings.py

@@ -28,6 +28,11 @@
     DJANGO_DB_HOST=str,
     DJANGO_DB_PORT=int,
     DJANGO_CORS_ORIGIN_REGEX_WHITELIST=(list, []),
+    # Redis
+    CELERY_REDIS_URL=str,
+    DJANGO_CACHE_REDIS_URL=str,
+    # -- For running test (Optional)
+    TEST_DJANGO_CACHE_REDIS_URL=(str, None),
     # Static, Media configs
     DJANGO_STATIC_URL=(str, '/static/'),
     DJANGO_MEDIA_URL=(str, '/media/'),
@@ -123,6 +128,7 @@
     'apps.user',
     'apps.project',
     'apps.questionnaire',
+    'apps.export',
     # 'apps.qbank',
 ]
 
@@ -391,18 +397,34 @@
 DEFAULT_PAGINATION_LIMIT = 50
 MAX_PAGINATION_LIMIT = 100
 
+# Redis
+CELERY_REDIS_URL = env('CELERY_REDIS_URL')
+DJANGO_CACHE_REDIS_URL = env('DJANGO_CACHE_REDIS_URL')
+TEST_DJANGO_CACHE_REDIS_URL = env('TEST_DJANGO_CACHE_REDIS_URL')
+
 # Caches
 CACHES = {
     'default': {
-        # XXX: Use redis if needed
-        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
-        'LOCATION': 'local-memory-01',
+        'BACKEND': 'django_redis.cache.RedisCache',
+        'LOCATION': DJANGO_CACHE_REDIS_URL,
+        'OPTIONS': {
+            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
+        },
+        'KEY_PREFIX': 'dj_cache-',
     },
     'local-memory': {
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
         'LOCATION': 'local-memory-02',
     }
 }
 
+# Celery
+CELERY_BROKER_URL = CELERY_REDIS_URL
+CELERY_RESULT_BACKEND = CELERY_REDIS_URL
+CELERY_TIMEZONE = TIME_ZONE
+CELERY_EVENT_QUEUE_PREFIX = 'qber-celery-'
+CELERY_ACKS_LATE = True
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
+
 # Misc
 ALLOW_DUMMY_DATA_SCRIPT = env('ALLOW_DUMMY_DATA_SCRIPT')