update settings, docker, actions

.envs/docker/.django

@@ -5,11 +5,11 @@ IPYTHONDIR=/app/.ipython
 
 # Redis
 # ------------------------------------------------------------------------------
-REDIS_URL=redis://localhost:6379/0
+REDIS_URL=redis://redis:6379/0
 
 # Celery
-CELERY_BROKER_URL=amqp://guest:guest@localhost:5672/
-CELERY_RESULT_BACKEND=redis://localhost:6379/2
+CELERY_BROKER_URL=amqp://guest:guest@mq:5672/
+CELERY_RESULT_BACKEND=redis://redis:6379/2
 # Flower
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug

.envs/local/.django

@@ -5,11 +5,11 @@ IPYTHONDIR=/app/.ipython
 
 # Redis
 # ------------------------------------------------------------------------------
-REDIS_URL=redis://localhost:6379/0
+REDIS_URL=redis://redis:6379/0
 
 # Celery
-CELERY_BROKER_URL=amqp://guest:guest@localhost:5672/
-CELERY_RESULT_BACKEND=redis://localhost:6379/2
+CELERY_BROKER_URL=amqp://guest:guest@mq:5672/
+CELERY_RESULT_BACKEND=redis://redis:6379/2
 # Flower
 CELERY_FLOWER_USER=debug
 CELERY_FLOWER_PASSWORD=debug

.github/workflows/integration-tests.yml

@@ -54,6 +54,5 @@ jobs:
           --env POSTGRES_USER=inspire
           --env POSTGRES_PASSWORD=inspire
           --env POSTGRES_HOST=127.0.0.1
-          --env OPENSEARCH_HOST=127.0.0.1:9200
           ${{ inputs.image }}
           run pytest

.github/workflows/pull-request-main.yml

@@ -1,20 +1,17 @@
-name: Push main
+name: Pull request main
 
 on:
-  push:
+  pull_request_target:
     branches: [main]
 
-defaults:
-  run:
-    shell: bash
-
 jobs:
   lint:
     uses: ./.github/workflows/lint.yml
     with:
-        ref: ${{ github.ref }}
+      ref: ${{ github.ref }}
+      secrets: inherit
   test:
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/tests.yml
     with:
-      ref: ${{ github.ref }}
+      ref: ${{ github.event.pull_request.head.sha }}
     secrets: inherit

.github/workflows/push-main.yml

@@ -1,18 +1,20 @@
-name: Pull request main
+name: Push main
 
 on:
-  pull_request_target:
+  push:
     branches: [main]
-    paths-ignore: [ "docs/**" ]
+
+defaults:
+  run:
+    shell: bash
 
 jobs:
   lint:
     uses: ./.github/workflows/lint.yml
     with:
-      ref: ${{ github.ref }}
-      secrets: inherit
+        ref: ${{ github.ref }}
   test:
-    uses: ./.github/workflows/tests.yml
+    uses: ./.github/workflows/test.yml
     with:
-      ref: ${{ github.event.pull_request.head.sha }}
+      ref: ${{ github.ref }}
     secrets: inherit

compose/local/django/Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   libpq-dev
 
 ARG POETRY_VERSION
-ENV POETRY_VERSION="${POETRY_VERSION:-1.4.2}"
+ENV POETRY_VERSION="${POETRY_VERSION:-1.1.14}"
 RUN curl -sSL https://install.python-poetry.org \
   | python - --version "${POETRY_VERSION}" \
   && poetry --version
@@ -23,4 +23,4 @@ COPY poetry.lock pyproject.toml ./
 RUN poetry install --no-root
 
 COPY . ${APP_HOME}
-RUN poetry Install
+RUN poetry install

compose/production/django/Dockerfile

@@ -1,96 +1,30 @@
-
-# define an alias for the specific python version used in this file.
 FROM python:3.11.6-slim-bullseye as python
 
-# Python build stage
-FROM python as python-build-stage
-
-ARG BUILD_ENVIRONMENT=production
-
-# Install apt packages
-RUN apt-get update && apt-get install --no-install-recommends -y \
-  # dependencies for building Python packages
-  build-essential \
-  # psycopg2 dependencies
-  libpq-dev
-
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements .
-
-# Create Python Dependency and Sub-Dependency Wheels.
-RUN pip wheel --wheel-dir /usr/src/app/wheels  \
-  -r ${BUILD_ENVIRONMENT}.txt
-
-
-# Python 'run' stage
-FROM python as python-run-stage
-
-ARG BUILD_ENVIRONMENT=production
 ARG APP_HOME=/app
-
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV BUILD_ENV ${BUILD_ENVIRONMENT}
-
 WORKDIR ${APP_HOME}
 
-RUN addgroup --system django \
-    && adduser --system --ingroup django django
+ARG BUILD_ENVIRONMENT=production
+ENV BUILD_ENV ${BUILD_ENVIRONMENT}
 
+ENV PATH="/root/.local/bin:${PATH}" \
+    POETRY_VIRTUALENVS_CREATE=false \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
 
-# Install required system dependencies
 RUN apt-get update && apt-get install --no-install-recommends -y \
-  # psycopg2 dependencies
-  libpq-dev \
-  # Translations dependencies
-  gettext \
-  # cleaning up unused files
+  curl libpq-dev build-essential gettext \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
-
-# use wheels to install python dependencies
-RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
-  && rm -rf /wheels/
-
-
-COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint
-RUN sed -i 's/\r$//g' /entrypoint
-RUN chmod +x /entrypoint
-
-
-COPY --chown=django:django ./compose/production/django/start /start
-RUN sed -i 's/\r$//g' /start
-RUN chmod +x /start
-COPY --chown=django:django ./compose/production/django/celery/worker/start /start-celeryworker
-RUN sed -i 's/\r$//g' /start-celeryworker
-RUN chmod +x /start-celeryworker
-
-
-COPY --chown=django:django ./compose/production/django/celery/beat/start /start-celerybeat
-RUN sed -i 's/\r$//g' /start-celerybeat
-RUN chmod +x /start-celerybeat
-
-
-COPY --chown=django:django ./compose/production/django/celery/flower/start /start-flower
-RUN sed -i 's/\r$//g' /start-flower
-RUN chmod +x /start-flower
-
-
-# copy application code to WORKDIR
-COPY --chown=django:django . ${APP_HOME}
-
-# make django owner of the WORKDIR directory as well.
-RUN chown django:django ${APP_HOME}
+ARG POETRY_VERSION
+ENV POETRY_VERSION="${POETRY_VERSION:-1.1.14}"
+RUN curl -sSL https://install.python-poetry.org \
+  | python - --version "${POETRY_VERSION}" \
+  && poetry --version
 
-USER django
+COPY poetry.lock pyproject.toml .
+RUN poetry install --no-root
 
-RUN DATABASE_URL="" \
-  CELERY_BROKER_URL="" \
-  DJANGO_SETTINGS_MODULE="config.settings.test" \
-  python manage.py compilemessages
+COPY . ${APP_HOME}
 
-ENTRYPOINT ["/entrypoint"]
+RUN poetry install

config/settings/local.py

@@ -18,8 +18,14 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#caches
 CACHES = {
     "default": {
-        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
-        "LOCATION": "",
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": env("REDIS_URL"),
+        "OPTIONS": {
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            # Mimicing memcache behavior.
+            # https://github.com/jazzband/django-redis#memcached-exceptions-behavior
+            "IGNORE_EXCEPTIONS": True,
+        },
     }
 }
 

config/settings/production.py

@@ -14,7 +14,7 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
 SECRET_KEY = env("DJANGO_SECRET_KEY")
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
-ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["example.com"])
+ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
 
 # DATABASES
 # ------------------------------------------------------------------------------

local.yml

@@ -55,7 +55,7 @@ services:
       - redis
       - postgres
     ports: []
-    command: celery -A config.celery_app beat -l INFO
+    command: celery -A config.celery_app worker -l INFO
     networks:
       - djangonetwork
 
@@ -67,9 +67,19 @@ services:
       - redis
       - postgres
     ports: []
-    command: celery -A config.celery_app worker -l INFO
+    command: celery -A config.celery_app beat -l INFO
     networks:
       - djangonetwork
+
+  mq:
+    image: rabbitmq:3.9.11-management
+    restart: 'always'
+    container_name: backoffice_local_mq
+    ports:
+      - '5672:5672'
+    networks:
+      - djangonetwork
+
 
 networks:
   djangonetwork:

production.yml

@@ -3,7 +3,6 @@ version: '3'
 volumes:
   production_postgres_data: {}
   production_postgres_data_backups: {}
-  production_traefik: {}
 
 services:
   django: &django
@@ -16,9 +15,9 @@ services:
       - postgres
       - redis
     env_file:
-      - ./.envs/.production/.django
-      - ./.envs/.production/.postgres
-    command: /start
+      - ./.envs/docker/.django
+      - ./.envs/docker/.postgres
+    command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py runserver 0.0.0.0:8000'
 
   postgres:
     build:
@@ -29,25 +28,41 @@ services:
       - production_postgres_data:/var/lib/postgresql/data
       - production_postgres_data_backups:/backups
     env_file:
-      - ./.envs/.production/.postgres
+      - ./.envs/docker/.postgres
 
   redis:
     image: redis:6
 
   celeryworker:
     <<: *django
     image: backoffice_production_celeryworker
-    command: /start-celeryworker
+    depends_on:
+      - redis
+      - postgres
+    ports: []
+    command: celery -A config.celery_app worker -l INFO
+    networks:
+      - djangonetwork
 
   celerybeat:
     <<: *django
     image: backoffice_production_celerybeat
-    command: /start-celerybeat
+    command: celery -A config.celery_app beat -l INFO
+    depends_on:
+      - redis
+      - postgres
+    ports: []
+    networks:
+      - djangonetwork
 
-  flower:
-    <<: *django
-    image: backoffice_production_flower
-    command: /start-flower
+  mq:
+    image: rabbitmq:3.9.11-management
+    restart: 'always'
+    container_name: backofficee_production_mq
+    ports:
+      - '5672:5672'
+    networks:
+      - djangonetwork
 
 networks:
     djangonetwork:

pyproject.toml

@@ -111,7 +111,7 @@ description = ""
 authors = ["Your Name <[email protected]>"]
 
 [tool.poetry.dependencies]
-python = "3.11"
+python = "~3.11"
 python-slugify = "8.0.1"
 Pillow = "10.0.1"
 argon2-cffi = "23.1.0"