diff --git a/.envs/docker/.django b/.envs/docker/.django index 024b9f3c..54082a77 100644 --- a/.envs/docker/.django +++ b/.envs/docker/.django @@ -5,11 +5,11 @@ IPYTHONDIR=/app/.ipython # Redis # ------------------------------------------------------------------------------ -REDIS_URL=redis://localhost:6379/0 +REDIS_URL=redis://redis:6379/0 # Celery -CELERY_BROKER_URL=amqp://guest:guest@localhost:5672/ -CELERY_RESULT_BACKEND=redis://localhost:6379/2 +CELERY_BROKER_URL=amqp://guest:guest@mq:5672/ +CELERY_RESULT_BACKEND=redis://redis:6379/2 # Flower CELERY_FLOWER_USER=debug CELERY_FLOWER_PASSWORD=debug diff --git a/.envs/local/.django b/.envs/local/.django index 024b9f3c..54082a77 100644 --- a/.envs/local/.django +++ b/.envs/local/.django @@ -5,11 +5,11 @@ IPYTHONDIR=/app/.ipython # Redis # ------------------------------------------------------------------------------ -REDIS_URL=redis://localhost:6379/0 +REDIS_URL=redis://redis:6379/0 # Celery -CELERY_BROKER_URL=amqp://guest:guest@localhost:5672/ -CELERY_RESULT_BACKEND=redis://localhost:6379/2 +CELERY_BROKER_URL=amqp://guest:guest@mq:5672/ +CELERY_RESULT_BACKEND=redis://redis:6379/2 # Flower CELERY_FLOWER_USER=debug CELERY_FLOWER_PASSWORD=debug diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 80348363..a9741db7 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -54,6 +54,5 @@ jobs: --env POSTGRES_USER=inspire --env POSTGRES_PASSWORD=inspire --env POSTGRES_HOST=127.0.0.1 - --env OPENSEARCH_HOST=127.0.0.1:9200 ${{ inputs.image }} run pytest \ No newline at end of file diff --git a/.github/workflows/pull-request-main.yml b/.github/workflows/pull-request-main.yml index eadd45c3..2e2924d1 100644 --- a/.github/workflows/pull-request-main.yml +++ b/.github/workflows/pull-request-main.yml @@ -1,20 +1,17 @@ -name: Push main +name: Pull request main on: - push: + pull_request_target: branches: [main] -defaults: - run: - shell: bash - jobs: lint: uses: ./.github/workflows/lint.yml with: - ref: ${{ github.ref }} + ref: ${{ github.ref }} + secrets: inherit test: - uses: ./.github/workflows/test.yml + uses: ./.github/workflows/tests.yml with: - ref: ${{ github.ref }} + ref: ${{ github.event.pull_request.head.sha }} secrets: inherit diff --git a/.github/workflows/push-main.yml b/.github/workflows/push-main.yml index ab7686bf..eadd45c3 100644 --- a/.github/workflows/push-main.yml +++ b/.github/workflows/push-main.yml @@ -1,18 +1,20 @@ -name: Pull request main +name: Push main on: - pull_request_target: + push: branches: [main] - paths-ignore: [ "docs/**" ] + +defaults: + run: + shell: bash jobs: lint: uses: ./.github/workflows/lint.yml with: - ref: ${{ github.ref }} - secrets: inherit + ref: ${{ github.ref }} test: - uses: ./.github/workflows/tests.yml + uses: ./.github/workflows/test.yml with: - ref: ${{ github.event.pull_request.head.sha }} + ref: ${{ github.ref }} secrets: inherit diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 91b5d91d..86c67363 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ libpq-dev ARG POETRY_VERSION -ENV POETRY_VERSION="${POETRY_VERSION:-1.4.2}" +ENV POETRY_VERSION="${POETRY_VERSION:-1.1.14}" RUN curl -sSL https://install.python-poetry.org \ | python - --version "${POETRY_VERSION}" \ && poetry --version @@ -23,4 +23,4 @@ COPY poetry.lock pyproject.toml ./ RUN poetry install --no-root COPY . ${APP_HOME} -RUN poetry Install +RUN poetry install diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 38617b78..43e55397 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,96 +1,30 @@ - -# define an alias for the specific python version used in this file. FROM python:3.11.6-slim-bullseye as python -# Python build stage -FROM python as python-build-stage - -ARG BUILD_ENVIRONMENT=production - -# Install apt packages -RUN apt-get update && apt-get install --no-install-recommends -y \ - # dependencies for building Python packages - build-essential \ - # psycopg2 dependencies - libpq-dev - -# Requirements are installed here to ensure they will be cached. -COPY ./requirements . - -# Create Python Dependency and Sub-Dependency Wheels. -RUN pip wheel --wheel-dir /usr/src/app/wheels \ - -r ${BUILD_ENVIRONMENT}.txt - - -# Python 'run' stage -FROM python as python-run-stage - -ARG BUILD_ENVIRONMENT=production ARG APP_HOME=/app - -ENV PYTHONUNBUFFERED 1 -ENV PYTHONDONTWRITEBYTECODE 1 -ENV BUILD_ENV ${BUILD_ENVIRONMENT} - WORKDIR ${APP_HOME} -RUN addgroup --system django \ - && adduser --system --ingroup django django +ARG BUILD_ENVIRONMENT=production +ENV BUILD_ENV ${BUILD_ENVIRONMENT} +ENV PATH="/root/.local/bin:${PATH}" \ + POETRY_VIRTUALENVS_CREATE=false \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 -# Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ - # psycopg2 dependencies - libpq-dev \ - # Translations dependencies - gettext \ - # cleaning up unused files + curl libpq-dev build-essential gettext \ && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && rm -rf /var/lib/apt/lists/* -# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction -# copy python dependency wheels from python-build-stage -COPY --from=python-build-stage /usr/src/app/wheels /wheels/ - -# use wheels to install python dependencies -RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \ - && rm -rf /wheels/ - - -COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint -RUN sed -i 's/\r$//g' /entrypoint -RUN chmod +x /entrypoint - - -COPY --chown=django:django ./compose/production/django/start /start -RUN sed -i 's/\r$//g' /start -RUN chmod +x /start -COPY --chown=django:django ./compose/production/django/celery/worker/start /start-celeryworker -RUN sed -i 's/\r$//g' /start-celeryworker -RUN chmod +x /start-celeryworker - - -COPY --chown=django:django ./compose/production/django/celery/beat/start /start-celerybeat -RUN sed -i 's/\r$//g' /start-celerybeat -RUN chmod +x /start-celerybeat - - -COPY --chown=django:django ./compose/production/django/celery/flower/start /start-flower -RUN sed -i 's/\r$//g' /start-flower -RUN chmod +x /start-flower - - -# copy application code to WORKDIR -COPY --chown=django:django . ${APP_HOME} - -# make django owner of the WORKDIR directory as well. -RUN chown django:django ${APP_HOME} +ARG POETRY_VERSION +ENV POETRY_VERSION="${POETRY_VERSION:-1.1.14}" +RUN curl -sSL https://install.python-poetry.org \ + | python - --version "${POETRY_VERSION}" \ + && poetry --version -USER django +COPY poetry.lock pyproject.toml . +RUN poetry install --no-root -RUN DATABASE_URL="" \ - CELERY_BROKER_URL="" \ - DJANGO_SETTINGS_MODULE="config.settings.test" \ - python manage.py compilemessages +COPY . ${APP_HOME} -ENTRYPOINT ["/entrypoint"] +RUN poetry install diff --git a/compose/production/django/celery/beat/start b/compose/production/django/celery/beat/start deleted file mode 100644 index 42ddca91..00000000 --- a/compose/production/django/celery/beat/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - -exec celery -A config.celery_app beat -l INFO diff --git a/compose/production/django/celery/flower/start b/compose/production/django/celery/flower/start deleted file mode 100644 index 4180d677..00000000 --- a/compose/production/django/celery/flower/start +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset - - -exec celery \ - -A config.celery_app \ - -b "${CELERY_BROKER_URL}" \ - flower \ - --basic_auth="${CELERY_FLOWER_USER}:${CELERY_FLOWER_PASSWORD}" diff --git a/compose/production/django/celery/worker/start b/compose/production/django/celery/worker/start deleted file mode 100644 index af0c8f7b..00000000 --- a/compose/production/django/celery/worker/start +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - - -exec celery -A config.celery_app worker -l INFO diff --git a/config/settings/local.py b/config/settings/local.py index 5a00431a..d8688274 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -18,8 +18,14 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#caches CACHES = { "default": { - "BACKEND": "django.core.cache.backends.locmem.LocMemCache", - "LOCATION": "", + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": env("REDIS_URL"), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + # Mimicing memcache behavior. + # https://github.com/jazzband/django-redis#memcached-exceptions-behavior + "IGNORE_EXCEPTIONS": True, + }, } } diff --git a/config/settings/production.py b/config/settings/production.py index ade67c52..1311a571 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -14,7 +14,7 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key SECRET_KEY = env("DJANGO_SECRET_KEY") # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts -ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["example.com"]) +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"]) # DATABASES # ------------------------------------------------------------------------------ diff --git a/local.yml b/local.yml index 053d58ca..6cc63275 100644 --- a/local.yml +++ b/local.yml @@ -55,7 +55,7 @@ services: - redis - postgres ports: [] - command: celery -A config.celery_app beat -l INFO + command: celery -A config.celery_app worker -l INFO networks: - djangonetwork @@ -67,9 +67,19 @@ services: - redis - postgres ports: [] - command: celery -A config.celery_app worker -l INFO + command: celery -A config.celery_app beat -l INFO networks: - djangonetwork + + mq: + image: rabbitmq:3.9.11-management + restart: 'always' + container_name: backoffice_local_mq + ports: + - '5672:5672' + networks: + - djangonetwork + networks: djangonetwork: diff --git a/production.yml b/production.yml index 57b41212..ea527460 100644 --- a/production.yml +++ b/production.yml @@ -3,7 +3,6 @@ version: '3' volumes: production_postgres_data: {} production_postgres_data_backups: {} - production_traefik: {} services: django: &django @@ -16,9 +15,9 @@ services: - postgres - redis env_file: - - ./.envs/.production/.django - - ./.envs/.production/.postgres - command: /start + - ./.envs/docker/.django + - ./.envs/docker/.postgres + command: sh -c 'poetry run python manage.py migrate && poetry run python manage.py runserver 0.0.0.0:8000' postgres: build: @@ -29,7 +28,7 @@ services: - production_postgres_data:/var/lib/postgresql/data - production_postgres_data_backups:/backups env_file: - - ./.envs/.production/.postgres + - ./.envs/docker/.postgres redis: image: redis:6 @@ -37,17 +36,33 @@ services: celeryworker: <<: *django image: backoffice_production_celeryworker - command: /start-celeryworker + depends_on: + - redis + - postgres + ports: [] + command: celery -A config.celery_app worker -l INFO + networks: + - djangonetwork celerybeat: <<: *django image: backoffice_production_celerybeat - command: /start-celerybeat + command: celery -A config.celery_app beat -l INFO + depends_on: + - redis + - postgres + ports: [] + networks: + - djangonetwork - flower: - <<: *django - image: backoffice_production_flower - command: /start-flower + mq: + image: rabbitmq:3.9.11-management + restart: 'always' + container_name: backofficee_production_mq + ports: + - '5672:5672' + networks: + - djangonetwork networks: djangonetwork: diff --git a/pyproject.toml b/pyproject.toml index 72499356..3910ab8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,7 +111,7 @@ description = "" authors = ["Your Name "] [tool.poetry.dependencies] -python = "3.11" +python = "~3.11" python-slugify = "8.0.1" Pillow = "10.0.1" argon2-cffi = "23.1.0"