chore: ajout de minio comme bucket S3 (dev & test)

.env.template

@@ -26,10 +26,10 @@ INCLUSION_CONNECT_CLIENT_SECRET=password
 PATH_TO_BACKUPS=~/path/to/backups
 
 # bucket for test purpose only
-CELLAR_ADDON_KEY_ID=__id_to_be_set__
-CELLAR_ADDON_KEY_SECRET=__key_to_be_set__
-CELLAR_ADDON_HOST=__host_to_be_set__
-CELLAR_ADDON_PROTOCOL=https
+CELLAR_ADDON_KEY_ID=minioadmin
+CELLAR_ADDON_KEY_SECRET=minioadmin
+CELLAR_ADDON_HOST=localhost:9000
+CELLAR_ADDON_PROTOCOL=http
 
 # itou-backups
 export RCLONE_S3_ACCESS_KEY_ID=ACCESS_KEY_ID

.github/workflows/ci.yml

@@ -17,7 +17,19 @@ jobs:
       POSTGRESQL_ADDON_DB: communaute
       POSTGRESQL_ADDON_USER: postgres
       POSTGRESQL_ADDON_PASSWORD: password
+      CELLAR_ADDON_KEY_ID: minioadmin
+      CELLAR_ADDON_KEY_SECRET: minioadmin
+      CELLAR_ADDON_PROTOCOL: http
+      CELLAR_ADDON_HOST: localhost:9000
     services:
+      minio:
+        image: bitnami/minio
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        ports:
+          - 9000:9000
+
       postgres:
         # Docker Hub image
         image: postgres:15-alpine
@@ -57,6 +69,8 @@ jobs:
           python manage.py compress
       - name: 🚧 Check pending migrations
         run: python manage.py makemigrations --check --dry-run --noinput
+      - name: 🚧 Configure bucket
+        run: python manage.py configure_bucket
       - name: 🤹‍ Django tests
         run: pytest --numprocesses=logical --create-db
         env:

README.md

@@ -22,22 +22,24 @@ $ poetry shell
 
 ## Démarrer les instances
 
-Démarer la base de données
+Démarer la base de données et le bucket S3
 
 ```bash
-$ docker-compose up postgres -d
+$ docker-compose up -d
 ```
 
 Démarrer le service web
 
 ```bash
-$ python manage.py runserver
+$ python manage.py runserver_plus
 ```
 
-## Peupler la base de données
+## Préparer l'environnement de données
 
 ```bash
-$ python manage.py loaddata populate
+$ python manage.py migrate
+$ python manage.py populate
+$ python manage.py configure_bucket
 ```
 ## Mises à jour
 

config/settings/base.py

@@ -264,9 +264,9 @@
 AWS_S3_ENDPOINT_URL = (
     f"{os.getenv('CELLAR_ADDON_PROTOCOL', 'https')}://{os.getenv('CELLAR_ADDON_HOST', 'set-var-env.com')}"
 )
-AWS_STORAGE_BUCKET_NAME = "c3-storage-prod"
-AWS_STORAGE_BUCKET_NAME_PUBLIC = "c3-storage-prod-public"
-AWS_S3_STORAGE_BUCKET_REGION = "eu-west-3"
+AWS_STORAGE_BUCKET_NAME = os.getenv("S3_STORAGE_BUCKET_NAME", "private-bucket")
+AWS_STORAGE_BUCKET_NAME_PUBLIC = os.getenv("S3_STORAGE_BUCKET_NAME_PUBLIC", "public-bucket")
+AWS_S3_STORAGE_BUCKET_REGION = os.getenv("S3_STORAGE_BUCKET_REGION", "eu-west-3")
 
 # MEDIA CONFIGURATION
 # ------------------------------------------------------------------------------

config/settings/dev.py

@@ -60,10 +60,4 @@
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
 MEDIA_URL = f"{AWS_S3_ENDPOINT_URL}/"  # noqa: F405
 
-# STORAGE (django >= 4.2)
-STORAGES = {
-    "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
-    "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
-}
-
 CSP_DEFAULT_SRC = ("*",)

docker-compose.yml

@@ -2,6 +2,19 @@ version: "3.8"
 
 services:
 
+  minio:
+    image: bitnami/minio
+    container_name: commu_minio
+    restart: unless-stopped
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    ports:
+      - "127.0.0.1:${MINIO_PORT_ON_DOCKER_HOST:-9000}:9000"
+      - "127.0.0.1:${MINIO_ADMIN_PORT_ON_DOCKER_HOST:-9001}:9001"
+    volumes:
+      - minio_data:/bitnami/minio/data
+
   postgres:
     container_name: commu_postgres
     image: postgres:15-alpine
@@ -18,6 +31,7 @@ services:
       - "127.0.0.1:${POSTGRESQL_ADDON_PORT:-5432}:5432"
 
   django:
+    profiles: [ "django" ]
     container_name: commu_django
     env_file:
       - .env
@@ -40,6 +54,7 @@ services:
       - "127.0.0.1:${DJANGO_DEBUGPY_PORT:-5678}:5678"
 
   mailhog:
+    profiles: [ "mailhog" ]
     image: mailhog/mailhog:latest
     restart: always
     ports:
@@ -49,3 +64,4 @@ services:
 volumes:
   postgres_data:
   postgres_data_backups:
+  minio_data:

lacommunaute/forum/management/commands/populate.py

@@ -1,6 +1,5 @@
 import sys
 
-from django.contrib.auth.hashers import make_password
 from django.core.management.base import BaseCommand
 from django.db import connection
 
@@ -14,7 +13,7 @@
     help = "hydratation d'un site de validation"
 
     def handle(self, *args, **options):
-        UserFactory(username="communaute", password=make_password("password"))
+        UserFactory(username="communaute", password="password", is_superuser=True, is_staff=True)
         sys.stdout.write("superuser created\n")
 
         forum = ForumFactory(name="Espace d'échanges", with_public_perms=True)

lacommunaute/forum_file/management/commands/configure_bucket.py

@@ -0,0 +1,57 @@
+import json
+from urllib.parse import urljoin
+
+import boto3
+import httpx
+from botocore.client import Config
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+def s3_client():
+    return boto3.client(
+        "s3",
+        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+        aws_access_key_id=settings.AWS_S3_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_S3_SECRET_ACCESS_KEY,
+        region_name=settings.AWS_S3_STORAGE_BUCKET_REGION,
+        config=Config(signature_version="s3v4"),
+    )
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        if self.check_minio():
+            client = s3_client()
+
+            for bucket_name in settings.AWS_STORAGE_BUCKET_NAME, settings.AWS_STORAGE_BUCKET_NAME_PUBLIC:
+                try:
+                    client.create_bucket(Bucket=bucket_name)
+                except client.exceptions.BucketAlreadyOwnedByYou:
+                    pass
+
+            # Set up public access to the AWS_STORAGE_BUCKET_NAME_PUBLIC bucket
+            public_bucket_policy = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "PublicRead",
+                        "Effect": "Allow",
+                        "Principal": "*",
+                        "Action": ["s3:GetObject"],
+                        "Resource": [f"arn:aws:s3:::{settings.AWS_STORAGE_BUCKET_NAME_PUBLIC}/*"],
+                    }
+                ],
+            }
+            client.put_bucket_policy(
+                Bucket=settings.AWS_STORAGE_BUCKET_NAME_PUBLIC, Policy=json.dumps(public_bucket_policy)
+            )
+
+    def check_minio(self):
+        # https://min.io/docs/minio/linux/operations/monitoring/healthcheck-probe.html#node-liveness
+        livecheck_url = urljoin(settings.AWS_S3_ENDPOINT_URL, "minio/health/live")
+        response = httpx.head(livecheck_url)
+        try:
+            return response.headers["Server"] == "MinIO"
+        except KeyError:
+            return False

lacommunaute/users/factories.py

@@ -1,22 +1,15 @@
-import functools
 import random
 
 import factory
 from django.contrib.auth.hashers import make_password
 from django.contrib.auth.models import Group
-from machina.core.db.models import get_model
 
+from lacommunaute.users.models import User
 
-User = get_model("users", "User")
 
 DEFAULT_PASSWORD = "supercalifragilisticexpialidocious"
 
 
-@functools.cache
-def default_password():
-    return make_password(DEFAULT_PASSWORD)
-
-
 class GroupFactory(factory.django.DjangoModelFactory):
     class Meta:
         model = Group
@@ -32,7 +25,7 @@ class Meta:
     first_name = factory.Faker("first_name")
     last_name = factory.Faker("last_name")
     email = factory.Faker("email")
-    password = factory.LazyFunction(default_password)
+    password = factory.Transformer(DEFAULT_PASSWORD, transform=make_password)
 
     @factory.post_generation
     def with_perm(obj, create, extracted, **kwargs):