chore: ajout de minio comme bucket S3 (dev & test)

.env.template

@@ -26,10 +26,10 @@ INCLUSION_CONNECT_CLIENT_SECRET=password
 PATH_TO_BACKUPS=~/path/to/backups
 
 # bucket for test purpose only
-CELLAR_ADDON_KEY_ID=__id_to_be_set__
-CELLAR_ADDON_KEY_SECRET=__key_to_be_set__
-CELLAR_ADDON_HOST=__host_to_be_set__
-CELLAR_ADDON_PROTOCOL=https
+CELLAR_ADDON_KEY_ID=minioadmin
+CELLAR_ADDON_KEY_SECRET=minioadmin
+CELLAR_ADDON_HOST=localhost:9000
+CELLAR_ADDON_PROTOCOL=http
 
 # itou-backups
 export RCLONE_S3_ACCESS_KEY_ID=ACCESS_KEY_ID

.github/workflows/ci.yml

@@ -17,7 +17,19 @@ jobs:
       POSTGRESQL_ADDON_DB: communaute
       POSTGRESQL_ADDON_USER: postgres
       POSTGRESQL_ADDON_PASSWORD: password
+      CELLAR_ADDON_KEY_ID: minioadmin
+      CELLAR_ADDON_KEY_SECRET: minioadmin
+      CELLAR_ADDON_PROTOCOL: http
+      CELLAR_ADDON_HOST: localhost:9000
     services:
+      minio:
+        image: bitnami/minio
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        ports:
+          - 9000:9000
+
       postgres:
         # Docker Hub image
         image: postgres:15-alpine
@@ -57,6 +69,8 @@ jobs:
           python manage.py compress
       - name: 🚧 Check pending migrations
         run: python manage.py makemigrations --check --dry-run --noinput
+      - name: 🚧 Configure bucket
+        run: python manage.py configure_bucket
       - name: 🤹‍ Django tests
         run: pytest --numprocesses=logical --create-db
         env:

README.md

@@ -22,22 +22,24 @@ $ poetry shell
 
 ## Démarrer les instances
 
-Démarer la base de données
+Démarer la base de données et le bucket S3
 
 ```bash
-$ docker-compose up postgres -d
+$ docker-compose up -d
 ```
 
 Démarrer le service web
 
 ```bash
-$ python manage.py runserver
+$ python manage.py runserver_plus
 ```
 
-## Peupler la base de données
+## Préparer l'environnement de données
 
 ```bash
-$ python manage.py loaddata populate
+$ python manage.py migrate
+$ python manage.py populate
+$ python manage.py configure_bucket
 ```
 ## Mises à jour
 

config/settings/base.py

@@ -264,9 +264,9 @@
 AWS_S3_ENDPOINT_URL = (
     f"{os.getenv('CELLAR_ADDON_PROTOCOL', 'https')}://{os.getenv('CELLAR_ADDON_HOST', 'set-var-env.com')}"
 )
-AWS_STORAGE_BUCKET_NAME = "c3-storage-prod"
-AWS_STORAGE_BUCKET_NAME_PUBLIC = "c3-storage-prod-public"
-AWS_S3_STORAGE_BUCKET_REGION = "eu-west-3"
+AWS_STORAGE_BUCKET_NAME = os.getenv("S3_STORAGE_BUCKET_NAME", "private-bucket")
+AWS_STORAGE_BUCKET_NAME_PUBLIC = os.getenv("S3_STORAGE_BUCKET_NAME_PUBLIC", "public-bucket")
+AWS_S3_STORAGE_BUCKET_REGION = os.getenv("S3_STORAGE_BUCKET_REGION", "eu-west-3")
 
 # MEDIA CONFIGURATION
 # ------------------------------------------------------------------------------

config/settings/dev.py

@@ -60,10 +60,5 @@
 # See: https://docs.djangoproject.com/en/dev/ref/settings/#media-url
 MEDIA_URL = f"{AWS_S3_ENDPOINT_URL}/"  # noqa: F405
 
-# STORAGE (django >= 4.2)
-STORAGES = {
-    "default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
-    "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
-}
-
 CSP_DEFAULT_SRC = ("*",)
+CSP_IMG_SRC += ("localhost:9000",)  # noqa: F405

docker-compose.yml

@@ -2,6 +2,19 @@ version: "3.8"
 
 services:
 
+  minio:
+    image: bitnami/minio
+    container_name: commu_minio
+    restart: unless-stopped
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    ports:
+      - "127.0.0.1:${MINIO_PORT_ON_DOCKER_HOST:-9000}:9000"
+      - "127.0.0.1:${MINIO_ADMIN_PORT_ON_DOCKER_HOST:-9001}:9001"
+    volumes:
+      - minio_data:/bitnami/minio/data
+
   postgres:
     container_name: commu_postgres
     image: postgres:15-alpine
@@ -18,6 +31,7 @@ services:
       - "127.0.0.1:${POSTGRESQL_ADDON_PORT:-5432}:5432"
 
   django:
+    profiles: [ "django" ]
     container_name: commu_django
     env_file:
       - .env
@@ -40,6 +54,7 @@ services:
       - "127.0.0.1:${DJANGO_DEBUGPY_PORT:-5678}:5678"
 
   mailhog:
+    profiles: [ "mailhog" ]
     image: mailhog/mailhog:latest
     restart: always
     ports:
@@ -49,3 +64,4 @@ services:
 volumes:
   postgres_data:
   postgres_data_backups:
+  minio_data:

lacommunaute/forum/factories.py

@@ -17,6 +17,9 @@ class ForumFactory(BaseForumFactory):
     class Meta:
         skip_postgeneration_save = True
 
+    class Params:
+        with_image = factory.Trait(image=factory.django.ImageField(filename="banner.jpg"))
+
     @factory.post_generation
     def with_public_perms(self, create, extracted, **kwargs):
         if not create or not extracted:
@@ -41,4 +44,4 @@ def with_child(self, create, extracted, **kwargs):
         if not create or not extracted:
             return
 
-        ForumFactory(parent=self, with_public_perms=True)
+        ForumFactory(parent=self, with_public_perms=True, with_image=True)

lacommunaute/forum/management/commands/populate.py

@@ -1,6 +1,5 @@
 import sys
 
-from django.contrib.auth.hashers import make_password
 from django.core.management.base import BaseCommand
 from django.db import connection
 
@@ -14,7 +13,7 @@ class Command(BaseCommand):
     help = "hydratation d'un site de validation"
 
     def handle(self, *args, **options):
-        UserFactory(username="communaute", password=make_password("password"))
+        UserFactory(username="communaute", password="password", is_superuser=True, is_staff=True)
         sys.stdout.write("superuser created\n")
 
         forum = ForumFactory(name="Espace d'échanges", with_public_perms=True)

lacommunaute/forum/migrations/0013_alter_forum_image.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.6 on 2024-05-21 14:55
+
+import storages.backends.s3
+from django.conf import settings
+from django.db import migrations, models
+
+import lacommunaute.forum.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("forum", "0012_forum_short_description_alter_forum_kind"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="forum",
+            name="image",
+            field=models.ImageField(
+                storage=storages.backends.s3.S3Storage(
+                    bucket_name=settings.AWS_STORAGE_BUCKET_NAME, file_overwrite=False
+                ),
+                upload_to="",
+                validators=[lacommunaute.forum.models.validate_image_size],
+            ),
+        ),
+    ]

lacommunaute/forum/models.py

@@ -1,15 +1,18 @@
 import uuid
 
+from django.conf import settings
 from django.contrib.auth.models import Group
 from django.contrib.contenttypes.fields import GenericRelation
 from django.db import models
 from django.urls import reverse
 from django.utils.functional import cached_property
 from machina.apps.forum.abstract_models import AbstractForum
+from storages.backends.s3boto3 import S3Boto3Storage
 
 from lacommunaute.forum.enums import Kind as Forum_Kind
 from lacommunaute.forum_conversation.models import Topic
 from lacommunaute.forum_upvote.models import UpVote
+from lacommunaute.utils.validators import validate_image_size
 
 
 class ForumQuerySet(models.QuerySet):
@@ -28,6 +31,10 @@ class Forum(AbstractForum):
     short_description = models.CharField(
         max_length=400, blank=True, null=True, verbose_name="Description courte (SEO)"
     )
+    image = models.ImageField(
+        storage=S3Boto3Storage(bucket_name=settings.AWS_STORAGE_BUCKET_NAME, file_overwrite=False),
+        validators=[validate_image_size],
+    )
 
     upvotes = GenericRelation(UpVote, related_query_name="forum")
 

lacommunaute/forum/tests/test_categoryforum_listview.py

@@ -3,7 +3,7 @@
 from pytest_django.asserts import assertContains, assertNotContains
 
 from lacommunaute.forum.enums import Kind as ForumKind
-from lacommunaute.forum.factories import ForumFactory
+from lacommunaute.forum.factories import CategoryForumFactory, ForumFactory
 from lacommunaute.forum.models import Forum
 from lacommunaute.users.factories import UserFactory
 
@@ -45,3 +45,12 @@ def test_display_create_category_button(client, db):
     user.save()
     response = client.get(url)
     assertContains(response, reverse("forum_extension:create_category"), status_code=200)
+
+
+def test_display_banners(client, db):
+    forum = CategoryForumFactory(with_child=True, with_public_perms=True)
+    ForumFactory(parent=forum, with_public_perms=True, with_image=True)
+    url = reverse("forum_extension:forum", kwargs={"pk": forum.pk, "slug": forum.slug})
+    response = client.get(url)
+    for child in forum.get_children():
+        assertContains(response, child.image.url.split("=")[0])

lacommunaute/forum/tests/tests_model.py

@@ -1,3 +1,4 @@
+from django.conf import settings
 from django.db import IntegrityError
 from django.test import TestCase
 
@@ -57,3 +58,14 @@ def test_upvotes_count(self):
         self.assertEqual(forum.upvotes_count(), 0)
         forum.upvotes.create(voter=UserFactory())
         self.assertEqual(forum.upvotes_count(), 1)
+
+    def test_image_is_imagefield(self):
+        forum = ForumFactory()
+        self.assertEqual(forum.image.field.__class__.__name__, "ImageField")
+
+    def test_image_url(self):
+        forum = ForumFactory(image="test.jpg")
+        self.assertEqual(
+            forum.image.url.split("?")[0], f"{settings.MEDIA_URL}{settings.AWS_STORAGE_BUCKET_NAME}/{forum.image.name}"
+        )
+        self.assertIn("AWSAccessKeyId=", forum.image.url)

lacommunaute/forum/tests/tests_views.py

@@ -421,3 +421,9 @@ def test_filtered_queryset_on_tag(self):
         )
         self.assertContains(response, topic.subject)
         self.assertNotContains(response, self.topic.subject)
+
+    def test_banner_display_on_subcategory_forum(self):
+        category_forum = CategoryForumFactory(with_child=True, with_public_perms=True)
+        forum = category_forum.get_children().first()
+        response = self.client.get(reverse("forum_extension:forum", kwargs={"pk": forum.pk, "slug": forum.slug}))
+        self.assertContains(response, forum.image.url.split("=")[0])

lacommunaute/forum_file/management/commands/configure_bucket.py

@@ -0,0 +1,57 @@
+import json
+from urllib.parse import urljoin
+
+import boto3
+import httpx
+from botocore.client import Config
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+
+def s3_client():
+    return boto3.client(
+        "s3",
+        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+        aws_access_key_id=settings.AWS_S3_ACCESS_KEY_ID,
+        aws_secret_access_key=settings.AWS_S3_SECRET_ACCESS_KEY,
+        region_name=settings.AWS_S3_STORAGE_BUCKET_REGION,
+        config=Config(signature_version="s3v4"),
+    )
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        if self.check_minio():
+            client = s3_client()
+
+            for bucket_name in settings.AWS_STORAGE_BUCKET_NAME, settings.AWS_STORAGE_BUCKET_NAME_PUBLIC:
+                try:
+                    client.create_bucket(Bucket=bucket_name)
+                except client.exceptions.BucketAlreadyOwnedByYou:
+                    pass
+
+            # Set up public access to the AWS_STORAGE_BUCKET_NAME_PUBLIC bucket
+            public_bucket_policy = {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Sid": "PublicRead",
+                        "Effect": "Allow",
+                        "Principal": "*",
+                        "Action": ["s3:GetObject"],
+                        "Resource": [f"arn:aws:s3:::{settings.AWS_STORAGE_BUCKET_NAME_PUBLIC}/*"],
+                    }
+                ],
+            }
+            client.put_bucket_policy(
+                Bucket=settings.AWS_STORAGE_BUCKET_NAME_PUBLIC, Policy=json.dumps(public_bucket_policy)
+            )
+
+    def check_minio(self):
+        # https://min.io/docs/minio/linux/operations/monitoring/healthcheck-probe.html#node-liveness
+        livecheck_url = urljoin(settings.AWS_S3_ENDPOINT_URL, "minio/health/live")
+        response = httpx.head(livecheck_url)
+        try:
+            return response.headers["Server"] == "MinIO"
+        except KeyError:
+            return False

lacommunaute/forum_file/models.py

@@ -1,17 +1,10 @@
 from django.conf import settings
-from django.core.exceptions import ValidationError
 from django.db import models
 from machina.models.abstract_models import DatedModel
 from storages.backends.s3boto3 import S3Boto3Storage
 
 from lacommunaute.users.models import User
-
-
-def validate_image_size(value):
-    max_size = 1024 * 1024 * 8
-
-    if value.size > max_size:
-        raise ValidationError("L'image ne doit pas dépasser 1 Mo")
+from lacommunaute.utils.validators import validate_image_size
 
 
 class PublicFile(DatedModel):

lacommunaute/forum_file/tests/test_models.py

@@ -32,11 +32,5 @@ def test_get_file_url(db, public_file):
     assert public_file.get_file_url() == expected_file_url
 
 
-def test_size_validator(db, public_file):
-    with pytest.raises(Exception):
-        public_file.file.size = 1024 * 1024 * 8 + 1
-        public_file.save()
-
-
 def test_file_field_is_imagefield(db):
     assert isinstance(PublicFile._meta.get_field("file"), ImageField)

lacommunaute/templates/forum/forum_detail.html

@@ -41,7 +41,8 @@ <h1>{{ forum.name }}</h1>
                                 {% include "partials/upvotes.html" with obj=forum kind="forum" %}
                                 {% include "partials/social_share_buttons.html" with text=forum.name instance=forum id=forum.pk %}
                             </div>
-                            <div class="col-12 col-sm-auto">
+                            <div class="col-12 col-sm-auto">{% include "forum/partials/forum_banner.html" with forum=forum only %}</div>
+                            <div class="col-12 col-sm-auto mt-3">
                                 <div class="textarea_cms_md">{{ forum.description.rendered|urlizetrunc_target_blank:30 }}</div>
                             </div>
                         </div>

lacommunaute/templates/forum/forum_list.html

@@ -57,7 +57,8 @@ <h3 class="h3 mb-0">
                             <p class="h3 lh-base">
                                 <i class="ri-newspaper-line font-weight-normal"></i> {{ node.obj.name }}
                             </p>
-                            {% if node.obj.short_description %}<div>{{ node.obj.short_description }}</div>{% endif %}
+                            <div>{% include "forum/partials/forum_banner.html" with forum=node.obj only %}</div>
+                            {% if node.obj.short_description %}<div class="mt-3">{{ node.obj.short_description }}</div>{% endif %}
                         </div>
                         <div class="card-footer text-end">
                             <a href="{% url 'forum_extension:forum' node.obj.slug node.obj.id %}"

lacommunaute/templates/forum/partials/forum_banner.html

@@ -0,0 +1,5 @@
+{% if forum.image %}
+    <div class="d-none d-md-block forum-image mt-3">
+        <img src="{{ forum.image.url }}" alt="{{ forum.name }}" class="rounded img-fluid" />
+    </div>
+{% endif %}