Redis cache (#3425)

* Configure redis

* Setup fetch_thumbnails for both redis and db

infra/env.public.production

@@ -10,6 +10,8 @@ POSTGRES_PASSWORD=concrexit
 
 # Used by concrexit and worker.
 CELERY_BROKER_URL=redis://redis:6379
+REDIS_CACHE_HOST=redis
+REDIS_CACHE_PORT=6379
 
 DJANGO_EMAIL_HOST=smtp-relay.gmail.com
 DJANGO_EMAIL_PORT=587

infra/env.public.staging

@@ -10,6 +10,8 @@ POSTGRES_PASSWORD=concrexit
 
 # Used by concrexit and worker.
 CELERY_BROKER_URL=redis://redis:6379
+REDIS_CACHE_HOST=redis
+REDIS_CACHE_PORT=6379
 
 DJANGO_EMAIL_HOST=smtp-relay.gmail.com
 DJANGO_EMAIL_PORT=587

website/activemembers/views.py

@@ -5,7 +5,7 @@
 from django.shortcuts import get_object_or_404
 from django.views.generic import DetailView, ListView
 
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 from utils.snippets import datetime_to_lectureyear
 
 from .models import Board, Committee, MemberGroupMembership, Society
@@ -47,7 +47,7 @@ def get_context_data(self, **kwargs) -> dict:
         members.sort(key=lambda x: x["since"])
 
         context.update({"members": members})
-        fetch_thumbnails_db([m["member"].profile.photo for m in members])
+        fetch_thumbnails([m["member"].profile.photo for m in members])
         return context
 
 
@@ -59,7 +59,7 @@ class CommitteeIndexView(ListView):
 
     def get_queryset(self) -> QuerySet:
         committees = Committee.active_objects.all()
-        fetch_thumbnails_db([c.photo for c in committees])
+        fetch_thumbnails([c.photo for c in committees])
         return committees
 
     def get_ordering(self) -> str:
@@ -81,7 +81,7 @@ class SocietyIndexView(ListView):
 
     def get_queryset(self) -> QuerySet:
         societies = Society.active_objects.all()
-        fetch_thumbnails_db([s.photo for s in societies])
+        fetch_thumbnails([s.photo for s in societies])
         return societies
 
     def get_ordering(self) -> str:
@@ -107,7 +107,7 @@ def get_queryset(self) -> QuerySet:
             boards = Board.objects.exclude(pk=self.current_board.pk)
         else:
             boards = Board.objects.all()
-        fetch_thumbnails_db([b.photo for b in boards])
+        fetch_thumbnails([b.photo for b in boards])
         return boards
 
     def get_context_data(self, **kwargs) -> dict:

website/events/api/v2/views.py

@@ -25,7 +25,7 @@
 from members.models import Membership
 from thaliawebsite.api.v2.permissions import IsAuthenticatedOrTokenHasScopeForMethod
 from thaliawebsite.api.v2.serializers import EmptySerializer
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 
 class EventListView(ListAPIView):
@@ -137,7 +137,7 @@ def get_queryset(self):
     def get_serializer(self, *args, **kwargs):
         if len(args) > 0:
             registrations = args[0]
-            fetch_thumbnails_db(
+            fetch_thumbnails(
                 [r.member.profile.photo for r in registrations if r.member]
             )
         return super().get_serializer(*args, **kwargs)

website/events/views.py

@@ -15,7 +15,7 @@
 from events.models import categories
 from events.services import is_user_registered
 from payments.models import Payment
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 from .forms import FieldsForm
 from .models import Event, EventRegistration
@@ -88,7 +88,7 @@ def get_context_data(self, **kwargs):
             "member", "member__profile"
         )
 
-        fetch_thumbnails_db(
+        fetch_thumbnails(
             [p.member.profile.photo for p in context["participants"] if p.member]
         )
 

website/members/api/v2/views.py

@@ -16,7 +16,7 @@
 from members.models import Member, Membership
 from thaliawebsite.api.openapi import OAuthAutoSchema
 from thaliawebsite.api.v2.permissions import IsAuthenticatedOrTokenHasScopeForMethod
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 
 class MemberListView(ListAPIView):
@@ -39,7 +39,7 @@ class MemberListView(ListAPIView):
     def get_serializer(self, *args, **kwargs):
         if len(args) > 0:
             members = args[0]
-            fetch_thumbnails_db([member.profile.photo for member in members])
+            fetch_thumbnails([member.profile.photo for member in members])
         return super().get_serializer(*args, **kwargs)
 
     permission_classes = [

website/members/views.py

@@ -21,7 +21,7 @@
 from members.decorators import membership_required
 from members.models import EmailChange, Member, Membership, Profile
 from thaliawebsite.views import PagedView
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 from utils.snippets import datetime_to_lectureyear
 
 from . import models
@@ -116,7 +116,7 @@ def get_context_data(self, **kwargs) -> dict:
             }
         )
 
-        fetch_thumbnails_db(
+        fetch_thumbnails(
             [x.profile.photo for x in context["object_list"] if x.profile.photo]
         )
 

website/partners/api/v2/views.py

@@ -7,7 +7,7 @@
 from partners.api.v2.serializers.partner import PartnerSerializer
 from partners.api.v2.serializers.vacancy import VacancySerializer
 from partners.models import Partner, Vacancy, VacancyCategory
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 
 class PartnerListView(ListAPIView):
@@ -23,7 +23,7 @@ class PartnerListView(ListAPIView):
     def get_serializer(self, *args, **kwargs):
         if len(args) > 0:
             partners = args[0]
-            fetch_thumbnails_db([partner.logo for partner in partners])
+            fetch_thumbnails([partner.logo for partner in partners])
         return super().get_serializer(*args, **kwargs)
 
     ordering_fields = ("name", "pk")

website/partners/templatetags/partner_banners.py

@@ -3,7 +3,7 @@
 from django import template
 
 from partners.models import Partner
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 register = template.Library()
 
@@ -30,7 +30,7 @@ def render_partner_banners(context):
     request.session["partner_sequence"] = rest + chosen
 
     partners = [p for p in all_partners if p.id in chosen]
-    fetch_thumbnails_db(
+    fetch_thumbnails(
         [p.alternate_logo or p.logo for p in partners],
         "medium",
     )

website/partners/views.py

@@ -3,7 +3,7 @@
 from django.shortcuts import get_object_or_404, render
 
 from partners.models import Partner, Vacancy, VacancyCategory
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 
 def index(request):
@@ -14,7 +14,7 @@ def index(request):
     main_partner = Partner.objects.filter(is_main_partner=True).first()
     local_partners = Partner.objects.filter(is_local_partner=True)
 
-    fetch_thumbnails_db([p.logo for p in partners])
+    fetch_thumbnails([p.logo for p in partners])
 
     context = {
         "main_partner": main_partner,
@@ -41,7 +41,7 @@ def vacancies(request):
         .select_related("partner")
         .prefetch_related("categories")
     )
-    fetch_thumbnails_db(v.get_company_logo() for v in vacancies)
+    fetch_thumbnails(v.get_company_logo() for v in vacancies)
     context = {
         "vacancies": vacancies,
         "categories": list(VacancyCategory.objects.all()),

website/photos/api/v2/views.py

@@ -14,7 +14,7 @@
     PhotoListSerializer,
 )
 from photos.models import Album, Like, Photo
-from utils.media.services import fetch_thumbnails_db
+from utils.media.services import fetch_thumbnails
 
 
 class AlbumListView(ListAPIView):
@@ -25,7 +25,7 @@ class AlbumListView(ListAPIView):
     def get_serializer(self, *args, **kwargs):
         if len(args) > 0:
             albums = args[0]
-            fetch_thumbnails_db([album.cover.file for album in albums if album.cover])
+            fetch_thumbnails([album.cover.file for album in albums if album.cover])
         return super().get_serializer(*args, **kwargs)
 
     queryset = Album.objects.filter(hidden=False).select_related("_cover")
@@ -55,7 +55,7 @@ def retrieve(self, request, *args, **kwargs):
 
     def get_object(self):
         object = super().get_object()
-        fetch_thumbnails_db([photo.file for photo in object.photo_set.all()])
+        fetch_thumbnails([photo.file for photo in object.photo_set.all()])
         return object
 
     def get_queryset(self):
@@ -95,7 +95,7 @@ def get(self, request, *args, **kwargs):
     def get_serializer(self, *args, **kwargs):
         if len(args) > 0:
             photos = args[0]
-            fetch_thumbnails_db([photo.file for photo in photos])
+            fetch_thumbnails([photo.file for photo in photos])
         return super().get_serializer(*args, **kwargs)
 
     def get_queryset(self):

website/photos/views.py

@@ -13,7 +13,7 @@
     is_album_accessible,
 )
 from thaliawebsite.views import PagedView
-from utils.media.services import fetch_thumbnails_db, get_media_url
+from utils.media.services import fetch_thumbnails, get_media_url
 
 COVER_FILENAME = "cover.jpg"
 
@@ -41,7 +41,7 @@ def get_queryset(self):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context["keywords"] = self.keywords
-        fetch_thumbnails_db([x.cover.file for x in context["object_list"] if x.cover])
+        fetch_thumbnails([x.cover.file for x in context["object_list"] if x.cover])
 
         return context
 
@@ -64,7 +64,7 @@ def get_context_data(self, **kwargs):
         photos = photos.order_by("pk")
 
         # Prefetch thumbnails for efficiency
-        fetch_thumbnails_db([p.file for p in photos])
+        fetch_thumbnails([p.file for p in photos])
 
         context["photos"] = photos
         return context
@@ -149,6 +149,6 @@ def get_queryset(self):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
 
-        fetch_thumbnails_db([p.file for p in context["photos"]])
+        fetch_thumbnails([p.file for p in context["photos"]])
 
         return context

website/thaliawebsite/settings.py

@@ -714,8 +714,21 @@ def from_env(
     },
 }
 
+REDIS_CACHE_PORT = int(
+    from_env("REDIS_CACHE_PORT", development="6379", production="6379")
+)
+REDIS_CACHE_HOST = from_env("REDIS_CACHE_HOST")
+REDIS_CACHE_URL = (
+    f"redis://{REDIS_CACHE_HOST}:{REDIS_CACHE_PORT}" if REDIS_CACHE_HOST else None
+)
+
 CACHES = {
     "default": {
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": REDIS_CACHE_URL,
+    }
+    if REDIS_CACHE_URL is not None
+    else {
         "BACKEND": "django.core.cache.backends.db.DatabaseCache",
         "LOCATION": "django_default_db_cache",
     },
@@ -864,10 +877,20 @@ def from_env(
 # See utils/model/signals.py for explanation
 SUSPEND_SIGNALS = False
 
-THUMBNAILS = {
-    "METADATA": {
+THUMBNAILS_METADATA = (
+    {
+        "BACKEND": "thumbnails.backends.metadata.RedisBackend",
+        "host": REDIS_CACHE_HOST,
+        "port": REDIS_CACHE_PORT,
+    }
+    if REDIS_CACHE_HOST
+    else {
         "BACKEND": "thumbnails.backends.metadata.DatabaseBackend",
-    },
+    }
+)
+
+THUMBNAILS = {
+    "METADATA": THUMBNAILS_METADATA,
     "STORAGE": {
         "BACKEND": DEFAULT_FILE_STORAGE,
     },

website/utils/media/services.py

@@ -5,6 +5,7 @@
 from django.db.models.fields.files import FieldFile, ImageFieldFile
 
 from thumbnails.backends.metadata import ImageMeta
+from thumbnails.fields import fetch_thumbnails as fetch_thumbnails_redis
 from thumbnails.files import ThumbnailedImageFile
 from thumbnails.images import Thumbnail
 from thumbnails.models import ThumbnailMeta
@@ -62,8 +63,8 @@ def get_thumbnail_url(
     return get_media_url(file, absolute_url=absolute_url)
 
 
-def fetch_thumbnails_db(images, sizes=None):
-    """Prefetches thumbnails from the database in one query.
+def fetch_thumbnails(images, sizes=None):
+    """Prefetches thumbnails from the database or redis efficiently.
 
     :param images: A list of images to prefetch thumbnails for.
     :param sizes: A list of sizes to prefetch. If None, all sizes will be prefetched.
@@ -72,6 +73,12 @@ def fetch_thumbnails_db(images, sizes=None):
     if not images:
         return
 
+    if (
+        settings.THUMBNAILS["METADATA"]["BACKEND"]
+        != "thumbnails.backends.metadata.DatabaseBackend"
+    ):
+        return fetch_thumbnails_redis(images, sizes)
+
     image_dict = {image.thumbnails.source_image.name: image for image in images}
     thumbnails = ThumbnailMeta.objects.select_related("source").filter(
         source__name__in=image_dict.keys()