Merge pull request #3238 from digitalfabrik/develop

Release `2024.12.0`

.circleci/config.yml

@@ -19,6 +19,7 @@ jobs:
             else
               python3 -m venv .venv
               source .venv/bin/activate
+              pip install --upgrade pip
               pip install -e .[dev-pinned,pinned]
             fi
       - save_cache:

.github/ISSUE_TEMPLATE/bug-report.md

@@ -34,3 +34,7 @@ labels: 'bug'
   ```
 
 </details>
+
+### Related Issues
+<!-- A collection of all issues that are related to this issue and could be useful. -->
+<!-- Please also include the related integreat-app issues, if possible. -->

.github/ISSUE_TEMPLATE/feature-request.md

@@ -28,3 +28,8 @@ As a [type of user] I want [goals or objectives] so that [values or benefits]. -
 
 ### Design Requirements
 <!-- If the customization includes input from our design team, the detailed requirements will be collected here. Note: These will exist mainly in German to simplify internal communication. -->
+
+### Related Issues
+<!-- A collection of all issues that are related to this issue and could be useful. -->
+<!-- Please also include the related integreat-app issues, if possible. -->
+<!-- If the app issue does not exist yet, ping Toni on Mattermost to create one and then link it here. -->

.gitignore

@@ -55,6 +55,7 @@ webpack-stats.json
 
 # Location file of redis unix socket
 .redis_socket_location
+dump.rdb
 
 # Media Library
 integreat_cms/media/
@@ -68,3 +69,7 @@ integreat_cms/xliff/download
 
 # Postgres folder
 .postgres
+
+# Celery
+celerybeat-schedule.db
+celerybeat-schedule

docs/src/debugging.rst

@@ -45,6 +45,40 @@ VSCodium
 ^^^^^^^^
 Coming soon(TM)?
 
+PyCharm (Professional)
+^^^^^^^^^^^^^^^^^^^^^^
+
+PyCharm Professional has built-in support for django projects, but it needs some configuration to work with the integreat CMS.
+This is unfortunately not the case for the free edition of PyCharm. Students can get a free license for the professional version, however.
+
+Enable Django Support
+---------------------
+
+#. Go to settings → Languages & Frameworks → Django
+#. Click the ``Enable Django Support`` Checkbox
+#. Set the root ``integreat_cms`` directory as the project root
+#. For ``Settings`` use ``integreat_cms/core/docker_settings.py``
+#. Check ``Do not use Django test runner``
+#. For ``Manage script``, use ``integreat_cms/integreat-cms-cli``. If PyCharm does not let you select this script, because it does not end in .py, you can manually specify it in ``.idea/integreat-cms.iml``.
+
+Your configuration should now look similar to this:
+
+   .. image:: images/debugging/debug-pycharm-01-django-config.png
+      :alt: Django configuration
+
+Create a Run Configuration
+--------------------------
+
+#. Create a new ``Django Server`` run configuration
+#. Use a different port (For example 8001) to avoid conflicts with the non-debug server at port 8000
+#. At `Environment Variables`, add these: ``PYTHONUNBUFFERED=1;DJANGO_SETTINGS_MODULE=integreat_cms.core.docker_settings;INTEGREAT_CMS_DEBUG=True``
+
+Start Debugging
+---------------
+
+#. First execute the ``./tools/run.sh`` to make sure that the database is available and all assets are compiled.
+#. Once the server has started, you can start debugging by launching the run configuration
+
 Neovim
 ^^^^^^
 

flake.nix

@@ -50,6 +50,13 @@
             createdb -h "$INTEGREAT_CMS_DB_HOST" -p "$INTEGREAT_CMS_DB_PORT" -U "$INTEGREAT_CMS_DB_USER" "$INTEGREAT_CMS_DB_NAME"
           '';
         };
+        redis = pkgs.writeShellApplication {
+          name = "redis";
+          runtimeInputs = [ pkgs.redis ];
+          text = ''
+            redis-server --daemonize yes --unixsocket "$INTEGREAT_CMS_REDIS_SOCKET_LOCATION" --port "$INTEGREAT_CMS_REDIS_PORT" > /dev/null
+          '';
+        };
       in
       {
         devShells.default = pkgs.mkShell {
@@ -72,6 +79,7 @@
             pg-start
             pg-stop
             pg-reset
+            redis
           ];
 
           LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib pkgs.file ];
@@ -105,6 +113,10 @@
             set_if_unset INTEGREAT_CMS_SUMM_AI_API_KEY "dummy"
             set_if_unset INTEGREAT_CMS_LINKCHECK_DISABLE_LISTENERS 1
 
+            # Start redis and make it discoverable to the CMS
+            set_if_unset INTEGREAT_CMS_REDIS_SOCKET_LOCATION "./.redis_socket_location"
+            set_if_unset INTEGREAT_CMS_REDIS_PORT 6379
+
             # Setting LD_LIBRARY_PATH can cause issues on non-NixOS systems
             if ! command -v nixos-version &> /dev/null; then
                 unset LD_LIBRARY_PATH

integreat_cms/api/v3/chat/user_chat.py

@@ -7,7 +7,6 @@
 import json
 import logging
 import random
-import socket
 from typing import TYPE_CHECKING
 
 from django.http import HttpResponse, JsonResponse
@@ -17,8 +16,8 @@
 
 from ....cms.models import ABTester, AttachmentMap, Language, Region, UserChat
 from ...decorators import json_response
-from .chat_bot import ChatBot
-from .zammad_api import ZammadChatAPI
+from .utils.chat_bot import process_answer, process_user_message
+from .utils.zammad_api import ZammadChatAPI
 
 if TYPE_CHECKING:
     from django.http import HttpRequest
@@ -227,15 +226,13 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse:
     """
     Receive webhooks from Zammad to update the latest article translation
     """
-    zammad_url = (
-        f"https://{socket.getnameinfo((request.META.get('REMOTE_ADDR'), 0), 0)[0]}"
+    region = get_object_or_404(
+        Region, zammad_webhook_token=request.GET.get("token", None)
     )
-    region = get_object_or_404(Region, zammad_url=zammad_url)
-    client = ZammadChatAPI(region)
+    if not region.integreat_chat_enabled:
+        return JsonResponse({"status": "Integreat Chat disabled"})
     webhook_message = json.loads(request.body)
     message_text = webhook_message["article"]["body"]
-    zammad_chat = UserChat.objects.get(zammad_id=webhook_message["ticket"]["id"])
-    chat_bot = ChatBot()
 
     actions = []
     if webhook_message["article"]["internal"]:
@@ -249,34 +246,14 @@ def zammad_webhook(request: HttpRequest) -> JsonResponse:
         webhook_message["article"]["created_by"]["login"]
         == "[email protected]"
     ):
-        actions.append("question translation")
-        client.send_message(
-            zammad_chat.zammad_id,
-            chat_bot.automatic_translation(
-                message_text, zammad_chat.language.slug, region.default_language.slug
-            ),
-            True,
-            True,
+        actions.append("question translation queued")
+        process_user_message.apply_async(
+            args=[message_text, region.slug, webhook_message["ticket"]["id"]]
         )
-        if answer := chat_bot.automatic_answer(
-            message_text, region, zammad_chat.language.slug
-        ):
-            actions.append("automatic answer")
-            client.send_message(
-                zammad_chat.zammad_id,
-                answer,
-                False,
-                True,
-            )
     else:
-        actions.append("answer translation")
-        client.send_message(
-            zammad_chat.zammad_id,
-            chat_bot.automatic_translation(
-                message_text, region.default_language.slug, zammad_chat.language.slug
-            ),
-            False,
-            True,
+        actions.append("answer translation queued")
+        process_answer.apply_async(
+            args=[message_text, region.slug, webhook_message["ticket"]["id"]]
         )
     return JsonResponse(
         {

integreat_cms/api/v3/chat/utils/__init__.py

@@ -0,0 +1,3 @@
+"""
+Utils for the Integreat Chat
+"""

integreat_cms/api/v3/chat/utils/chat_bot.py

@@ -0,0 +1,111 @@
+"""
+Wrapper for the Chat Bot / LLM API
+"""
+
+from __future__ import annotations
+
+import requests
+from celery import shared_task
+from django.conf import settings
+
+from integreat_cms.cms.models import Region, UserChat
+from integreat_cms.cms.utils.content_translation_utils import (
+    get_public_translation_for_link,
+)
+
+from .zammad_api import ZammadChatAPI
+
+
+def format_message(response: dict) -> str:
+    """
+    Transform JSON into readable message
+    """
+    if "answer" not in response or not response["answer"]:
+        raise ValueError("Could not format message, no answer attribute in response")
+    if "sources" not in response or not response["sources"]:
+        return response["answer"]
+    sources = "".join(
+        [
+            f"<li><a href='{settings.WEBAPP_URL}{path}'>{title}</a></li>"
+            for path in response["sources"]
+            if (title := get_public_translation_for_link(settings.WEBAPP_URL + path))
+        ]
+    )
+    return f"{response['answer']}\n<ul>{sources}</ul>"
+
+
+def automatic_answer(message: str, region: Region, language_slug: str) -> str | None:
+    """
+    Get automatic answer to question
+    """
+    url = (
+        f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/extract_answer/"
+    )
+    body = {"message": message, "language": language_slug, "region": region.slug}
+    r = requests.post(url, json=body, timeout=120)
+    return format_message(r.json())
+
+
+def automatic_translation(
+    message: str, source_language_slug: str, target_language_slug: str
+) -> str:
+    """
+    Use LLM to translate message
+    """
+    url = f"https://{settings.INTEGREAT_CHAT_BACK_END_DOMAIN}/chatanswers/translate_message/"
+    body = {
+        "message": message,
+        "source_language": source_language_slug,
+        "target_language": target_language_slug,
+    }
+    response = requests.post(url, json=body, timeout=120).json()
+    if "status" in response and response["status"] == "success":
+        return response["translation"]
+    raise ValueError("Did not receive success response for translation request.")
+
+
+@shared_task
+def process_user_message(
+    message_text: str, region_slug: str, zammad_ticket_id: int
+) -> None:
+    """
+    Process the message from an Integreat App user
+    """
+    zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id)
+    region = Region.objects.get(slug=region_slug)
+    client = ZammadChatAPI(region)
+    if translation := automatic_translation(
+        message_text, zammad_chat.language.slug, region.default_language.slug
+    ):
+        client.send_message(
+            zammad_chat.zammad_id,
+            translation,
+            True,
+            True,
+        )
+    if answer := automatic_answer(message_text, region, zammad_chat.language.slug):
+        client.send_message(
+            zammad_chat.zammad_id,
+            answer,
+            False,
+            True,
+        )
+
+
+@shared_task
+def process_answer(message_text: str, region_slug: str, zammad_ticket_id: int) -> None:
+    """
+    Process automatic or counselor answers
+    """
+    zammad_chat = UserChat.objects.get(zammad_id=zammad_ticket_id)
+    region = Region.objects.get(slug=region_slug)
+    client = ZammadChatAPI(region)
+    if translation := automatic_translation(
+        message_text, region.default_language.slug, zammad_chat.language.slug
+    ):
+        client.send_message(
+            zammad_chat.zammad_id,
+            translation,
+            False,
+            True,
+        )