Squashed commit of the following:

commit 803cc09837acc734ccdafc7e4b1d4adc553a7355 Author: Marco Bonetti <[email protected]> Date: Mon Oct 7 11:59:30 2024 +0200 Changelog commit 4808f4abc5c6830b609c9e1bafc4ba5daf67a19c Author: Marco Bonetti <[email protected]> Date: Mon Oct 7 11:56:55 2024 +0200 fix flake8 commit 38781c1 Author: Halit Celik <[email protected]> Date: Mon Oct 7 11:53:52 2024 +0200 remove venv commit 46d4fdf Merge: b32840d 1a341ad Author: Halit Celik <[email protected]> Date: Mon Oct 7 10:53:43 2024 +0200 merge remote and fix conflict commit b32840d Author: Halit Celik <[email protected]> Date: Mon Oct 7 10:41:56 2024 +0200 rename function and fix tests commit 2dbc232 Author: Halit Celik <[email protected]> Date: Mon Sep 30 13:26:05 2024 +0200 send variable type as html attribute commit 7e1c1cf Author: Halit Celik <[email protected]> Date: Mon Jun 10 15:07:57 2024 +0200 remove content encodig gzip from vcr cassettes commit 872f081 Merge: 3cc0680 62422e4 Author: Halit Celik <[email protected]> Date: Mon Jun 10 14:46:21 2024 +0200 sync fork commit 3cc0680 Author: Halit Celik <[email protected]> Date: Mon Jun 10 14:39:37 2024 +0200 Wraps and unwraps django variables in xml tags for deepl not to translate them
mbi · Oct 7, 2024 · 6139629 · 6139629
1 parent 1a341ad
commit 6139629
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 4 deletions.
diff --git a/CHANGES b/CHANGES
@@ -5,6 +5,8 @@ Version 0.10.2 (unreleased)
 ---------------------------
 * Tests: update flake8 in tox tests
 * Format all rendered assets (html, css, js) in a pre-commit task. (PR #294, thanks @balazs-endresz)
+* Fix Deepl translations containing variables (#276, PR #290, thanks @halitcelik)
+
 
 
 Version 0.10.1

diff --git a/rosetta/static/admin/rosetta/js/rosetta.js b/rosetta/static/admin/rosetta/js/rosetta.js
@@ -24,7 +24,7 @@ $(document).ready(function () {
 
                 orig = unescape(orig)
                     .replace(/<br\s?\/?>/g, "\n")
-                    .replace(/<code>/, "")
+                    .replace(/<code>/g, "")
                     .replace(/<\/code>/g, "")
                     .replace(/&gt;/g, ">")
                     .replace(/&lt;/g, "<");

diff --git a/rosetta/tests/tests.py b/rosetta/tests/tests.py
@@ -1009,6 +1009,39 @@ def test_47_2_deeps_ajax_translation(self):
         )
         self.assertContains(r, '"Salut tout le monde"')
 
+    @vcr.use_cassette(
+        "fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml",
+        match_on=["method", "scheme", "port", "path", "query", "raw_body"],
+        record_mode="once",
+    )
+    @override_settings(
+        DEEPL_AUTH_KEY="FAKE",
+        AZURE_CLIENT_SECRET=None,
+    )
+    def test_deepl_ajax_translation_with_variables(self):
+        text = "Ci sono %(items)d %(name)s disponibili."
+        r = self.client.get(
+            reverse("rosetta.translate_text") + f"?from=it&to=en&text={text}"
+        )
+        self.assertEqual(
+            r.json().get("translation"), "There are %(items)d %(name)s available."
+        )
+
+    def test_formating_text_to_and_from_deepl(self):
+        from ..translate_utils import format_text_from_deepl, format_text_to_deepl
+
+        samples = [
+            "Es gibt %(items)d %(name)s verfügbar.",
+            "Ci sono %(items)d %(name)s disponibili.",
+            "Há %(items)d %(name)s disponíveis.",
+            "Stokta %(items)d %(name)s var.",
+        ]
+        for sample in samples:
+            to_deepl = format_text_to_deepl(sample)
+            from_deepl = format_text_from_deepl(to_deepl)
+            back_to_deepl = format_text_to_deepl(from_deepl)
+            self.assertEqual(to_deepl, back_to_deepl)
+
     @override_settings(ROSETTA_REQUIRES_AUTH=True)
     def test_48_requires_auth_not_respected_issue_203(self):
         self.client.logout()

diff --git a/rosetta/translate_utils.py b/rosetta/translate_utils.py
@@ -1,4 +1,5 @@
 import json
+import re
 import uuid
 
 import requests
@@ -50,7 +51,41 @@ def translate(text, from_language, to_language):
         raise TranslationException("No translation API service is configured.")
 
 
+def format_text_to_deepl(text):
+    pattern = r"%\((\w+)\)(\w)"
+
+    def replace_variable(match):
+        # Our pattern will always catch 2 groups, the first group being '%('
+        # Second group being ')d' or ')s'
+        variable = match.group(1)
+        type_specifier = match.group(2)
+        if variable and type_specifier:
+            return f'<var type="{type_specifier}">{variable}</var>'
+        else:
+            raise TranslationException("Badly formatted variable in translation")
+
+    return re.sub(pattern, replace_variable, text)
+
+
+def format_text_from_deepl(text):
+    for g in re.finditer(
+        r'.*?<var type="(?P<type>[rsd])">(?P<variable>[0-9a-zA-Z_]+)</var>.*?', text
+    ):
+        t, v = g.groups()
+        text = text.replace(f'<var type="{t}">{v}</var>', f"%({v}){t}")
+    return text
+
+
 def translate_by_deepl(text, to_language, auth_key):
+    """
+    This method connects to the translator Deepl API and fetches a response with translations.
+    :param text: The source text to be translated
+    :param to_language: The target language to translate the text into
+    Wraps variables in <var></var> tags and instructs Deepl not to translate those.
+    Then from Deepl response, converts back these tags to django variable syntax.
+    %(name)s becomes <var type="s">name</var> and back to %(name)s in the response text.
+    :return: Returns the response from the Deepl as a python object.
+    """
     if auth_key.lower().endswith(":fx"):
         endpoint = "https://api-free.deepl.com"
     else:
@@ -60,16 +95,19 @@ def translate_by_deepl(text, to_language, auth_key):
         f"{endpoint}/v2/translate",
         headers={"Authorization": f"DeepL-Auth-Key {auth_key}"},
         data={
+            "tag_handling": "xml",
+            "ignore_tags": "var",
             "target_lang": to_language.upper(),
-            "text": text,
+            "text": format_text_to_deepl(text),
         },
     )
     if r.status_code != 200:
         raise TranslationException(
             f"Deepl response is {r.status_code}. Please check your API key or try again later."
         )
+
     try:
-        return r.json().get("translations")[0].get("text")
+        return format_text_from_deepl(r.json().get("translations")[0].get("text"))
     except Exception:
         raise TranslationException("Deepl returned a non-JSON or unexpected response.")
 

diff --git a/testproject/fixtures/vcr_cassettes/test_47_2_deeps_ajax_translation.yaml b/testproject/fixtures/vcr_cassettes/test_47_2_deeps_ajax_translation.yaml
@@ -1,6 +1,6 @@
 interactions:
 - request:
-    body: target_lang=FR&text=hello+world
+    body: tag_handling=xml&ignore_tags=var&target_lang=FR&text=hello+world
     headers:
       Accept:
       - '*/*'

diff --git a/testproject/fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml b/testproject/fixtures/vcr_cassettes/test_deepl_ajax_translation_with_variables.yaml
@@ -0,0 +1,46 @@
+interactions:
+- request:
+    body: tag_handling=xml&ignore_tags=var&target_lang=EN&text=Ci+sono+%3Cvar+type%3D%22d%22%3Eitems%3C%2Fvar%3E+%3Cvar+type%3D%22s%22%3Ename%3C%2Fvar%3E+disponibili.
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Encoding:
+      - gzip, deflate
+      Authorization:
+      - DeepL-Auth-Key FAKE
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '156'
+      Content-Type:
+      - application/x-www-form-urlencoded
+      User-Agent:
+      - python-requests/2.32.3
+    method: POST
+    uri: https://api-free.deepl.com/v2/translate
+  response:
+    body:
+      string: '{"translations": [{"detected_source_language": "IT", "text": "There are <var type=\"d\">items</var> <var type=\"s\">name</var> available."}]}'
+    headers:
+      access-control-allow-origin:
+      - '*'
+      access-control-expose-headers:
+      - Server-Timing, X-Trace-ID
+      content-type:
+      - application/json
+      date:
+      - Mon, 07 Oct 2024 07:54:03 GMT
+      server-timing:
+      - l7_lb_tls;dur=77, l7_lb_idle;dur=2, l7_lb_receive;dur=1, l7_lb_total;dur=124
+      strict-transport-security:
+      - max-age=63072000; includeSubDomains; preload
+      transfer-encoding:
+      - chunked
+      vary:
+      - Accept-Encoding
+      x-trace-id:
+      - 0ec04e3eef784472bedc5be15a1f8259
+    status:
+      code: 200
+      message: OK
+version: 1