Drop support for the .lang format (mozilla#3163)

* Fix remaining failing test --------- Co-authored-by: Matjaž Horvat <[email protected]>
mathjazz · Apr 4, 2024 · e996d94 · e996d94
1 parent de25e2d
commit e996d94
Show file tree

Hide file tree

Showing 37 changed files with 116 additions and 667 deletions.
diff --git a/docs/index.rst b/docs/index.rst
@@ -5,11 +5,16 @@ Pontoon - Mozilla's Localization Platform
 system used and developed by the Mozilla localization community.
 It can handle any project that uses one of the supported file formats:
 
-+-------+---------------+-------------+--------+-----------------------+-------------------+
-| .dtd  | .ftl (Fluent) | .inc        | .ini   | .json (WebExtensions) | .json (key-value) |
-+-------+---------------+-------------+--------+-----------------------+-------------------+
-| .lang | .po (Gettext) | .properties | .xliff | .xml (Android)        |                   |
-+-------+---------------+-------------+--------+-----------------------+-------------------+
+- .dtd
+- .ftl (Fluent)
+- .inc
+- .ini
+- .json (WebExtensions)
+- .json (key-value)
+- .po (Gettext)
+- .properties
+- .xliff
+- .xml (Android)
 
 Pontoon can pull strings it needs to translate from an external source, and write
 them back periodically. Typically these external sources are version control

diff --git a/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py b/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py
@@ -441,12 +441,7 @@ class Migration(migrations.Migration):
                 ("rejected", models.BooleanField(default=False)),
                 ("rejected_date", models.DateTimeField(blank=True, null=True)),
                 ("unrejected_date", models.DateTimeField(blank=True, null=True)),
-                (
-                    "extra",
-                    jsonfield.fields.JSONField(
-                        default=pontoon.base.models.extra_default
-                    ),
-                ),
+                ("extra", jsonfield.fields.JSONField(default=dict)),
                 (
                     "approved_user",
                     models.ForeignKey(

diff --git a/pontoon/base/migrations/0057_remove_lang_format.py b/pontoon/base/migrations/0057_remove_lang_format.py
@@ -0,0 +1,38 @@
+# Generated by Django 3.2.24 on 2024-04-02 18:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0056_remove_fuzzy_and_non-approved_tm_entries"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="translation",
+            name="extra",
+        ),
+        migrations.AlterField(
+            model_name="resource",
+            name="format",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("dtd", "dtd"),
+                    ("ftl", "ftl"),
+                    ("inc", "inc"),
+                    ("ini", "ini"),
+                    ("json", "json"),
+                    ("po", "po"),
+                    ("properties", "properties"),
+                    ("xlf", "xliff"),
+                    ("xliff", "xliff"),
+                    ("xml", "xml"),
+                ],
+                max_length=20,
+                verbose_name="Format",
+            ),
+        ),
+    ]
diff --git a/pontoon/base/models.py b/pontoon/base/models.py
@@ -2134,7 +2134,6 @@ class Format(models.TextChoices):
         INC = "inc", "inc"
         INI = "ini", "ini"
         JSON = "json", "json"
-        LANG = "lang", "lang"
         PO = "po", "po"
         PROPERTIES = "properties", "properties"
         XLF = "xlf", "xliff"
@@ -3095,11 +3094,6 @@ class Meta:
         unique_together = ("entity", "locale")
 
 
-def extra_default():
-    """Default value for the Translation.extra field."""
-    return {}
-
-
 class TranslationQuerySet(models.QuerySet):
     def translated_resources(self, locale):
         return TranslatedResource.objects.filter(
@@ -3251,11 +3245,6 @@ class MachinerySource(models.TextChoices):
 
     objects = TranslationQuerySet.as_manager()
 
-    # extra stores data that we want to save for the specific format
-    # this translation is stored in, but that we otherwise don't care
-    # about.
-    extra = JSONField(default=extra_default)
-
     class Meta:
         index_together = (
             ("entity", "user", "approved", "pretranslated"),

diff --git a/pontoon/checks/__init__.py b/pontoon/checks/__init__.py
@@ -3,7 +3,6 @@
     "dtd",
     "ftl",
     "ini",
-    "lang",
     "properties",
     "xml",
 )

diff --git a/pontoon/checks/libraries/__init__.py b/pontoon/checks/libraries/__init__.py
@@ -64,8 +64,6 @@ def run_checks(
                         "startwhitespace",
                     ]
                 )
-        elif resource_ext == "lang":
-            tt_disabled_checks.update(["newlines"])
 
         tt_checks = translate_toolkit.run_checks(
             original, string, locale_code, tt_disabled_checks

diff --git a/pontoon/checks/libraries/pontoon_db.py b/pontoon/checks/libraries/pontoon_db.py
@@ -1,31 +1,13 @@
-import html
-import re
-
-import bleach
-
 from collections import defaultdict
 from fluent.syntax import FluentParser, ast
 from fluent.syntax.visitor import Visitor
 
 from pontoon.sync.formats.ftl import localizable_entries
 
 
-MAX_LENGTH_RE = re.compile(r"MAX_LENGTH:( *)(\d+)", re.MULTILINE)
 parser = FluentParser()
 
 
-def get_max_length(comment):
-    """
-    Return max length value for an entity with MAX_LENTH.
-    """
-    max_length = re.findall(MAX_LENGTH_RE, comment or "")
-
-    if max_length:
-        return int(max_length[0][1])
-
-    return None
-
-
 class IsEmptyVisitor(Visitor):
     def __init__(self):
         self.is_empty = False
@@ -61,22 +43,6 @@ def run_checks(entity, original, string):
     checks = defaultdict(list)
     resource_ext = entity.resource.format
 
-    if resource_ext == "lang":
-        # Newlines are not allowed in .lang files (bug 1190754)
-        if "\n" in string:
-            checks["pErrors"].append("Newline characters are not allowed")
-
-        # Prevent translations exceeding the given length limit
-        max_length = get_max_length(entity.comment)
-
-        if max_length:
-            string_length = len(
-                html.unescape(bleach.clean(string, strip=True, tags=()))
-            )
-
-            if string_length > max_length:
-                checks["pErrors"].append("Translation too long")
-
     # Bug 1599056: Original and translation must either both end in a newline,
     # or none of them should.
     if resource_ext == "po":

diff --git a/pontoon/checks/tests/test_pontoon_db.py b/pontoon/checks/tests/test_pontoon_db.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from pontoon.checks.libraries.pontoon_db import get_max_length, run_checks
+from pontoon.checks.libraries.pontoon_db import run_checks
 
 
 @pytest.fixture()
@@ -23,83 +23,6 @@ def _f(extension, comment="", string="", allows_empty_translations=False):
     yield _f
 
 
-@pytest.mark.parametrize(
-    "comment, expected",
-    (
-        ("MAX_LENGTH: 24", 24),
-        ("MAX_LENGTH: 4", 4),
-        ("MAX_LENGTH:  4", 4),
-        ("MAX_LENGTH:4 ", 4),
-        ("MAX_LENGTH:  42  ", 42),
-        ("MAX_LENGTH:  42\n MAX_LENGTH: 10 ", 42),
-        ("MAX_LENGTH: 123 characters", 123),
-        ("MAX_LENGTH: 4\naaaa", 4),
-        ("bbbb \n MAX_LENGTH: 4\naaaa", 4),
-        ("MAX_LENGTH: 4 characters\naaaa", 4),
-        ("bbbb\nMAX_xLENGTH: 4 characters\naaaa", None),
-        ("bbbb\nMAX_LENGTH: z characters\naaaa", None),
-        ("bbbb\nMAX_LENGTH:\n 4 characters\naaaa", None),
-    ),
-)
-def test_too_long_translation_max_length(comment, expected):
-    """
-    Checks should return an error if a translation is too long.
-    """
-    assert get_max_length(comment) == expected
-
-
-def test_too_long_translation_valid_length(get_entity_mock):
-    """
-    Checks shouldn't return an error if a translation isn't too long.
-    """
-    assert run_checks(get_entity_mock("lang", "MAX_LENGTH: 4"), "", "0123") == {}
-
-
-def test_too_long_translation_html_tags(get_entity_mock):
-    """
-    HTML tags can't be included in the MAX_LENGTH check.
-    """
-    assert (
-        run_checks(
-            get_entity_mock("lang", "MAX_LENGTH: 4"),
-            "",
-            '<a href="pontoon.mozilla.org">01</a><i>23</i>',
-        )
-        == {}
-    )
-
-    assert run_checks(
-        get_entity_mock("lang", "MAX_LENGTH: 4"),
-        "",
-        '<a href="pontoon.mozilla.org">012</a><i>23</i>',
-    ) == {"pErrors": ["Translation too long"]}
-
-    # Check if entities are causing false errors
-    assert (
-        run_checks(
-            get_entity_mock("lang", "MAX_LENGTH: 4"),
-            "",
-            '<a href="pontoon.mozilla.org">ł&nbsp;</a><i>ń&nbsp;</i>',
-        )
-        == {}
-    )
-
-    assert run_checks(
-        get_entity_mock("lang", "MAX_LENGTH: 4"),
-        "",
-        '<a href="pontoon.mozilla.org">ł&nbsp;&nbsp;</a><i>ń&nbsp;</i>',
-    ) == {"pErrors": ["Translation too long"]}
-
-
-def test_too_long_translation_invalid_length(get_entity_mock):
-    """
-    Checks should return an error if a translation is too long.
-    """
-    assert run_checks(get_entity_mock("lang", "MAX_LENGTH: 2"), "", "0123") == {
-        "pErrors": ["Translation too long"]
-    }
-
-
 def test_ending_newline(get_entity_mock):
     """
     Original and translation in a PO file must either both end
@@ -196,12 +119,7 @@ def test_empty_translations(get_entity_mock):
     )
 
 
-def test_lang_newlines(get_entity_mock):
-    """Newlines aren't allowed in lang files"""
-    assert run_checks(get_entity_mock("lang"), "", "aaa\nbbb") == {
-        "pErrors": ["Newline characters are not allowed"]
-    }
-
+def test_po_newlines(get_entity_mock):
     assert run_checks(get_entity_mock("po"), "", "aaa\nbbb") == {}
 
 

diff --git a/pontoon/sync/changeset.py b/pontoon/sync/changeset.py
@@ -299,7 +299,6 @@ def update_entity_translations_from_vcs(
                     db_translation.approved_date = self.now
                 db_translation.rejected = False
                 db_translation.fuzzy = vcs_translation.fuzzy
-                db_translation.extra = vcs_translation.extra
 
                 if db_translation.is_dirty():
                     self.translations_to_update[db_translation.pk] = db_translation
@@ -323,7 +322,6 @@ def update_entity_translations_from_vcs(
                         approved_date=self.now if not vcs_translation.fuzzy else None,
                         user=user,
                         fuzzy=vcs_translation.fuzzy,
-                        extra=vcs_translation.extra,
                     )
                 )
 
@@ -488,7 +486,6 @@ def bulk_update_translations(self):
                     "approved_date",
                     "rejected",
                     "fuzzy",
-                    "extra",
                 ],
             )
 

diff --git a/pontoon/sync/formats/__init__.py b/pontoon/sync/formats/__init__.py
@@ -10,7 +10,6 @@
     ftl,
     json_extensions,
     json_keyvalue,
-    lang,
     po,
     silme,
     xliff,
@@ -27,7 +26,6 @@
     "*.ini": silme.parse_ini,
     "*messages.json": json_extensions.parse,
     "*.json": json_keyvalue.parse,
-    "*.lang": lang.parse,
     "*.po": po.parse,
     "*.pot": po.parse,
     "*.properties": silme.parse_properties,
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,7 +3,6 @@ @@
         "dtd",
         "ftl",
         "ini",
-        "lang",
         "properties",
         "xml",
     )
@@ Expand Down @@