From e996d94cfafff6e289efe5364fbbc6538ac2040c Mon Sep 17 00:00:00 2001 From: Eemeli Aro Date: Thu, 4 Apr 2024 20:54:44 +0300 Subject: [PATCH] Drop support for the .lang format (#3163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix remaining failing test --------- Co-authored-by: Matjaž Horvat --- docs/index.rst | 15 +- .../0001_squashed_0154_auto_20200206_1736.py | 7 +- .../migrations/0057_remove_lang_format.py | 38 ++++ pontoon/base/models.py | 11 - pontoon/checks/__init__.py | 1 - pontoon/checks/libraries/__init__.py | 2 - pontoon/checks/libraries/pontoon_db.py | 34 --- pontoon/checks/tests/test_pontoon_db.py | 86 +------- pontoon/sync/changeset.py | 3 - pontoon/sync/formats/__init__.py | 2 - pontoon/sync/formats/lang.py | 208 ------------------ pontoon/sync/tests/__init__.py | 13 +- .../fake-checkout/inactive-locale/main.lang | 5 - .../fake-checkout/inactive-locale/main.po | 5 + .../fake-checkout/inactive-locale/other.lang | 5 - .../fake-checkout/inactive-locale/other.po | 6 + .../tests/fake-checkout/templates/main.lang | 5 - .../tests/fake-checkout/templates/main.pot | 6 + .../fake-checkout/templates/missing.lang | 2 - .../tests/fake-checkout/templates/missing.pot | 2 + .../tests/fake-checkout/templates/other.lang | 5 - .../tests/fake-checkout/templates/other.pot | 5 + .../fake-checkout/translated-locale/main.lang | 5 - .../fake-checkout/translated-locale/main.po | 5 + .../translated-locale/other.lang | 5 - .../fake-checkout/translated-locale/other.po | 5 + pontoon/sync/tests/formats/bom.lang | 4 - pontoon/sync/tests/formats/test_lang.py | 173 --------------- pontoon/sync/tests/test_changeset.py | 9 +- pontoon/sync/tests/test_checks.py | 20 +- pontoon/sync/tests/test_core.py | 6 +- pontoon/sync/vcs/models.py | 9 - requirements/default.in | 1 - requirements/default.txt | 5 - .../editor/components/TranslationLength.css | 4 - .../components/TranslationLength.test.js | 38 +--- .../editor/components/TranslationLength.tsx | 28 +-- 37 files changed, 116 insertions(+), 667 deletions(-) create mode 100644 pontoon/base/migrations/0057_remove_lang_format.py delete mode 100644 pontoon/sync/formats/lang.py delete mode 100644 pontoon/sync/tests/fake-checkout/inactive-locale/main.lang create mode 100644 pontoon/sync/tests/fake-checkout/inactive-locale/main.po delete mode 100644 pontoon/sync/tests/fake-checkout/inactive-locale/other.lang create mode 100644 pontoon/sync/tests/fake-checkout/inactive-locale/other.po delete mode 100644 pontoon/sync/tests/fake-checkout/templates/main.lang create mode 100644 pontoon/sync/tests/fake-checkout/templates/main.pot delete mode 100644 pontoon/sync/tests/fake-checkout/templates/missing.lang create mode 100644 pontoon/sync/tests/fake-checkout/templates/missing.pot delete mode 100644 pontoon/sync/tests/fake-checkout/templates/other.lang create mode 100644 pontoon/sync/tests/fake-checkout/templates/other.pot delete mode 100644 pontoon/sync/tests/fake-checkout/translated-locale/main.lang create mode 100644 pontoon/sync/tests/fake-checkout/translated-locale/main.po delete mode 100644 pontoon/sync/tests/fake-checkout/translated-locale/other.lang create mode 100644 pontoon/sync/tests/fake-checkout/translated-locale/other.po delete mode 100644 pontoon/sync/tests/formats/bom.lang delete mode 100644 pontoon/sync/tests/formats/test_lang.py diff --git a/docs/index.rst b/docs/index.rst index 67751efa19..7fb1458c61 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,11 +5,16 @@ Pontoon - Mozilla's Localization Platform system used and developed by the Mozilla localization community. It can handle any project that uses one of the supported file formats: -+-------+---------------+-------------+--------+-----------------------+-------------------+ -| .dtd | .ftl (Fluent) | .inc | .ini | .json (WebExtensions) | .json (key-value) | -+-------+---------------+-------------+--------+-----------------------+-------------------+ -| .lang | .po (Gettext) | .properties | .xliff | .xml (Android) | | -+-------+---------------+-------------+--------+-----------------------+-------------------+ +- .dtd +- .ftl (Fluent) +- .inc +- .ini +- .json (WebExtensions) +- .json (key-value) +- .po (Gettext) +- .properties +- .xliff +- .xml (Android) Pontoon can pull strings it needs to translate from an external source, and write them back periodically. Typically these external sources are version control diff --git a/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py b/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py index c9106b854e..41d5b4cb9e 100644 --- a/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py +++ b/pontoon/base/migrations/0001_squashed_0154_auto_20200206_1736.py @@ -441,12 +441,7 @@ class Migration(migrations.Migration): ("rejected", models.BooleanField(default=False)), ("rejected_date", models.DateTimeField(blank=True, null=True)), ("unrejected_date", models.DateTimeField(blank=True, null=True)), - ( - "extra", - jsonfield.fields.JSONField( - default=pontoon.base.models.extra_default - ), - ), + ("extra", jsonfield.fields.JSONField(default=dict)), ( "approved_user", models.ForeignKey( diff --git a/pontoon/base/migrations/0057_remove_lang_format.py b/pontoon/base/migrations/0057_remove_lang_format.py new file mode 100644 index 0000000000..936291b346 --- /dev/null +++ b/pontoon/base/migrations/0057_remove_lang_format.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.24 on 2024-04-02 18:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0056_remove_fuzzy_and_non-approved_tm_entries"), + ] + + operations = [ + migrations.RemoveField( + model_name="translation", + name="extra", + ), + migrations.AlterField( + model_name="resource", + name="format", + field=models.CharField( + blank=True, + choices=[ + ("dtd", "dtd"), + ("ftl", "ftl"), + ("inc", "inc"), + ("ini", "ini"), + ("json", "json"), + ("po", "po"), + ("properties", "properties"), + ("xlf", "xliff"), + ("xliff", "xliff"), + ("xml", "xml"), + ], + max_length=20, + verbose_name="Format", + ), + ), + ] diff --git a/pontoon/base/models.py b/pontoon/base/models.py index e318a087d0..2a899ec5b5 100644 --- a/pontoon/base/models.py +++ b/pontoon/base/models.py @@ -2134,7 +2134,6 @@ class Format(models.TextChoices): INC = "inc", "inc" INI = "ini", "ini" JSON = "json", "json" - LANG = "lang", "lang" PO = "po", "po" PROPERTIES = "properties", "properties" XLF = "xlf", "xliff" @@ -3095,11 +3094,6 @@ class Meta: unique_together = ("entity", "locale") -def extra_default(): - """Default value for the Translation.extra field.""" - return {} - - class TranslationQuerySet(models.QuerySet): def translated_resources(self, locale): return TranslatedResource.objects.filter( @@ -3251,11 +3245,6 @@ class MachinerySource(models.TextChoices): objects = TranslationQuerySet.as_manager() - # extra stores data that we want to save for the specific format - # this translation is stored in, but that we otherwise don't care - # about. - extra = JSONField(default=extra_default) - class Meta: index_together = ( ("entity", "user", "approved", "pretranslated"), diff --git a/pontoon/checks/__init__.py b/pontoon/checks/__init__.py index 4d91a7c12c..fa8a52edfe 100644 --- a/pontoon/checks/__init__.py +++ b/pontoon/checks/__init__.py @@ -3,7 +3,6 @@ "dtd", "ftl", "ini", - "lang", "properties", "xml", ) diff --git a/pontoon/checks/libraries/__init__.py b/pontoon/checks/libraries/__init__.py index 893b71328e..562cb4c2d3 100644 --- a/pontoon/checks/libraries/__init__.py +++ b/pontoon/checks/libraries/__init__.py @@ -64,8 +64,6 @@ def run_checks( "startwhitespace", ] ) - elif resource_ext == "lang": - tt_disabled_checks.update(["newlines"]) tt_checks = translate_toolkit.run_checks( original, string, locale_code, tt_disabled_checks diff --git a/pontoon/checks/libraries/pontoon_db.py b/pontoon/checks/libraries/pontoon_db.py index 8944359f86..9e911b2aaa 100644 --- a/pontoon/checks/libraries/pontoon_db.py +++ b/pontoon/checks/libraries/pontoon_db.py @@ -1,8 +1,3 @@ -import html -import re - -import bleach - from collections import defaultdict from fluent.syntax import FluentParser, ast from fluent.syntax.visitor import Visitor @@ -10,22 +5,9 @@ from pontoon.sync.formats.ftl import localizable_entries -MAX_LENGTH_RE = re.compile(r"MAX_LENGTH:( *)(\d+)", re.MULTILINE) parser = FluentParser() -def get_max_length(comment): - """ - Return max length value for an entity with MAX_LENTH. - """ - max_length = re.findall(MAX_LENGTH_RE, comment or "") - - if max_length: - return int(max_length[0][1]) - - return None - - class IsEmptyVisitor(Visitor): def __init__(self): self.is_empty = False @@ -61,22 +43,6 @@ def run_checks(entity, original, string): checks = defaultdict(list) resource_ext = entity.resource.format - if resource_ext == "lang": - # Newlines are not allowed in .lang files (bug 1190754) - if "\n" in string: - checks["pErrors"].append("Newline characters are not allowed") - - # Prevent translations exceeding the given length limit - max_length = get_max_length(entity.comment) - - if max_length: - string_length = len( - html.unescape(bleach.clean(string, strip=True, tags=())) - ) - - if string_length > max_length: - checks["pErrors"].append("Translation too long") - # Bug 1599056: Original and translation must either both end in a newline, # or none of them should. if resource_ext == "po": diff --git a/pontoon/checks/tests/test_pontoon_db.py b/pontoon/checks/tests/test_pontoon_db.py index 8b0d2b2b34..266bdbf669 100644 --- a/pontoon/checks/tests/test_pontoon_db.py +++ b/pontoon/checks/tests/test_pontoon_db.py @@ -2,7 +2,7 @@ import pytest -from pontoon.checks.libraries.pontoon_db import get_max_length, run_checks +from pontoon.checks.libraries.pontoon_db import run_checks @pytest.fixture() @@ -23,83 +23,6 @@ def _f(extension, comment="", string="", allows_empty_translations=False): yield _f -@pytest.mark.parametrize( - "comment, expected", - ( - ("MAX_LENGTH: 24", 24), - ("MAX_LENGTH: 4", 4), - ("MAX_LENGTH: 4", 4), - ("MAX_LENGTH:4 ", 4), - ("MAX_LENGTH: 42 ", 42), - ("MAX_LENGTH: 42\n MAX_LENGTH: 10 ", 42), - ("MAX_LENGTH: 123 characters", 123), - ("MAX_LENGTH: 4\naaaa", 4), - ("bbbb \n MAX_LENGTH: 4\naaaa", 4), - ("MAX_LENGTH: 4 characters\naaaa", 4), - ("bbbb\nMAX_xLENGTH: 4 characters\naaaa", None), - ("bbbb\nMAX_LENGTH: z characters\naaaa", None), - ("bbbb\nMAX_LENGTH:\n 4 characters\naaaa", None), - ), -) -def test_too_long_translation_max_length(comment, expected): - """ - Checks should return an error if a translation is too long. - """ - assert get_max_length(comment) == expected - - -def test_too_long_translation_valid_length(get_entity_mock): - """ - Checks shouldn't return an error if a translation isn't too long. - """ - assert run_checks(get_entity_mock("lang", "MAX_LENGTH: 4"), "", "0123") == {} - - -def test_too_long_translation_html_tags(get_entity_mock): - """ - HTML tags can't be included in the MAX_LENGTH check. - """ - assert ( - run_checks( - get_entity_mock("lang", "MAX_LENGTH: 4"), - "", - '0123', - ) - == {} - ) - - assert run_checks( - get_entity_mock("lang", "MAX_LENGTH: 4"), - "", - '01223', - ) == {"pErrors": ["Translation too long"]} - - # Check if entities are causing false errors - assert ( - run_checks( - get_entity_mock("lang", "MAX_LENGTH: 4"), - "", - 'ł ń ', - ) - == {} - ) - - assert run_checks( - get_entity_mock("lang", "MAX_LENGTH: 4"), - "", - 'ł  ń ', - ) == {"pErrors": ["Translation too long"]} - - -def test_too_long_translation_invalid_length(get_entity_mock): - """ - Checks should return an error if a translation is too long. - """ - assert run_checks(get_entity_mock("lang", "MAX_LENGTH: 2"), "", "0123") == { - "pErrors": ["Translation too long"] - } - - def test_ending_newline(get_entity_mock): """ Original and translation in a PO file must either both end @@ -196,12 +119,7 @@ def test_empty_translations(get_entity_mock): ) -def test_lang_newlines(get_entity_mock): - """Newlines aren't allowed in lang files""" - assert run_checks(get_entity_mock("lang"), "", "aaa\nbbb") == { - "pErrors": ["Newline characters are not allowed"] - } - +def test_po_newlines(get_entity_mock): assert run_checks(get_entity_mock("po"), "", "aaa\nbbb") == {} diff --git a/pontoon/sync/changeset.py b/pontoon/sync/changeset.py index b607160273..0c9cd89784 100644 --- a/pontoon/sync/changeset.py +++ b/pontoon/sync/changeset.py @@ -299,7 +299,6 @@ def update_entity_translations_from_vcs( db_translation.approved_date = self.now db_translation.rejected = False db_translation.fuzzy = vcs_translation.fuzzy - db_translation.extra = vcs_translation.extra if db_translation.is_dirty(): self.translations_to_update[db_translation.pk] = db_translation @@ -323,7 +322,6 @@ def update_entity_translations_from_vcs( approved_date=self.now if not vcs_translation.fuzzy else None, user=user, fuzzy=vcs_translation.fuzzy, - extra=vcs_translation.extra, ) ) @@ -488,7 +486,6 @@ def bulk_update_translations(self): "approved_date", "rejected", "fuzzy", - "extra", ], ) diff --git a/pontoon/sync/formats/__init__.py b/pontoon/sync/formats/__init__.py index 97eb9400ab..9a8c0444e7 100644 --- a/pontoon/sync/formats/__init__.py +++ b/pontoon/sync/formats/__init__.py @@ -10,7 +10,6 @@ ftl, json_extensions, json_keyvalue, - lang, po, silme, xliff, @@ -27,7 +26,6 @@ "*.ini": silme.parse_ini, "*messages.json": json_extensions.parse, "*.json": json_keyvalue.parse, - "*.lang": lang.parse, "*.po": po.parse, "*.pot": po.parse, "*.properties": silme.parse_properties, diff --git a/pontoon/sync/formats/lang.py b/pontoon/sync/formats/lang.py deleted file mode 100644 index 34545f9ad6..0000000000 --- a/pontoon/sync/formats/lang.py +++ /dev/null @@ -1,208 +0,0 @@ -""" -Parser for the .lang translation format. -""" -import codecs -import re - -from parsimonious.exceptions import ( - ParseError as ParsimoniousParseError, - VisitationError, -) -from parsimonious.grammar import Grammar -from parsimonious.nodes import NodeVisitor - -from pontoon.sync.exceptions import ParseError -from pontoon.sync.formats.base import ParsedResource -from pontoon.sync.vcs.models import VCSTranslation - - -BLANK_LINE = "blank_line" -TAG_REGEX = re.compile(r"\{(ok|l10n-extra)\}") - - -class LangComment: - def __init__(self, marker, content, end): - self.marker = marker - self.raw_content = content - self.end = end - - @property - def content(self): - return self.raw_content.strip() - - @property - def raw(self): - return self.marker + self.raw_content + self.end - - -class LangEntity(VCSTranslation): - def __init__(self, source_string, translation_string, tags): - super().__init__( - key=source_string, # Langfiles use the source as the key. - context="", - source_string=source_string, - strings={None: translation_string}, # Langfiles lack plural support - comments=[], - fuzzy=False, # Langfiles don't support fuzzy status - ) - - self.tags = set(tags) - - # If the translation matches the source string without the {ok} - # tag, then the translation isn't actually valid, so we remove - # it. - if source_string == translation_string and "ok" not in tags: - del self.strings[None] - - @property - def extra(self): - return {"tags": list(self.tags)} - - -class LangResource(ParsedResource): - def __init__(self, path, children): - self.path = path - self.children = children - - @property - def translations(self): - return [c for c in self.children if isinstance(c, LangEntity)] - - def save(self, locale): - with codecs.open(self.path, "w", "utf-8") as f: - for child in self.children: - if isinstance(child, LangEntity): - self.write_entity(f, child) - elif isinstance(child, LangComment): - f.write(child.raw) - elif child == BLANK_LINE: - f.write("\n") - - def write_entity(self, f, entity): - f.write(f";{entity.source_string}\n") - - translation = entity.strings.get(None, None) - if translation is None: - # No translation? Output the source string and remove {ok}. - translation = entity.source_string - entity.tags.discard("ok") - elif translation == entity.source_string: - # Translation is equal to the source? Include {ok}. - entity.tags.add("ok") - elif translation != entity.source_string: - # Translation is different? Remove {ok}, it's unneeded. - entity.tags.discard("ok") - - if entity.extra.get("tags"): - tags = [f"{{{t}}}" for t in entity.tags] - translation = "{translation} {tags}".format( - translation=translation, tags=" ".join(tags) - ) - - f.write(f"{translation}\n") - - -class LangVisitor(NodeVisitor): - grammar = Grammar( - r""" - lang_file = (comment / entity / blank_line)* - - comment = "#"+ line_content line_ending - line_content = ~r".*" - line_ending = ~r"$\n?"m # Match at EOL and EOF without newline. - - blank_line = ~r"((?!\n)\s)*" line_ending - - entity = string translation - string = ";" line_content line_ending - translation = line_content line_ending - """ - ) - - def visit_lang_file(self, node, children): - """ - Find comments that are associated with an entity and add them - to the entity's comments list. Also assign order to entities. - """ - comments = [] - order = 0 - for child in children: - if isinstance(child, LangComment): - comments.append(child) - continue - - if isinstance(child, LangEntity): - child.comments = [c.content for c in comments] - child.order = order - order += 1 - - comments = [] - - return children - - def visit_comment(self, node, node_info): - marker, content, end = node_info - return LangComment(node_text(marker), node_text(content), node_text(end)) - - def visit_blank_line(self, node, _): - return BLANK_LINE - - def visit_entity(self, node, node_info): - string, translation = node_info - - # Strip tags out of translation if they exist. - tags = [] - tag_matches = list(re.finditer(TAG_REGEX, translation)) - if tag_matches: - tags = [m.group(1) for m in tag_matches] - translation = translation[: tag_matches[0].start()].strip() - - if translation == "": - raise ParsimoniousParseError( - "Blank translation for key {key} is not allowed in langfiles.".format( - key=string - ) - ) - - return LangEntity(string, translation, tags) - - def visit_string(self, node, node_info): - marker, content, end = node_info - return content.text.strip() - - def visit_translation(self, node, node_info): - content, end = node_info - return content.text.strip() - - def generic_visit(self, node, children): - if children and len(children) == 1: - return children[0] - else: - return children or node - - -def node_text(node): - """ - Convert a Parsimonious node into text, including nodes that may - actually be a list of nodes due to repetition. - """ - if node is None: - return "" - elif isinstance(node, list): - return "".join([n.text for n in node]) - else: - return node.text - - -def parse(path, source_path=None, locale=None): - # Read as utf-8-sig in case there's a BOM at the start of the file - # that we want to remove. - with codecs.open(path, "r", "utf-8-sig") as f: - content = f.read() - - try: - children = LangVisitor().parse(content) - except (ParsimoniousParseError, VisitationError) as err: - raise ParseError(f"Failed to parse {path}: {err}") from err - - return LangResource(path, children) diff --git a/pontoon/sync/tests/__init__.py b/pontoon/sync/tests/__init__.py index 253a2c09cc..ce1c8f2317 100644 --- a/pontoon/sync/tests/__init__.py +++ b/pontoon/sync/tests/__init__.py @@ -100,13 +100,19 @@ def setUp(self): repositories=[self.repository], ) self.main_db_resource = ResourceFactory.create( - project=self.db_project, path="main.lang", format="lang" + project=self.db_project, + path="main.po", + format="po", ) self.other_db_resource = ResourceFactory.create( - project=self.db_project, path="other.lang", format="lang" + project=self.db_project, + path="other.po", + format="po", ) self.missing_db_resource = ResourceFactory.create( - project=self.db_project, path="missing.lang", format="lang" + project=self.db_project, + path="missing.po", + format="po", ) self.main_db_entity = EntityFactory.create( resource=self.main_db_resource, @@ -127,7 +133,6 @@ def setUp(self): string="Translated String", date=aware_datetime(1970, 1, 1), approved=True, - extra={"tags": []}, ) # Load paths from the fake locale directory. diff --git a/pontoon/sync/tests/fake-checkout/inactive-locale/main.lang b/pontoon/sync/tests/fake-checkout/inactive-locale/main.lang deleted file mode 100644 index 3107e47083..0000000000 --- a/pontoon/sync/tests/fake-checkout/inactive-locale/main.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Source String -Inactive String - -;Common String -Inactive Common String diff --git a/pontoon/sync/tests/fake-checkout/inactive-locale/main.po b/pontoon/sync/tests/fake-checkout/inactive-locale/main.po new file mode 100644 index 0000000000..fafd88a93a --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/inactive-locale/main.po @@ -0,0 +1,5 @@ +msgid "Source String" +msgstr "Inactive String" + +msgid "Common String" +msgstr "Inactive Common String" diff --git a/pontoon/sync/tests/fake-checkout/inactive-locale/other.lang b/pontoon/sync/tests/fake-checkout/inactive-locale/other.lang deleted file mode 100644 index 47c86a88f3..0000000000 --- a/pontoon/sync/tests/fake-checkout/inactive-locale/other.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Other Source String -Inactive Other Source String - -;Common String -Inactive Common String diff --git a/pontoon/sync/tests/fake-checkout/inactive-locale/other.po b/pontoon/sync/tests/fake-checkout/inactive-locale/other.po new file mode 100644 index 0000000000..55fb396d0a --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/inactive-locale/other.po @@ -0,0 +1,6 @@ +msgid "Other Source String" +msgstr "Inactive Other Source String" + +msgid "Common String" +msgstr "Inactive Common String" + diff --git a/pontoon/sync/tests/fake-checkout/templates/main.lang b/pontoon/sync/tests/fake-checkout/templates/main.lang deleted file mode 100644 index 0137d45d42..0000000000 --- a/pontoon/sync/tests/fake-checkout/templates/main.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Source String -Source String - -;Common String -Common String diff --git a/pontoon/sync/tests/fake-checkout/templates/main.pot b/pontoon/sync/tests/fake-checkout/templates/main.pot new file mode 100644 index 0000000000..c9791a9d33 --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/templates/main.pot @@ -0,0 +1,6 @@ +msgid "Source String" +msgstr "Source String" + +msgid "Common String" +msgstr "Common String" + diff --git a/pontoon/sync/tests/fake-checkout/templates/missing.lang b/pontoon/sync/tests/fake-checkout/templates/missing.lang deleted file mode 100644 index cfe59bcdca..0000000000 --- a/pontoon/sync/tests/fake-checkout/templates/missing.lang +++ /dev/null @@ -1,2 +0,0 @@ -;This file doesn't exist in the translated locale. -This file doesn't exist in the translated locale. diff --git a/pontoon/sync/tests/fake-checkout/templates/missing.pot b/pontoon/sync/tests/fake-checkout/templates/missing.pot new file mode 100644 index 0000000000..eba66666f8 --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/templates/missing.pot @@ -0,0 +1,2 @@ +msgid "This file does not exist in the translated locale." +msgstr "This file does not exist in the translated locale." diff --git a/pontoon/sync/tests/fake-checkout/templates/other.lang b/pontoon/sync/tests/fake-checkout/templates/other.lang deleted file mode 100644 index 065a5669bb..0000000000 --- a/pontoon/sync/tests/fake-checkout/templates/other.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Other Source String -Other Source String - -;Common String -Common String diff --git a/pontoon/sync/tests/fake-checkout/templates/other.pot b/pontoon/sync/tests/fake-checkout/templates/other.pot new file mode 100644 index 0000000000..776ab27f15 --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/templates/other.pot @@ -0,0 +1,5 @@ +msgid "Other Source String" +msgstr "Other Source String" + +msgid "Common String" +msgstr "Common String" diff --git a/pontoon/sync/tests/fake-checkout/translated-locale/main.lang b/pontoon/sync/tests/fake-checkout/translated-locale/main.lang deleted file mode 100644 index b41ceb4d8f..0000000000 --- a/pontoon/sync/tests/fake-checkout/translated-locale/main.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Source String -Translated String - -;Common String -Translated Common String diff --git a/pontoon/sync/tests/fake-checkout/translated-locale/main.po b/pontoon/sync/tests/fake-checkout/translated-locale/main.po new file mode 100644 index 0000000000..4e3ed79c64 --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/translated-locale/main.po @@ -0,0 +1,5 @@ +msgid "Source String" +msgstr "Translated String" + +msgid "Common String" +msgstr "Translated Common String" diff --git a/pontoon/sync/tests/fake-checkout/translated-locale/other.lang b/pontoon/sync/tests/fake-checkout/translated-locale/other.lang deleted file mode 100644 index 463b89a6b7..0000000000 --- a/pontoon/sync/tests/fake-checkout/translated-locale/other.lang +++ /dev/null @@ -1,5 +0,0 @@ -;Other Source String -Translated Other Source String - -;Common String -Translated Common String diff --git a/pontoon/sync/tests/fake-checkout/translated-locale/other.po b/pontoon/sync/tests/fake-checkout/translated-locale/other.po new file mode 100644 index 0000000000..87718f59b4 --- /dev/null +++ b/pontoon/sync/tests/fake-checkout/translated-locale/other.po @@ -0,0 +1,5 @@ +msgid "Other Source String" +msgstr "Translated Other Source String" + +msgid "Common String" +msgstr "Translated Common String" diff --git a/pontoon/sync/tests/formats/bom.lang b/pontoon/sync/tests/formats/bom.lang deleted file mode 100644 index da037d62f6..0000000000 --- a/pontoon/sync/tests/formats/bom.lang +++ /dev/null @@ -1,4 +0,0 @@ -## This is a langfile with a UTF-8 encoded BOM at the start of the file. - -;Source String -Translated String diff --git a/pontoon/sync/tests/formats/test_lang.py b/pontoon/sync/tests/formats/test_lang.py deleted file mode 100644 index e4c2814f4e..0000000000 --- a/pontoon/sync/tests/formats/test_lang.py +++ /dev/null @@ -1,173 +0,0 @@ -import os.path -from textwrap import dedent - -import pytest - -from pontoon.base.tests import assert_attributes_equal, TestCase -from pontoon.sync.exceptions import ParseError -from pontoon.sync.formats import lang -from pontoon.sync.tests.formats import FormatTestsMixin - - -BASE_LANG_FILE = """ -# Sample comment -;Source String -Translated String - -# First comment -# Second comment -;Multiple Comments -Translated Multiple Comments - -;No Comments or Sources -Translated No Comments or Sources -""" - - -class LangTests(FormatTestsMixin, TestCase): - parse = staticmethod(lang.parse) - supports_source = False - supports_keys = False - supports_source_string = True - - def test_parse_basic(self): - self.run_parse_basic(BASE_LANG_FILE, 0) - - def test_parse_multiple_comments(self): - self.run_parse_multiple_comments(BASE_LANG_FILE, 1) - - def test_parse_no_comments_no_sources(self): - self.run_parse_no_comments_no_sources(BASE_LANG_FILE, 2) - - def test_save_basic(self): - input_string = dedent( - """ - # Comment - ;SourceString - Source String - """ - ) - expected_string = dedent( - """ - # Comment - ;SourceString - New Translated String - """ - ) - - self.run_save_basic(input_string, expected_string) - - def test_save_remove(self): - """ - Deleting strings shouled replace the translation with the source - string. - """ - input_string = dedent( - """ - # Comment - ;SourceString - Translated String - """ - ) - expected_string = dedent( - """ - # Comment - ;SourceString - SourceString - """ - ) - - self.run_save_remove(input_string, expected_string) - - def test_load_utf8_bom(self): - """ - Ensure that the langfile parser can load UTF-8 files with an encoded - BOM at the beginning of the file. - - See https://docs.python.org/2/library/codecs.html for details (search - for the string "utf-8-sig"). - """ - current_dir = os.path.dirname(__file__) - resource = lang.parse(os.path.join(current_dir, "bom.lang")) - assert len(resource.translations) == 1 - assert_attributes_equal( - resource.translations[0], - source_string="Source String", - strings={None: "Translated String"}, - ) - - def test_parse_comments_arbitrary_hash(self): - """Comments can have an arbitrary amount of #s at the start.""" - path, resource = self.parse_string( - dedent( - """ - # 1 hash - ## 2 hash - ### 3 hash - - # 1 hash - ## 2 hash - ### 3 hash - ; Source - Translated - """ - ) - ) - - assert resource.children[1].content == "1 hash" - assert resource.children[2].content == "2 hash" - assert resource.children[3].content == "3 hash" - assert resource.translations[0].comments == ["1 hash", "2 hash", "3 hash"] - - def test_parse_eof(self): - """Langfiles do not need to end in a newline.""" - path, resource = self.parse_string(";Source\nTranslation") - assert resource.translations[0].source_string == "Source" - assert resource.translations[0].strings == {None: "Translation"} - - def test_preserve_comment_hashes(self): - """ - If a langfile is parsed and then saved without being modified, - it should not modify the contents of the file. - """ - expected = dedent( - """ - ## Example langfile with a few constructs that might break - - # Entity comment - ;Source - Translated - - # Single hash standalone comment - - ### Entity comment with multiple hashes - ;Identical - Identical {ok} - - ###Free standing comment at the end. WOW - """ - ) - path, resource = self.parse_string(expected) - resource.save(self.locale) - self.assert_file_content(path, expected) - - def test_parse_empty_translation(self): - """ - If an entity has an empty translation, parse should raise a - ParseError. - """ - with pytest.raises(ParseError): - self.parse_string( - dedent( - """ - # Comment - ;Source - Translated - - ;Empty - - ;Not Empty - Nope - """ - ) - ) diff --git a/pontoon/sync/tests/test_changeset.py b/pontoon/sync/tests/test_changeset.py index e8ab42e9f1..4ebf9097e7 100644 --- a/pontoon/sync/tests/test_changeset.py +++ b/pontoon/sync/tests/test_changeset.py @@ -198,7 +198,6 @@ def test_update_db_existing_translation(self): """ # Set up DB and VCS to differ and require an update. self.main_db_translation.fuzzy = True - self.main_db_translation.extra = {} self.main_db_translation.save() self.main_vcs_entity.key = "Source String" @@ -207,9 +206,6 @@ def test_update_db_existing_translation(self): self.main_vcs_entity.string_plural = "plural string" self.main_vcs_entity.source = ["foo.py:87"] self.main_vcs_translation.fuzzy = False - # The test translation is from a langfile so we can use tags - # for testing extra. - self.main_vcs_translation.tags = {"ok"} self.update_main_db_entity() self.main_db_entity.refresh_from_db() @@ -223,9 +219,7 @@ def test_update_db_existing_translation(self): ) self.main_db_translation.refresh_from_db() - assert_attributes_equal( - self.main_db_translation, fuzzy=False, extra={"tags": ["ok"]} - ) + assert_attributes_equal(self.main_db_translation, fuzzy=False) def test_update_db_clean_entity_translation(self): """ @@ -294,7 +288,6 @@ def test_update_db_new_translation(self): approved=True, approved_date=aware_datetime(1970, 1, 1), fuzzy=False, - extra={"tags": []}, ) assert ActionLog.objects.filter( diff --git a/pontoon/sync/tests/test_checks.py b/pontoon/sync/tests/test_checks.py index a4d337c7c9..538cce2be8 100644 --- a/pontoon/sync/tests/test_checks.py +++ b/pontoon/sync/tests/test_checks.py @@ -1,7 +1,7 @@ from unittest.mock import patch, PropertyMock from pontoon.base.utils import aware_datetime -from pontoon.base.tests import TranslationFactory +from pontoon.base.tests import EntityFactory, ResourceFactory, TranslationFactory from pontoon.checks.models import ( Warning, Error, @@ -59,14 +59,26 @@ def test_bulk_check_invalid_translations(self): * check if errors are detected * check if only valid translation will land in the Translate Memory """ + db_resource = ResourceFactory.create( + project=self.db_project, + path="resource.ftl", + format="ftl", + ) + db_entity = EntityFactory.create( + resource=db_resource, + string="key = Source String", + key="key", + obsolete=False, + ) invalid_translation, valid_translation = TranslationFactory.create_batch( 2, locale=self.translated_locale, - entity=self.main_db_entity, + entity=db_entity, + string="key = Translated String", approved=True, date=aware_datetime(2015, 1, 1), ) - invalid_translation.string = "a\nb" + invalid_translation.string = "Translated String" invalid_translation.save() # Clear TM entries for those translations @@ -85,7 +97,7 @@ def test_bulk_check_invalid_translations(self): (error,) = Error.objects.all() assert error.library == FailedCheck.Library.PONTOON - assert error.message == "Newline characters are not allowed" + assert error.message == 'Expected token: "="' assert error.translation == invalid_translation self.changeset.translations_to_update = { diff --git a/pontoon/sync/tests/test_core.py b/pontoon/sync/tests/test_core.py index 08052f9c7f..180a717868 100644 --- a/pontoon/sync/tests/test_core.py +++ b/pontoon/sync/tests/test_core.py @@ -145,9 +145,9 @@ def test_order(self): # Check if Resource.order gets reset for all Project resources. self.other_db_resource.delete() - assert self.main_db_resource.order == 0 # path="main.lang" - assert self.missing_db_resource.order == 0 # path="missing.lang" - assert self.other_db_resource.order == 0 # path="other.lang" + assert self.main_db_resource.order == 0 # path="main.po" + assert self.missing_db_resource.order == 0 # path="missing.po" + assert self.other_db_resource.order == 0 # path="other.po" update_resources(self.db_project, self.vcs_project) self.missing_db_resource.refresh_from_db() diff --git a/pontoon/sync/vcs/models.py b/pontoon/sync/vcs/models.py index 7bfac01ade..942e34782b 100644 --- a/pontoon/sync/vcs/models.py +++ b/pontoon/sync/vcs/models.py @@ -918,15 +918,6 @@ def __init__( self.last_translator = last_translator self.last_updated = last_updated - @property - def extra(self): - """ - Return a dict of custom properties to store in the database. - Useful for subclasses from specific formats that have extra data - that needs to be preserved. - """ - return {} - def update_from_db(self, db_translations): """ Update translation with current DB state. diff --git a/requirements/default.in b/requirements/default.in index 80e3f954a0..07f948f2d8 100644 --- a/requirements/default.in +++ b/requirements/default.in @@ -40,7 +40,6 @@ markupsafe==2.0.1 mercurial==6.4.2 newrelic==9.6.0 openai==1.12.0 -parsimonious==0.10.0 polib==1.0.6 psycopg2==2.9.6 PyJWT==2.4.0 diff --git a/requirements/default.txt b/requirements/default.txt index 70efb0f760..bf7874272b 100644 --- a/requirements/default.txt +++ b/requirements/default.txt @@ -613,10 +613,6 @@ packaging==21.3 \ --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 # via bleach -parsimonious==0.10.0 \ - --hash=sha256:8281600da180ec8ae35427a4ab4f7b82bfec1e3d1e52f80cb60ea82b9512501c \ - --hash=sha256:982ab435fabe86519b57f6b35610aa4e4e977e9f02a14353edf4bbc75369fc0f - # via -r requirements/default.in polib==1.0.6 \ --hash=sha256:20d2a0d589a692c11df549bd7cda83c665eef2a83e017b843fecdf956edbad74 \ --hash=sha256:b1ea141d58ed5e48aed2674f7c894dfb83f639c3286d7b32b2e19fa032a5b400 @@ -910,7 +906,6 @@ regex==2023.5.5 \ --hash=sha256:fb2b495dd94b02de8215625948132cc2ea360ae84fe6634cd19b6567709c8ae2 \ --hash=sha256:fee0016cc35a8a91e8cc9312ab26a6fe638d484131a7afa79e1ce6165328a135 # via - # parsimonious # sacrebleu # sacremoses requests==2.26.0 \ diff --git a/translate/src/modules/editor/components/TranslationLength.css b/translate/src/modules/editor/components/TranslationLength.css index 39602dd709..2bf353b543 100644 --- a/translate/src/modules/editor/components/TranslationLength.css +++ b/translate/src/modules/editor/components/TranslationLength.css @@ -4,7 +4,3 @@ line-height: 22px; padding: 9px 5px; } - -.translation-length .countdown .overflow { - color: var(--status-error); -} diff --git a/translate/src/modules/editor/components/TranslationLength.test.js b/translate/src/modules/editor/components/TranslationLength.test.js index 709e453f38..b0f809de83 100644 --- a/translate/src/modules/editor/components/TranslationLength.test.js +++ b/translate/src/modules/editor/components/TranslationLength.test.js @@ -30,7 +30,6 @@ describe('', () => { it('shows translation length and original string length', () => { const wrapper = mountTranslationLength('', '12345', '1234567', ''); - expect(wrapper.find('.countdown')).toHaveLength(0); const div = wrapper.find('.translation-vs-original'); expect(div.childAt(0).text()).toEqual('7'); expect(div.childAt(1).text()).toEqual('|'); @@ -57,42 +56,7 @@ describe('', () => { expect(div.childAt(2).text()).toEqual('6'); }); - it('shows countdown if MAX_LENGTH provided in LANG entity comment', () => { - const wrapper = mountTranslationLength( - 'lang', - '12345', - '123', - 'MAX_LENGTH: 5\nThis is an actual comment.', - ); - - expect(wrapper.find('.translation-vs-original')).toHaveLength(0); - expect(wrapper.find('.countdown span').text()).toEqual('2'); - expect(wrapper.find('.countdown span.overflow')).toHaveLength(0); - }); - - it('marks countdown overflow', () => { - const wrapper = mountTranslationLength( - 'lang', - '12345', - '123456', - 'MAX_LENGTH: 5\nThis is an actual comment.', - ); - - expect(wrapper.find('.countdown span.overflow')).toHaveLength(1); - }); - - it('strips html from translation when calculating countdown', () => { - const wrapper = mountTranslationLength( - 'lang', - '12345', - '123456', - 'MAX_LENGTH: 5\nThis is an actual comment.', - ); - - expect(wrapper.find('.countdown span').text()).toEqual('-1'); - }); - - it('does not strips html from translation when calculating length', () => { + it('does not strip html from translation when calculating length', () => { const wrapper = mountTranslationLength( '', '12345', diff --git a/translate/src/modules/editor/components/TranslationLength.tsx b/translate/src/modules/editor/components/TranslationLength.tsx index 0efe96ee59..495d3f93ca 100644 --- a/translate/src/modules/editor/components/TranslationLength.tsx +++ b/translate/src/modules/editor/components/TranslationLength.tsx @@ -5,13 +5,7 @@ import { getPlainMessage } from '~/utils/message'; import './TranslationLength.css'; -/** - * Shows translation length vs. original string length, or countdown. - * - * Countdown is currently only supported for LANG strings, which use special - * syntax in the comment to define maximum translation length. MAX_LENGTH - * is provided for strings without HTML tags, so they need to be stripped. - */ +/** Shows translation length vs. original string length. */ export function TranslationLength(): React.ReactElement<'div'> | null { const { entity } = useContext(EntityView); const source = useEntitySource(); @@ -23,26 +17,6 @@ export function TranslationLength(): React.ReactElement<'div'> | null { } const text = edit[0].value; - - const maxLength = - entity.format === 'lang' && entity.comment.match(/^MAX_LENGTH: (\d+)/); - if (maxLength) { - const limit = parseInt(maxLength[1]); - - // Source: https://stackoverflow.com/a/47140708 - const doc = new DOMParser().parseFromString(text, 'text/html'); - const length = doc.body?.textContent?.length ?? 0; - return ( -
-
- limit ? 'overflow' : undefined}> - {limit - length} - -
-
- ); - } - const srcText = getPlainMessage(source, entity.format); return (