From 16d7c82fb270022c722bde8afbc3dbefbfd0eba4 Mon Sep 17 00:00:00 2001 From: Andrew <11490628+andrew000@users.noreply.github.com> Date: Thu, 25 Jul 2024 16:07:17 +0300 Subject: [PATCH] Fix issue-1 (#2) Now all `ast.Entry` like objects stores in `FluentKey` dataclass and get position attribute, to locate them in `.ftl` file --- .gitignore | 1 - Makefile | 2 +- src/ftl_extract/cli.py | 10 ++++ src/ftl_extract/code_extractor.py | 7 +-- src/ftl_extract/exceptions.py | 4 +- src/ftl_extract/ftl_extractor.py | 6 +++ src/ftl_extract/ftl_importer.py | 31 +++++++----- src/ftl_extract/matcher.py | 22 +++++++-- src/ftl_extract/process/kwargs_extractor.py | 2 +- src/ftl_extract/process/serializer.py | 23 ++++----- tests/test_extract/test_cli_extractor.py | 26 ++++++++++ tests/test_extract/test_ftl_comment.py | 2 +- tests/test_ftl_import.py | 14 +++--- tests/test_serializer.py | 55 ++++++++++++++++++--- 14 files changed, 154 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index 9d793f7..4212899 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,6 @@ /.venv/ /tokei.exe __pycache__/ -*.ftl /.coverage /dist/ /docs/ diff --git a/Makefile b/Makefile index c15e3f7..a3f3416 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ lint: format: @echo "Running ruff check with --fix..." - @poetry run ruff check --config pyproject.toml --fix $(code_dir) $(tests_dir) + @poetry run ruff check --config pyproject.toml --fix --unsafe-fixes $(code_dir) $(tests_dir) @echo "Running ruff..." @poetry run ruff format --config pyproject.toml $(code_dir) $(tests_dir) diff --git a/src/ftl_extract/cli.py b/src/ftl_extract/cli.py index 6a84457..9ec94df 100644 --- a/src/ftl_extract/cli.py +++ b/src/ftl_extract/cli.py @@ -33,12 +33,21 @@ show_default=True, help="Beautify output FTL files.", ) +@click.option( + "--comment-junks", + is_flag=True, + default=False, + show_default=True, + help="Comments Junk elements.", +) +@click.version_option() def cli_extract( code_path: Path, output_path: Path, language: tuple[str, ...], i18n_keys: tuple[str, ...], beauty: bool = False, + comment_junks: bool = False, ) -> None: click.echo(f"Extracting from {code_path}...") @@ -48,4 +57,5 @@ def cli_extract( language=language, i18n_keys=i18n_keys, beauty=beauty, + comment_junks=comment_junks, ) diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index 73bf108..305abd0 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -1,9 +1,10 @@ from __future__ import annotations from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast import libcst as cst +from fluent.syntax import ast from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, @@ -89,8 +90,8 @@ def find_conflicts( if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation): raise FTLExtractorDifferentTranslationError( key, - current_fluent_keys[key].translation, - new_fluent_keys[key].translation, + cast(ast.Message, current_fluent_keys[key].translation), + cast(ast.Message, new_fluent_keys[key].translation), ) diff --git a/src/ftl_extract/exceptions.py b/src/ftl_extract/exceptions.py index 80cebd7..189d20c 100644 --- a/src/ftl_extract/exceptions.py +++ b/src/ftl_extract/exceptions.py @@ -26,8 +26,8 @@ class FTLExtractorDifferentTranslationError(FTLExtractorError): def __init__( self, key: str, - current_translation: ast.Message | ast.Comment, - new_translation: ast.Message | ast.Comment, + current_translation: ast.Message, + new_translation: ast.Message, ) -> None: self.current_translation = current_translation self.new_translation = new_translation diff --git a/src/ftl_extract/ftl_extractor.py b/src/ftl_extract/ftl_extractor.py index db10765..e84930f 100644 --- a/src/ftl_extract/ftl_extractor.py +++ b/src/ftl_extract/ftl_extractor.py @@ -24,6 +24,7 @@ def extract( language: tuple[str, ...], i18n_keys: tuple[str, ...], beauty: bool = False, + comment_junks: bool = False, ) -> None: serializer: FluentSerializer | BeautyFluentSerializer @@ -77,6 +78,11 @@ def extract( for fluent_key in keys_to_comment.values(): comment_ftl_key(fluent_key, serializer) + # Comment Junk elements if needed + if comment_junks is True: + for fluent_key in leave_as_is: + comment_ftl_key(fluent_key, serializer) + sorted_fluent_keys = sort_fluent_keys_by_path(stored_fluent_keys) for path, keys in sort_fluent_keys_by_path(keys_to_add).items(): diff --git a/src/ftl_extract/ftl_importer.py b/src/ftl_extract/ftl_importer.py index 61f491d..056ec90 100644 --- a/src/ftl_extract/ftl_importer.py +++ b/src/ftl_extract/ftl_importer.py @@ -1,26 +1,26 @@ from __future__ import annotations from pathlib import Path +from typing import TYPE_CHECKING from fluent.syntax import ast, parse from ftl_extract.matcher import FluentKey +if TYPE_CHECKING: + from fluent.syntax.ast import Resource + def import_from_ftl( path: Path, locale: str -) -> tuple[ - dict[str, FluentKey], - ast.Resource, - list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk], -]: +) -> tuple[dict[str, FluentKey], Resource, list[FluentKey]]: """Import `FluentKey`s from FTL.""" ftl_keys = {} leave_as_is = [] resource = parse(path.read_text(encoding="utf-8"), with_spans=True) - for entry in resource.body: + for position, entry in enumerate(resource.body, start=0): if isinstance(entry, ast.Message): ftl_keys[entry.id.name] = FluentKey( code_path=Path(), @@ -28,19 +28,24 @@ def import_from_ftl( translation=entry, path=path, locale=locale, + position=position, ) else: - leave_as_is.append(entry) + leave_as_is.append( + FluentKey( + code_path=Path(), + key="", + translation=entry, + path=path, + locale=locale, + position=position, + ) + ) return ftl_keys, resource, leave_as_is -def import_ftl_from_dir( - path: Path, locale: str -) -> tuple[ - dict[str, FluentKey], - list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk], -]: +def import_ftl_from_dir(path: Path, locale: str) -> tuple[dict[str, FluentKey], list[FluentKey]]: """Import `FluentKey`s from directory of FTL files.""" ftl_files = (path / locale).rglob("*.ftl") if path.is_dir() else [path] ftl_keys = {} diff --git a/src/ftl_extract/matcher.py b/src/ftl_extract/matcher.py index 5ed4294..58e8528 100644 --- a/src/ftl_extract/matcher.py +++ b/src/ftl_extract/matcher.py @@ -2,6 +2,7 @@ from collections.abc import Sequence from dataclasses import dataclass, field +from math import inf from pathlib import Path from typing import TYPE_CHECKING, Callable, cast @@ -41,9 +42,12 @@ class FluentKey: code_path: Path key: str - translation: ast.Message | ast.Comment + translation: ( + ast.Message | ast.Comment | ast.Term | ast.GroupComment | ast.ResourceComment | ast.Junk + ) path: Path = field(default=Path("_default.ftl")) locale: str | None = field(default=None) + position: int | float = field(default=inf) class I18nMatcher: @@ -127,14 +131,22 @@ def extract_matches(self, module: cst.Module) -> None: elements=[ast.TextElement(value=cast(cst.Name, match["key"]).value)] ), ) - fluent_key = FluentKey(code_path=self.code_path, key=key, translation=translation) + fluent_key = FluentKey( + code_path=self.code_path, + key=key, + translation=translation, + ) elif isinstance(match["key"], cst.SimpleString): key = cast(cst.SimpleString, match["key"]).raw_value translation = ast.Message( id=ast.Identifier(name=key), value=ast.Pattern(elements=[ast.TextElement(value=key)]), ) - fluent_key = FluentKey(code_path=self.code_path, key=key, translation=translation) + fluent_key = FluentKey( + code_path=self.code_path, + key=key, + translation=translation, + ) else: msg = f"Unknown type of key: {type(match['key'])} | {match['key']}" raise TypeError(msg) @@ -169,8 +181,8 @@ def extract_matches(self, module: cst.Module) -> None: if not self.fluent_keys[fluent_key.key].translation.equals(fluent_key.translation): raise FTLExtractorDifferentTranslationError( fluent_key.key, - fluent_key.translation, - self.fluent_keys[fluent_key.key].translation, + cast(ast.Message, fluent_key.translation), + cast(ast.Message, self.fluent_keys[fluent_key.key].translation), ) else: diff --git a/src/ftl_extract/process/kwargs_extractor.py b/src/ftl_extract/process/kwargs_extractor.py index 6bbcf6f..de14c40 100644 --- a/src/ftl_extract/process/kwargs_extractor.py +++ b/src/ftl_extract/process/kwargs_extractor.py @@ -6,7 +6,7 @@ def extract_kwargs(key: FluentKey) -> set[str]: kwargs: set[str] = set() - if isinstance(key.translation, ast.Comment): + if not isinstance(key.translation, ast.Message): return kwargs if not key.translation.value: diff --git a/src/ftl_extract/process/serializer.py b/src/ftl_extract/process/serializer.py index a0fd983..1866a0e 100644 --- a/src/ftl_extract/process/serializer.py +++ b/src/ftl_extract/process/serializer.py @@ -1,11 +1,14 @@ from __future__ import annotations +import operator from typing import TYPE_CHECKING from fluent.syntax import FluentSerializer, ast from fluent.syntax.serializer import serialize_junk, serialize_message, serialize_term if TYPE_CHECKING: + from collections.abc import Iterable + from fluent.syntax.ast import Resource from ftl_extract.matcher import FluentKey @@ -50,22 +53,20 @@ def serialize_comment( def generate_ftl( - fluent_keys: dict[str, FluentKey] | list[FluentKey], + fluent_keys: Iterable[FluentKey], serializer: FluentSerializer, - leave_as_is: list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk] - | None = None, + leave_as_is: Iterable[FluentKey], ) -> tuple[str, Resource]: """Generate FTL translations from `fluent_keys`.""" resource = ast.Resource(body=None) - if leave_as_is is not None: - resource.body.extend(leave_as_is) + listed_fluent_keys = list(fluent_keys) + listed_fluent_keys.extend(leave_as_is) + + # Sort fluent keys by position + listed_fluent_keys.sort(key=operator.attrgetter("position")) - if isinstance(fluent_keys, list): - for fluent_key in fluent_keys: - resource.body.append(fluent_key.translation) - else: - for fluent_key in fluent_keys.values(): - resource.body.append(fluent_key.translation) + for fluent_key in listed_fluent_keys: + resource.body.append(fluent_key.translation) return serializer.serialize(resource), resource diff --git a/tests/test_extract/test_cli_extractor.py b/tests/test_extract/test_cli_extractor.py index f7ec315..c71a1ed 100644 --- a/tests/test_extract/test_cli_extractor.py +++ b/tests/test_extract/test_cli_extractor.py @@ -54,6 +54,17 @@ def mock_extract_function() -> patch: yield mock +@pytest.fixture() +def mock_leave_as_is() -> list: + return [ + MagicMock(spec=ast.Comment), + MagicMock(spec=ast.GroupComment), + MagicMock(spec=ast.ResourceComment), + MagicMock(spec=ast.Term), + MagicMock(spec=ast.Junk), + ] + + def test_extract_with_beauty_enabled( setup_environment: tuple[Path, Path], mock_fluent_key: FluentKey, @@ -200,3 +211,18 @@ def test_extraction_with_invalid_i18n_keys_ignores_them( ) assert result.exit_code == 0 assert mock_extract_function.call_args[1]["i18n_keys"] == ("nonexistent_key",) + + +def test_extract_comments_junk_elements_if_needed( + setup_environment: tuple[Path, Path], + mock_leave_as_is: list, +) -> None: + code_path, output_path = setup_environment + + with ( + patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value={}), + patch("ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({}, mock_leave_as_is)), + patch("ftl_extract.ftl_extractor.comment_ftl_key") as mock_comment_ftl_key, + ): + extract(code_path, output_path, ("en",), ("i18n",), comment_junks=True) + assert mock_comment_ftl_key.call_count == len(mock_leave_as_is) diff --git a/tests/test_extract/test_ftl_comment.py b/tests/test_extract/test_ftl_comment.py index a1bccb9..25d38bc 100644 --- a/tests/test_extract/test_ftl_comment.py +++ b/tests/test_extract/test_ftl_comment.py @@ -44,7 +44,7 @@ def test_ftl_comment(tmp_path: Path) -> None: comment_ftl_key(ftl_keys["key-4"], serializer=serializer) comment_ftl_key(ftl_keys["key-5"], serializer=serializer) - ftl, _ = generate_ftl(ftl_keys, serializer=serializer) + ftl, _ = generate_ftl(ftl_keys.values(), serializer=serializer, leave_as_is=leave_as_is) (tmp_path / "test.ftl").write_text(ftl, encoding="utf-8") ftl = (tmp_path / "test.ftl").read_text(encoding="utf-8") diff --git a/tests/test_ftl_import.py b/tests/test_ftl_import.py index a48b511..6b30e57 100644 --- a/tests/test_ftl_import.py +++ b/tests/test_ftl_import.py @@ -16,6 +16,7 @@ def mock_ftl_content() -> str: -term = Term value hello = Hello, world! welcome = Welcome, { $name }! +Junk """ @@ -24,7 +25,7 @@ def test_import_from_ftl_with_valid_ftl_file(mock_ftl_content: str) -> None: keys, resource, leave_as_is = import_from_ftl(Path("/path/to/locale/en/example.ftl"), "en") assert "hello" in keys assert "welcome" in keys - assert len(resource.body) == 6 # noqa: PLR2004 + assert len(resource.body) == 7 # noqa: PLR2004 def test_import_from_ftl_with_empty_ftl_file() -> None: @@ -59,8 +60,9 @@ def test_import_ftl_from_dir_with_nonexistent_directory() -> None: def test_import_from_ftl_appends_non_message_entries_correctly(mock_ftl_content: str) -> None: with patch("pathlib.Path.read_text", return_value=mock_ftl_content): _, _, leave_as_is = import_from_ftl(Path("/path/to/locale/en/various_entries.ftl"), "en") - assert len(leave_as_is) == 4 # noqa: PLR2004 - assert isinstance(leave_as_is[0], ast.Comment) - assert isinstance(leave_as_is[1], ast.GroupComment) - assert isinstance(leave_as_is[2], ast.ResourceComment) - assert isinstance(leave_as_is[3], ast.Term) + assert len(leave_as_is) == 5 # noqa: PLR2004 + assert isinstance(leave_as_is[0].translation, ast.Comment) + assert isinstance(leave_as_is[1].translation, ast.GroupComment) + assert isinstance(leave_as_is[2].translation, ast.ResourceComment) + assert isinstance(leave_as_is[3].translation, ast.Term) + assert isinstance(leave_as_is[4].translation, ast.Junk) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 5f36e26..59290d0 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -51,7 +51,11 @@ def empty_fluent_keys() -> list[FluentKey]: def test_custom_serializer_produces_correct_ftl_for_single_key( single_fluent_key: list[FluentKey], ) -> None: - ftl_string, resource = generate_ftl(single_fluent_key, serializer=BeautyFluentSerializer()) + ftl_string, resource = generate_ftl( + single_fluent_key, + serializer=BeautyFluentSerializer(), + leave_as_is=[], + ) assert "greeting = Hello, world!" in ftl_string assert len(resource.body) == 1 @@ -59,7 +63,11 @@ def test_custom_serializer_produces_correct_ftl_for_single_key( def test_custom_serializer_produces_correct_ftl_for_multiple_keys( multiple_fluent_keys: list[FluentKey], ) -> None: - ftl_string, resource = generate_ftl(multiple_fluent_keys, serializer=BeautyFluentSerializer()) + ftl_string, resource = generate_ftl( + multiple_fluent_keys, + serializer=BeautyFluentSerializer(), + leave_as_is=[], + ) assert "greeting = Hello, world!" in ftl_string assert "farewell = Goodbye, world!" in ftl_string assert len(resource.body) == 2 # noqa: PLR2004 @@ -68,7 +76,11 @@ def test_custom_serializer_produces_correct_ftl_for_multiple_keys( def test_custom_serializer_handles_empty_fluent_keys_list_properly( empty_fluent_keys: list[FluentKey], ) -> None: - ftl_string, resource = generate_ftl(empty_fluent_keys, serializer=BeautyFluentSerializer()) + ftl_string, resource = generate_ftl( + empty_fluent_keys, + serializer=BeautyFluentSerializer(), + leave_as_is=[], + ) assert ftl_string == "" assert resource.body is None or len(resource.body) == 0 @@ -85,14 +97,43 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: ), ) ], - serializer=BeautyFluentSerializer(), + serializer=BeautyFluentSerializer(with_junk=True), leave_as_is=[ - ast.Comment(content="This is a comment"), - ast.GroupComment(content="This is a group comment"), - ast.ResourceComment(content="This is a resource comment"), + FluentKey( + code_path=Path(), + key="", + translation=ast.Comment(content="This is a comment"), + ), + FluentKey( + code_path=Path(), + key="", + translation=ast.GroupComment(content="This is a group comment"), + ), + FluentKey( + code_path=Path(), + key="", + translation=ast.ResourceComment(content="This is a resource comment"), + ), + FluentKey( + code_path=Path(), + key="", + translation=ast.Term( + id=ast.Identifier("test_term"), + value=ast.Pattern(elements=[ast.TextElement("Test term content")]), + ), + ), + FluentKey( + code_path=Path(), + key="", + translation=ast.Junk( + content="This is junk content", + ), + ), ], ) assert "This is a comment" in ftl_string assert "This is a group comment" in ftl_string assert "This is a resource comment" in ftl_string + assert "test_term = Test term content" in ftl_string + assert "This is junk content" in ftl_string assert "Test message content" in ftl_string