Fix issue-1 (#2)

Now all `ast.Entry` like objects stores in `FluentKey` dataclass and get position attribute, to locate them in `.ftl` file

.gitignore

@@ -6,7 +6,6 @@
 /.venv/
 /tokei.exe
 __pycache__/
-*.ftl
 /.coverage
 /dist/
 /docs/

Makefile

@@ -19,7 +19,7 @@ lint:
 
 format:
 	@echo "Running ruff check with --fix..."
-	@poetry run ruff check --config pyproject.toml --fix $(code_dir) $(tests_dir)
+	@poetry run ruff check --config pyproject.toml --fix --unsafe-fixes $(code_dir) $(tests_dir)
 
 	@echo "Running ruff..."
 	@poetry run ruff format --config pyproject.toml $(code_dir) $(tests_dir)

src/ftl_extract/cli.py

@@ -33,12 +33,21 @@
     show_default=True,
     help="Beautify output FTL files.",
 )
+@click.option(
+    "--comment-junks",
+    is_flag=True,
+    default=False,
+    show_default=True,
+    help="Comments Junk elements.",
+)
+@click.version_option()
 def cli_extract(
     code_path: Path,
     output_path: Path,
     language: tuple[str, ...],
     i18n_keys: tuple[str, ...],
     beauty: bool = False,
+    comment_junks: bool = False,
 ) -> None:
     click.echo(f"Extracting from {code_path}...")
 
@@ -48,4 +57,5 @@ def cli_extract(
         language=language,
         i18n_keys=i18n_keys,
         beauty=beauty,
+        comment_junks=comment_junks,
     )

src/ftl_extract/code_extractor.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 import libcst as cst
+from fluent.syntax import ast
 
 from ftl_extract.exceptions import (
     FTLExtractorDifferentPathsError,
@@ -89,8 +90,8 @@ def find_conflicts(
         if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation):
             raise FTLExtractorDifferentTranslationError(
                 key,
-                current_fluent_keys[key].translation,
-                new_fluent_keys[key].translation,
+                cast(ast.Message, current_fluent_keys[key].translation),
+                cast(ast.Message, new_fluent_keys[key].translation),
             )
 
 

src/ftl_extract/exceptions.py

@@ -26,8 +26,8 @@ class FTLExtractorDifferentTranslationError(FTLExtractorError):
     def __init__(
         self,
         key: str,
-        current_translation: ast.Message | ast.Comment,
-        new_translation: ast.Message | ast.Comment,
+        current_translation: ast.Message,
+        new_translation: ast.Message,
     ) -> None:
         self.current_translation = current_translation
         self.new_translation = new_translation

src/ftl_extract/ftl_extractor.py

@@ -24,6 +24,7 @@ def extract(
     language: tuple[str, ...],
     i18n_keys: tuple[str, ...],
     beauty: bool = False,
+    comment_junks: bool = False,
 ) -> None:
     serializer: FluentSerializer | BeautyFluentSerializer
 
@@ -77,6 +78,11 @@ def extract(
         for fluent_key in keys_to_comment.values():
             comment_ftl_key(fluent_key, serializer)
 
+        # Comment Junk elements if needed
+        if comment_junks is True:
+            for fluent_key in leave_as_is:
+                comment_ftl_key(fluent_key, serializer)
+
         sorted_fluent_keys = sort_fluent_keys_by_path(stored_fluent_keys)
 
         for path, keys in sort_fluent_keys_by_path(keys_to_add).items():

src/ftl_extract/ftl_importer.py

@@ -1,46 +1,51 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from fluent.syntax import ast, parse
 
 from ftl_extract.matcher import FluentKey
 
+if TYPE_CHECKING:
+    from fluent.syntax.ast import Resource
+
 
 def import_from_ftl(
     path: Path, locale: str
-) -> tuple[
-    dict[str, FluentKey],
-    ast.Resource,
-    list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk],
-]:
+) -> tuple[dict[str, FluentKey], Resource, list[FluentKey]]:
     """Import `FluentKey`s from FTL."""
     ftl_keys = {}
     leave_as_is = []
 
     resource = parse(path.read_text(encoding="utf-8"), with_spans=True)
 
-    for entry in resource.body:
+    for position, entry in enumerate(resource.body, start=0):
         if isinstance(entry, ast.Message):
             ftl_keys[entry.id.name] = FluentKey(
                 code_path=Path(),
                 key=entry.id.name,
                 translation=entry,
                 path=path,
                 locale=locale,
+                position=position,
             )
         else:
-            leave_as_is.append(entry)
+            leave_as_is.append(
+                FluentKey(
+                    code_path=Path(),
+                    key="",
+                    translation=entry,
+                    path=path,
+                    locale=locale,
+                    position=position,
+                )
+            )
 
     return ftl_keys, resource, leave_as_is
 
 
-def import_ftl_from_dir(
-    path: Path, locale: str
-) -> tuple[
-    dict[str, FluentKey],
-    list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk],
-]:
+def import_ftl_from_dir(path: Path, locale: str) -> tuple[dict[str, FluentKey], list[FluentKey]]:
     """Import `FluentKey`s from directory of FTL files."""
     ftl_files = (path / locale).rglob("*.ftl") if path.is_dir() else [path]
     ftl_keys = {}

src/ftl_extract/matcher.py

@@ -2,6 +2,7 @@
 
 from collections.abc import Sequence
 from dataclasses import dataclass, field
+from math import inf
 from pathlib import Path
 from typing import TYPE_CHECKING, Callable, cast
 
@@ -41,9 +42,12 @@ class FluentKey:
 
     code_path: Path
     key: str
-    translation: ast.Message | ast.Comment
+    translation: (
+        ast.Message | ast.Comment | ast.Term | ast.GroupComment | ast.ResourceComment | ast.Junk
+    )
     path: Path = field(default=Path("_default.ftl"))
     locale: str | None = field(default=None)
+    position: int | float = field(default=inf)
 
 
 class I18nMatcher:
@@ -127,14 +131,22 @@ def extract_matches(self, module: cst.Module) -> None:
                         elements=[ast.TextElement(value=cast(cst.Name, match["key"]).value)]
                     ),
                 )
-                fluent_key = FluentKey(code_path=self.code_path, key=key, translation=translation)
+                fluent_key = FluentKey(
+                    code_path=self.code_path,
+                    key=key,
+                    translation=translation,
+                )
             elif isinstance(match["key"], cst.SimpleString):
                 key = cast(cst.SimpleString, match["key"]).raw_value
                 translation = ast.Message(
                     id=ast.Identifier(name=key),
                     value=ast.Pattern(elements=[ast.TextElement(value=key)]),
                 )
-                fluent_key = FluentKey(code_path=self.code_path, key=key, translation=translation)
+                fluent_key = FluentKey(
+                    code_path=self.code_path,
+                    key=key,
+                    translation=translation,
+                )
             else:
                 msg = f"Unknown type of key: {type(match['key'])} | {match['key']}"
                 raise TypeError(msg)
@@ -169,8 +181,8 @@ def extract_matches(self, module: cst.Module) -> None:
                 if not self.fluent_keys[fluent_key.key].translation.equals(fluent_key.translation):
                     raise FTLExtractorDifferentTranslationError(
                         fluent_key.key,
-                        fluent_key.translation,
-                        self.fluent_keys[fluent_key.key].translation,
+                        cast(ast.Message, fluent_key.translation),
+                        cast(ast.Message, self.fluent_keys[fluent_key.key].translation),
                     )
 
             else:

src/ftl_extract/process/kwargs_extractor.py

@@ -6,7 +6,7 @@
 def extract_kwargs(key: FluentKey) -> set[str]:
     kwargs: set[str] = set()
 
-    if isinstance(key.translation, ast.Comment):
+    if not isinstance(key.translation, ast.Message):
         return kwargs
 
     if not key.translation.value:

src/ftl_extract/process/serializer.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
+import operator
 from typing import TYPE_CHECKING
 
 from fluent.syntax import FluentSerializer, ast
 from fluent.syntax.serializer import serialize_junk, serialize_message, serialize_term
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
+
     from fluent.syntax.ast import Resource
 
     from ftl_extract.matcher import FluentKey
@@ -50,22 +53,20 @@ def serialize_comment(
 
 
 def generate_ftl(
-    fluent_keys: dict[str, FluentKey] | list[FluentKey],
+    fluent_keys: Iterable[FluentKey],
     serializer: FluentSerializer,
-    leave_as_is: list[ast.Term | ast.Comment | ast.GroupComment | ast.ResourceComment | ast.Junk]
-    | None = None,
+    leave_as_is: Iterable[FluentKey],
 ) -> tuple[str, Resource]:
     """Generate FTL translations from `fluent_keys`."""
     resource = ast.Resource(body=None)
 
-    if leave_as_is is not None:
-        resource.body.extend(leave_as_is)
+    listed_fluent_keys = list(fluent_keys)
+    listed_fluent_keys.extend(leave_as_is)
+
+    # Sort fluent keys by position
+    listed_fluent_keys.sort(key=operator.attrgetter("position"))
 
-    if isinstance(fluent_keys, list):
-        for fluent_key in fluent_keys:
-            resource.body.append(fluent_key.translation)
-    else:
-        for fluent_key in fluent_keys.values():
-            resource.body.append(fluent_key.translation)
+    for fluent_key in listed_fluent_keys:
+        resource.body.append(fluent_key.translation)
 
     return serializer.serialize(resource), resource

tests/test_extract/test_cli_extractor.py

@@ -54,6 +54,17 @@ def mock_extract_function() -> patch:
         yield mock
 
 
+@pytest.fixture()
+def mock_leave_as_is() -> list:
+    return [
+        MagicMock(spec=ast.Comment),
+        MagicMock(spec=ast.GroupComment),
+        MagicMock(spec=ast.ResourceComment),
+        MagicMock(spec=ast.Term),
+        MagicMock(spec=ast.Junk),
+    ]
+
+
 def test_extract_with_beauty_enabled(
     setup_environment: tuple[Path, Path],
     mock_fluent_key: FluentKey,
@@ -200,3 +211,18 @@ def test_extraction_with_invalid_i18n_keys_ignores_them(
     )
     assert result.exit_code == 0
     assert mock_extract_function.call_args[1]["i18n_keys"] == ("nonexistent_key",)
+
+
+def test_extract_comments_junk_elements_if_needed(
+    setup_environment: tuple[Path, Path],
+    mock_leave_as_is: list,
+) -> None:
+    code_path, output_path = setup_environment
+
+    with (
+        patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value={}),
+        patch("ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({}, mock_leave_as_is)),
+        patch("ftl_extract.ftl_extractor.comment_ftl_key") as mock_comment_ftl_key,
+    ):
+        extract(code_path, output_path, ("en",), ("i18n",), comment_junks=True)
+        assert mock_comment_ftl_key.call_count == len(mock_leave_as_is)

tests/test_extract/test_ftl_comment.py

@@ -44,7 +44,7 @@ def test_ftl_comment(tmp_path: Path) -> None:
     comment_ftl_key(ftl_keys["key-4"], serializer=serializer)
     comment_ftl_key(ftl_keys["key-5"], serializer=serializer)
 
-    ftl, _ = generate_ftl(ftl_keys, serializer=serializer)
+    ftl, _ = generate_ftl(ftl_keys.values(), serializer=serializer, leave_as_is=leave_as_is)
     (tmp_path / "test.ftl").write_text(ftl, encoding="utf-8")
 
     ftl = (tmp_path / "test.ftl").read_text(encoding="utf-8")

tests/test_ftl_import.py

@@ -16,6 +16,7 @@ def mock_ftl_content() -> str:
 -term = Term value
 hello = Hello, world!
 welcome = Welcome, { $name }!
+Junk
 """
 
 
@@ -24,7 +25,7 @@ def test_import_from_ftl_with_valid_ftl_file(mock_ftl_content: str) -> None:
         keys, resource, leave_as_is = import_from_ftl(Path("/path/to/locale/en/example.ftl"), "en")
         assert "hello" in keys
         assert "welcome" in keys
-        assert len(resource.body) == 6  # noqa: PLR2004
+        assert len(resource.body) == 7  # noqa: PLR2004
 
 
 def test_import_from_ftl_with_empty_ftl_file() -> None:
@@ -59,8 +60,9 @@ def test_import_ftl_from_dir_with_nonexistent_directory() -> None:
 def test_import_from_ftl_appends_non_message_entries_correctly(mock_ftl_content: str) -> None:
     with patch("pathlib.Path.read_text", return_value=mock_ftl_content):
         _, _, leave_as_is = import_from_ftl(Path("/path/to/locale/en/various_entries.ftl"), "en")
-        assert len(leave_as_is) == 4  # noqa: PLR2004
-        assert isinstance(leave_as_is[0], ast.Comment)
-        assert isinstance(leave_as_is[1], ast.GroupComment)
-        assert isinstance(leave_as_is[2], ast.ResourceComment)
-        assert isinstance(leave_as_is[3], ast.Term)
+        assert len(leave_as_is) == 5  # noqa: PLR2004
+        assert isinstance(leave_as_is[0].translation, ast.Comment)
+        assert isinstance(leave_as_is[1].translation, ast.GroupComment)
+        assert isinstance(leave_as_is[2].translation, ast.ResourceComment)
+        assert isinstance(leave_as_is[3].translation, ast.Term)
+        assert isinstance(leave_as_is[4].translation, ast.Junk)