[BREAKING CHANGES] Drop libcst (#3)

* [BREAKING CHANGES]
Drop `libcst`.
Drop argument `beauty`.
Drop `BeautyFluentSerializer`

Add use of `ast` (built-in python lib)
Add arguments `ignore_attributes` & `expand_ignore_attributes`
Add time measurements

Fix non-Junk objects commenting
Fix leave_as_is path sorting

Rewrite tests

Update dependencies

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       - id: "check-json"
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.5.4
+    rev: v0.5.5
     hooks:
       - id: ruff
         args: [ "--fix" ]

pyproject.toml

@@ -13,7 +13,6 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.9"
 fluent-syntax = "^0.19"
-libcst = "^1.4"
 
 [tool.poetry.group.test.dependencies]
 pytest = "8.3.2"
@@ -33,7 +32,7 @@ pytz = "2024.1"
 black = "24.4.2"
 isort = "5.13.2"
 pre-commit = "3.7.1"
-ruff = "0.5.4"
+ruff = "0.5.5"
 mypy = "1.11.0"
 typing-extensions = "4.12.2"
 

src/ftl_extract/cli.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from time import perf_counter_ns
 
 import click
 
@@ -27,11 +28,17 @@
     help="Names of function that is used to get translation.",
 )
 @click.option(
-    "--beauty",
-    is_flag=True,
-    default=False,
+    "--ignore-attributes",
+    default=("set_locale", "use_locale", "use_context", "set_context"),
+    multiple=True,
     show_default=True,
-    help="Beautify output FTL files.",
+    help="Ignore attributes, like `i18n.set_locale`.",
+)
+@click.option(
+    "--expand-ignore-attributes",
+    "-a",
+    multiple=True,
+    help="Expand default|targeted ignore attributes.",
 )
 @click.option(
     "--comment-junks",
@@ -46,16 +53,21 @@ def cli_extract(
     output_path: Path,
     language: tuple[str, ...],
     i18n_keys: tuple[str, ...],
-    beauty: bool = False,
+    ignore_attributes: tuple[str, ...],
+    expand_ignore_attributes: tuple[str, ...] | None = None,
     comment_junks: bool = False,
 ) -> None:
     click.echo(f"Extracting from {code_path}...")
+    start_time = perf_counter_ns()
 
     extract(
         code_path=code_path,
         output_path=output_path,
         language=language,
         i18n_keys=i18n_keys,
-        beauty=beauty,
+        ignore_attributes=ignore_attributes,
+        expand_ignore_attributes=expand_ignore_attributes,
         comment_junks=comment_junks,
     )
+
+    click.echo(f"Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.")

src/ftl_extract/code_extractor.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
+import ast
 from pathlib import Path
 from typing import TYPE_CHECKING, cast
 
-import libcst as cst
-from fluent.syntax import ast
+from fluent.syntax import ast as fluent_ast
 
 from ftl_extract.exceptions import (
     FTLExtractorDifferentPathsError,
@@ -13,7 +13,7 @@
 from ftl_extract.matcher import I18nMatcher
 
 if TYPE_CHECKING:
-    from collections.abc import Iterator, Sequence
+    from collections.abc import Iterable, Iterator
 
     from ftl_extract.matcher import FluentKey
 
@@ -30,20 +30,26 @@ def find_py_files(path: Path) -> Iterator[Path]:
     yield from path.rglob("[!{.}]*.py") if path.is_dir() else [path]
 
 
-def parse_file(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]:
+def parse_file(
+    path: Path,
+    i18n_keys: str | Iterable[str],
+    ignore_attributes: Iterable[str],
+) -> dict[str, FluentKey]:
     """
     Second step: parse given .py file and find all i18n calls.
 
     :param path: Path to .py file.
     :type path: Path
     :param i18n_keys: Names of function that is used to get translation.
     :type i18n_keys: str | Sequence[str]
+    :param ignore_attributes: Ignore attributes, like `i18n.set_locale`.
+    :type ignore_attributes: Sequence[str]
     :return: Dict with `key` and `FluentKey`.
     :rtype: dict[str, FluentKey]
     """
-    module = cst.parse_module(path.read_bytes())
-    matcher = I18nMatcher(code_path=path, func_names=i18n_keys)
-    matcher.extract_matches(module)
+    node = ast.parse(path.read_bytes())
+    matcher = I18nMatcher(code_path=path, func_names=i18n_keys, ignore_attributes=ignore_attributes)
+    matcher.visit(node)
     return matcher.fluent_keys
 
 
@@ -90,27 +96,33 @@ def find_conflicts(
         if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation):
             raise FTLExtractorDifferentTranslationError(
                 key,
-                cast(ast.Message, current_fluent_keys[key].translation),
-                cast(ast.Message, new_fluent_keys[key].translation),
+                cast(fluent_ast.Message, current_fluent_keys[key].translation),
+                cast(fluent_ast.Message, new_fluent_keys[key].translation),
             )
 
 
-def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]:
+def extract_fluent_keys(
+    path: Path,
+    i18n_keys: str | Iterable[str],
+    ignore_attributes: Iterable[str],
+) -> dict[str, FluentKey]:
     """
     Extract all `FluentKey`s from given path.
 
     :param path: Path to [.py file] / [directory with .py files].
     :type path: Path
     :param i18n_keys: Names of function that is used to get translation.
     :type i18n_keys: str | Sequence[str]
+    :param ignore_attributes: Ignore attributes, like `i18n.set_locale`.
+    :type ignore_attributes: Sequence[str]
     :return: Dict with `key` and `FluentKey`.
     :rtype: dict[str, FluentKey]
 
     """
     fluent_keys: dict[str, FluentKey] = {}
 
     for file in find_py_files(path):
-        keys = parse_file(file, i18n_keys)
+        keys = parse_file(path=file, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes)
         post_process_fluent_keys(keys)
         find_conflicts(fluent_keys, keys)
         fluent_keys.update(keys)

src/ftl_extract/const.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from typing import Literal
+
+I18N_LITERAL: Literal["i18n"] = "i18n"
+GET_LITERAL: Literal["get"] = "get"
+PATH_LITERAL: Literal["_path"] = "_path"
+IGNORE_ATTRIBUTES: frozenset[str] = frozenset(
+    {"set_locale", "use_locale", "use_context", "set_context"}
+)

src/ftl_extract/ftl_extractor.py

@@ -4,15 +4,18 @@
 
 from click import echo
 from fluent.syntax import FluentSerializer
+from fluent.syntax import ast as fl_ast
 
 from ftl_extract import extract_fluent_keys
 from ftl_extract.code_extractor import sort_fluent_keys_by_path
+from ftl_extract.const import IGNORE_ATTRIBUTES
 from ftl_extract.ftl_importer import import_ftl_from_dir
 from ftl_extract.process.commentator import comment_ftl_key
 from ftl_extract.process.kwargs_extractor import extract_kwargs
-from ftl_extract.process.serializer import BeautyFluentSerializer, generate_ftl
+from ftl_extract.process.serializer import generate_ftl
 
 if TYPE_CHECKING:
+    from collections.abc import Iterable
     from pathlib import Path
 
     from ftl_extract.matcher import FluentKey
@@ -23,18 +26,23 @@ def extract(
     output_path: Path,
     language: tuple[str, ...],
     i18n_keys: tuple[str, ...],
-    beauty: bool = False,
+    ignore_attributes: Iterable[str] = IGNORE_ATTRIBUTES,
+    expand_ignore_attributes: Iterable[str] | None = None,
     comment_junks: bool = False,
+    serializer: FluentSerializer | None = None,
 ) -> None:
-    serializer: FluentSerializer | BeautyFluentSerializer
+    if expand_ignore_attributes is not None:
+        ignore_attributes = frozenset(set(ignore_attributes) | set(expand_ignore_attributes or []))
 
-    if beauty is True:
-        serializer = BeautyFluentSerializer(with_junk=True)
-    else:
+    if serializer is None:
         serializer = FluentSerializer(with_junk=True)
 
     # Extract fluent keys from code
-    in_code_fluent_keys = extract_fluent_keys(code_path, i18n_keys)
+    in_code_fluent_keys = extract_fluent_keys(
+        path=code_path,
+        i18n_keys=i18n_keys,
+        ignore_attributes=ignore_attributes,
+    )
 
     for lang in language:
         # Import fluent keys from existing FTL files
@@ -81,7 +89,8 @@ def extract(
         # Comment Junk elements if needed
         if comment_junks is True:
             for fluent_key in leave_as_is:
-                comment_ftl_key(fluent_key, serializer)
+                if isinstance(fluent_key.translation, fl_ast.Junk):
+                    comment_ftl_key(fluent_key, serializer)
 
         sorted_fluent_keys = sort_fluent_keys_by_path(stored_fluent_keys)
 
@@ -91,8 +100,17 @@ def extract(
         for path, keys in sort_fluent_keys_by_path(keys_to_comment).items():
             sorted_fluent_keys.setdefault(path, []).extend(keys)
 
+        leave_as_is_with_path: dict[Path, list[FluentKey]] = {}
+
+        for fluent_key in leave_as_is:
+            leave_as_is_with_path.setdefault(
+                fluent_key.path.relative_to(output_path / lang), []
+            ).append(fluent_key)
+
         for path, keys in sorted_fluent_keys.items():
-            ftl, _ = generate_ftl(keys, serializer=serializer, leave_as_is=leave_as_is)
+            ftl, _ = generate_ftl(
+                keys, serializer=serializer, leave_as_is=leave_as_is_with_path.get(path, [])
+            )
             (output_path / lang / path).parent.mkdir(parents=True, exist_ok=True)
             (output_path / lang / path).write_text(ftl, encoding="utf-8")
             echo(f"File {output_path / lang / path} has been saved. {len(keys)} keys updated.")