diff --git a/poetry.lock b/poetry.lock index e919311..3e17837 100644 --- a/poetry.lock +++ b/poetry.lock @@ -518,46 +518,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "libcst" -version = "1.4.0" -description = "A concrete syntax tree with AST-like properties for Python 3.0 through 3.12 programs." -optional = false -python-versions = ">=3.9" -files = [ - {file = "libcst-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:279b54568ea1f25add50ea4ba3d76d4f5835500c82f24d54daae4c5095b986aa"}, - {file = "libcst-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3401dae41fe24565387a65baee3887e31a44e3e58066b0250bc3f3ccf85b1b5a"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1989fa12d3cd79118ebd29ebe2a6976d23d509b1a4226bc3d66fcb7cb50bd5d"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addc6d585141a7677591868886f6bda0577529401a59d210aa8112114340e129"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17d71001cb25e94cfe8c3d997095741a8c4aa7a6d234c0f972bc42818c88dfaf"}, - {file = "libcst-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:2d47de16d105e7dd5f4e01a428d9f4dc1e71efd74f79766daf54528ce37f23c3"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6227562fc5c9c1efd15dfe90b0971ae254461b8b6b23c1b617139b6003de1c1"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3399e6c95df89921511b44d8c5bf6a75bcbc2d51f1f6429763609ba005c10f6b"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48601e3e590e2d6a7ab8c019cf3937c70511a78d778ab3333764531253acdb33"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42797309bb725f0f000510d5463175ccd7155395f09b5e7723971b0007a976d"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb4e42ea107a37bff7f9fdbee9532d39f9ea77b89caa5c5112b37057b12e0838"}, - {file = "libcst-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:9d0cc3c5a2a51fa7e1d579a828c0a2e46b2170024fd8b1a0691c8a52f3abb2d9"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7ece51d935bc9bf60b528473d2e5cc67cbb88e2f8146297e40ee2c7d80be6f13"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:81653dea1cdfa4c6520a7c5ffb95fa4d220cbd242e446c7a06d42d8636bfcbba"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6abce0e66bba2babfadc20530fd3688f672d565674336595b4623cd800b91ef"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5da9d7dc83801aba3b8d911f82dc1a375db0d508318bad79d9fb245374afe068"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c54aa66c86d8ece9c93156a2cf5ca512b0dce40142fe9e072c86af2bf892411"}, - {file = "libcst-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:62e2682ee1567b6a89c91853865372bf34f178bfd237853d84df2b87b446e654"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8ecdba8934632b4dadacb666cd3816627a6ead831b806336972ccc4ba7ca0e9"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8e54c777b8d27339b70f304d16fc8bc8674ef1bd34ed05ea874bf4921eb5a313"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:061d6855ef30efe38b8a292b7e5d57c8e820e71fc9ec9846678b60a934b53bbb"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb0abf627ee14903d05d0ad9b2c6865f1b21eb4081e2c7bea1033f85db2b8bae"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d024f44059a853b4b852cfc04fec33e346659d851371e46fc8e7c19de24d3da9"}, - {file = "libcst-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:3c6a8faab9da48c5b371557d0999b4ca51f4f2cbd37ee8c2c4df0ac01c781465"}, - {file = "libcst-1.4.0.tar.gz", hash = "sha256:449e0b16604f054fa7f27c3ffe86ea7ef6c409836fe68fe4e752a1894175db00"}, -] - -[package.dependencies] -pyyaml = ">=5.2" - -[package.extras] -dev = ["Sphinx (>=5.1.1)", "black (==23.12.1)", "build (>=0.10.0)", "coverage (>=4.5.4)", "fixit (==2.1.0)", "flake8 (==7.0.0)", "hypothesis (>=4.36.0)", "hypothesmith (>=0.0.4)", "jinja2 (==3.1.4)", "jupyter (>=1.0.0)", "maturin (>=0.8.3,<1.6)", "nbsphinx (>=0.4.2)", "prompt-toolkit (>=2.0.9)", "pyre-check (==0.9.18)", "setuptools-rust (>=1.5.2)", "setuptools-scm (>=6.0.1)", "slotscheck (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "ufmt (==2.6.0)", "usort (==1.0.8.post1)"] - [[package]] name = "markupsafe" version = "2.1.5" @@ -1511,4 +1471,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "b91af9c4cf2d52ad6904bb0b429a0eebbb71c1ad33edf6fd066ad24e05875f70" +content-hash = "9870b2a76e62b54b19f2332826ddce3202cb2295eebed2a31b0e974e6f93ed00" diff --git a/pyproject.toml b/pyproject.toml index bc1dea2..953a1da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ packages = [ [tool.poetry.dependencies] python = ">=3.9" fluent-syntax = "^0.19" -libcst = "^1.4" [tool.poetry.group.test.dependencies] pytest = "8.3.2" diff --git a/src/ftl_extract/cli.py b/src/ftl_extract/cli.py index 9ec94df..89f7fbd 100644 --- a/src/ftl_extract/cli.py +++ b/src/ftl_extract/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from time import perf_counter_ns import click @@ -27,11 +28,17 @@ help="Names of function that is used to get translation.", ) @click.option( - "--beauty", - is_flag=True, - default=False, + "--ignore-attributes", + default=("set_locale", "use_locale", "use_context", "set_context"), + multiple=True, show_default=True, - help="Beautify output FTL files.", + help="Ignore attributes, like `i18n.set_locale`.", +) +@click.option( + "--expand-ignore-attributes", + "-a", + multiple=True, + help="Expand default|targeted ignore attributes.", ) @click.option( "--comment-junks", @@ -46,16 +53,21 @@ def cli_extract( output_path: Path, language: tuple[str, ...], i18n_keys: tuple[str, ...], - beauty: bool = False, + ignore_attributes: tuple[str, ...], + expand_ignore_attributes: tuple[str, ...] | None = None, comment_junks: bool = False, ) -> None: click.echo(f"Extracting from {code_path}...") + start_time = perf_counter_ns() extract( code_path=code_path, output_path=output_path, language=language, i18n_keys=i18n_keys, - beauty=beauty, + ignore_attributes=ignore_attributes, + expand_ignore_attributes=expand_ignore_attributes, comment_junks=comment_junks, ) + + click.echo(f"Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.") diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index 305abd0..fd5491c 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -1,10 +1,10 @@ from __future__ import annotations +import ast from pathlib import Path from typing import TYPE_CHECKING, cast -import libcst as cst -from fluent.syntax import ast +from fluent.syntax import ast as fluent_ast from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, @@ -13,7 +13,7 @@ from ftl_extract.matcher import I18nMatcher if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterable, Iterator from ftl_extract.matcher import FluentKey @@ -30,7 +30,11 @@ def find_py_files(path: Path) -> Iterator[Path]: yield from path.rglob("[!{.}]*.py") if path.is_dir() else [path] -def parse_file(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]: +def parse_file( + path: Path, + i18n_keys: str | Iterable[str], + ignore_attributes: Iterable[str], +) -> dict[str, FluentKey]: """ Second step: parse given .py file and find all i18n calls. @@ -38,12 +42,14 @@ def parse_file(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKe :type path: Path :param i18n_keys: Names of function that is used to get translation. :type i18n_keys: str | Sequence[str] + :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. + :type ignore_attributes: Sequence[str] :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] """ - module = cst.parse_module(path.read_bytes()) - matcher = I18nMatcher(code_path=path, func_names=i18n_keys) - matcher.extract_matches(module) + node = ast.parse(path.read_bytes()) + matcher = I18nMatcher(code_path=path, func_names=i18n_keys, ignore_attributes=ignore_attributes) + matcher.visit(node) return matcher.fluent_keys @@ -90,12 +96,16 @@ def find_conflicts( if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation): raise FTLExtractorDifferentTranslationError( key, - cast(ast.Message, current_fluent_keys[key].translation), - cast(ast.Message, new_fluent_keys[key].translation), + cast(fluent_ast.Message, current_fluent_keys[key].translation), + cast(fluent_ast.Message, new_fluent_keys[key].translation), ) -def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]: +def extract_fluent_keys( + path: Path, + i18n_keys: str | Iterable[str], + ignore_attributes: Iterable[str], +) -> dict[str, FluentKey]: """ Extract all `FluentKey`s from given path. @@ -103,6 +113,8 @@ def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, :type path: Path :param i18n_keys: Names of function that is used to get translation. :type i18n_keys: str | Sequence[str] + :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. + :type ignore_attributes: Sequence[str] :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] @@ -110,7 +122,7 @@ def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, fluent_keys: dict[str, FluentKey] = {} for file in find_py_files(path): - keys = parse_file(file, i18n_keys) + keys = parse_file(path=file, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes) post_process_fluent_keys(keys) find_conflicts(fluent_keys, keys) fluent_keys.update(keys) diff --git a/src/ftl_extract/const.py b/src/ftl_extract/const.py new file mode 100644 index 0000000..1120bcb --- /dev/null +++ b/src/ftl_extract/const.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from typing import Literal + +I18N_LITERAL: Literal["i18n"] = "i18n" +GET_LITERAL: Literal["get"] = "get" +PATH_LITERAL: Literal["_path"] = "_path" +IGNORE_ATTRIBUTES: frozenset[str] = frozenset( + {"set_locale", "use_locale", "use_context", "set_context"} +) diff --git a/src/ftl_extract/ftl_extractor.py b/src/ftl_extract/ftl_extractor.py index e84930f..ed1e4be 100644 --- a/src/ftl_extract/ftl_extractor.py +++ b/src/ftl_extract/ftl_extractor.py @@ -7,12 +7,14 @@ from ftl_extract import extract_fluent_keys from ftl_extract.code_extractor import sort_fluent_keys_by_path +from ftl_extract.const import IGNORE_ATTRIBUTES from ftl_extract.ftl_importer import import_ftl_from_dir from ftl_extract.process.commentator import comment_ftl_key from ftl_extract.process.kwargs_extractor import extract_kwargs -from ftl_extract.process.serializer import BeautyFluentSerializer, generate_ftl +from ftl_extract.process.serializer import generate_ftl if TYPE_CHECKING: + from collections.abc import Iterable from pathlib import Path from ftl_extract.matcher import FluentKey @@ -23,18 +25,23 @@ def extract( output_path: Path, language: tuple[str, ...], i18n_keys: tuple[str, ...], - beauty: bool = False, + ignore_attributes: Iterable[str] = IGNORE_ATTRIBUTES, + expand_ignore_attributes: Iterable[str] | None = None, comment_junks: bool = False, + serializer: FluentSerializer | None = None, ) -> None: - serializer: FluentSerializer | BeautyFluentSerializer + if expand_ignore_attributes is not None: + ignore_attributes = frozenset(set(ignore_attributes) | set(expand_ignore_attributes or [])) - if beauty is True: - serializer = BeautyFluentSerializer(with_junk=True) - else: + if serializer is None: serializer = FluentSerializer(with_junk=True) # Extract fluent keys from code - in_code_fluent_keys = extract_fluent_keys(code_path, i18n_keys) + in_code_fluent_keys = extract_fluent_keys( + path=code_path, + i18n_keys=i18n_keys, + ignore_attributes=ignore_attributes, + ) for lang in language: # Import fluent keys from existing FTL files diff --git a/src/ftl_extract/matcher.py b/src/ftl_extract/matcher.py index cd538f5..4e13c6f 100644 --- a/src/ftl_extract/matcher.py +++ b/src/ftl_extract/matcher.py @@ -1,26 +1,21 @@ from __future__ import annotations -from collections.abc import Sequence +import ast from dataclasses import dataclass, field from math import inf from pathlib import Path -from typing import TYPE_CHECKING, Callable, cast +from typing import TYPE_CHECKING, cast -import libcst as cst -from fluent.syntax import ast -from libcst import matchers as m +from fluent.syntax import ast as fluent_ast +from ftl_extract.const import GET_LITERAL, I18N_LITERAL, IGNORE_ATTRIBUTES, PATH_LITERAL from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, FTLExtractorDifferentTranslationError, ) if TYPE_CHECKING: - from typing import Literal - -I18N_LITERAL: Literal["i18n"] = "i18n" -GET_LITERAL: Literal["get"] = "get" -PATH_LITERAL: Literal["_path"] = "_path" + from collections.abc import Iterable @dataclass @@ -42,14 +37,19 @@ class FluentKey: code_path: Path key: str - translation: ast.EntryType + translation: fluent_ast.EntryType path: Path = field(default=Path("_default.ftl")) locale: str | None = field(default=None) position: int | float = field(default=inf) -class I18nMatcher: - def __init__(self, code_path: Path, func_names: str | Sequence[str] = I18N_LITERAL) -> None: +class I18nMatcher(ast.NodeVisitor): + def __init__( + self, + code_path: Path, + func_names: str | Iterable[str] = I18N_LITERAL, + ignore_attributes: str | Iterable[str] = IGNORE_ATTRIBUTES, + ) -> None: """ :param code_path: Path to .py file where visitor will be used. @@ -58,130 +58,123 @@ def __init__(self, code_path: Path, func_names: str | Sequence[str] = I18N_LITER :type func_names: str | Sequence[str] """ self.code_path = code_path - self._func_names = {func_names} if isinstance(func_names, str) else set(func_names) + self.func_names = ( + frozenset({func_names}) if isinstance(func_names, str) else frozenset(func_names) + ) + self.ignore_attributes = ( + frozenset({ignore_attributes}) + if isinstance(ignore_attributes, str) + else frozenset(ignore_attributes) + ) self.fluent_keys: dict[str, FluentKey] = {} - self._matcher = m.OneOf( - m.Call( - func=m.Attribute( - value=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - attr=m.SaveMatchedNode(matcher=~m.Name(GET_LITERAL) & m.Name(), name="key"), - ), - args=[ - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + # Check if the call matches the pattern + if isinstance(node.func, ast.Attribute): + attr: ast.Attribute | ast.expr = node.func + attrs = [] + while isinstance(attr, ast.Attribute): + attrs.append(attr.attr) + attr = attr.value + + if isinstance(attr, ast.Name) and attr.id in self.func_names: + if len(attrs) == 1 and attrs[0] == GET_LITERAL: + # Check if the call has args + if not node.args: + return # Skip if no args + + # Add the first arg as the translation key + attrs.clear() + if isinstance(arg := node.args[0], ast.Constant): + key = cast(ast.Constant, arg).value + + else: + self.generic_visit(node) + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key=key, + keywords=node.keywords, ) - ], - ), - m.Call( - func=m.Attribute( - value=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - attr=m.Name(value=GET_LITERAL), - ), - args=[ - m.Arg( - value=m.SaveMatchedNode(matcher=m.SimpleString(), name="key"), keyword=None - ), - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", - ), - ], - ), - m.Call( - func=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - args=[ - m.Arg( - value=m.SaveMatchedNode(matcher=m.SimpleString(), name="key"), keyword=None - ), - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", - ), - ], - ), - ) - - def extract_matches(self, module: cst.Module) -> None: - for match in m.extractall(module, self._matcher): - # Key - if isinstance(match["key"], cst.Name): - key = cast(cst.Name, match["key"]).value - translation = ast.Message( - id=ast.Identifier(name=key), - value=ast.Pattern( - elements=[ast.TextElement(value=cast(cst.Name, match["key"]).value)] - ), - ) - fluent_key = FluentKey( - code_path=self.code_path, - key=key, - translation=translation, - ) - elif isinstance(match["key"], cst.SimpleString): - key = cast(cst.SimpleString, match["key"]).raw_value - translation = ast.Message( - id=ast.Identifier(name=key), - value=ast.Pattern(elements=[ast.TextElement(value=key)]), - ) - fluent_key = FluentKey( - code_path=self.code_path, - key=key, - translation=translation, - ) - else: - msg = f"Unknown type of key: {type(match['key'])} | {match['key']}" - raise TypeError(msg) - # Kwargs - for kwarg in cast(Sequence[m.Arg], match["kwargs"]): - keyword = cast(cst.Name, kwarg.keyword) - if keyword.value == PATH_LITERAL: - fluent_key.path = Path(cast(cst.SimpleString, kwarg.value).raw_value) + process_fluent_key(self.fluent_keys, fluent_key) else: - if ( - isinstance(fluent_key.translation, ast.Message) - and fluent_key.translation.value is not None - ): - fluent_key.translation.value.elements.append( - ast.Placeable( - expression=ast.VariableReference( - id=ast.Identifier(name=keyword.value) - ) - ) - ) - - if fluent_key.key in self.fluent_keys: - if self.fluent_keys[fluent_key.key].path != fluent_key.path: - raise FTLExtractorDifferentPathsError( - fluent_key.key, - fluent_key.path, - self.fluent_keys[fluent_key.key].path, + if attrs[-1] in self.ignore_attributes: + self.generic_visit(node) + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key="-".join(reversed(attrs)), + keywords=node.keywords, ) - - if not self.fluent_keys[fluent_key.key].translation.equals(fluent_key.translation): - raise FTLExtractorDifferentTranslationError( - fluent_key.key, - cast(ast.Message, fluent_key.translation), - cast(ast.Message, self.fluent_keys[fluent_key.key].translation), - ) - + process_fluent_key(self.fluent_keys, fluent_key) else: - self.fluent_keys[fluent_key.key] = fluent_key + self.generic_visit(node) + + elif isinstance(node.func, ast.Name) and node.func.id in self.func_names: + if not node.args: + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key=cast(ast.Constant, node.args[0]).value, + keywords=node.keywords, + ) + process_fluent_key(self.fluent_keys, fluent_key) + + else: + self.generic_visit(node) + + self.generic_visit(node) + + +def create_fluent_key( + code_path: Path, + key: str, + keywords: list[ast.keyword], +) -> FluentKey: + fluent_key = FluentKey( + code_path=code_path, + key=key, + translation=fluent_ast.Message( + id=fluent_ast.Identifier(name=key), + value=fluent_ast.Pattern(elements=[fluent_ast.TextElement(value=key)]), + ), + ) + + for kw in keywords: + if kw.arg == PATH_LITERAL: + if kw.value is not None and isinstance(kw.value, ast.Constant): + fluent_key.path = Path(kw.value.value) + elif isinstance(kw.arg, str): + cast( + fluent_ast.Pattern, cast(fluent_ast.Message, fluent_key.translation).value + ).elements.append( + fluent_ast.Placeable( + expression=fluent_ast.VariableReference(id=fluent_ast.Identifier(name=kw.arg)) + ) + ) + + return fluent_key + + +def process_fluent_key(fluent_keys: dict[str, FluentKey], new_fluent_key: FluentKey) -> None: + if new_fluent_key.key in fluent_keys: + if fluent_keys[new_fluent_key.key].path != new_fluent_key.path: + raise FTLExtractorDifferentPathsError( + new_fluent_key.key, + new_fluent_key.path, + fluent_keys[new_fluent_key.key].path, + ) + if not fluent_keys[new_fluent_key.key].translation.equals(new_fluent_key.translation): + raise FTLExtractorDifferentTranslationError( + new_fluent_key.key, + cast(fluent_ast.Message, new_fluent_key.translation), + cast(fluent_ast.Message, fluent_keys[new_fluent_key.key].translation), + ) + + else: + fluent_keys[new_fluent_key.key] = new_fluent_key diff --git a/src/ftl_extract/process/serializer.py b/src/ftl_extract/process/serializer.py index 1866a0e..d8c21f7 100644 --- a/src/ftl_extract/process/serializer.py +++ b/src/ftl_extract/process/serializer.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING from fluent.syntax import FluentSerializer, ast -from fluent.syntax.serializer import serialize_junk, serialize_message, serialize_term if TYPE_CHECKING: from collections.abc import Iterable @@ -14,44 +13,6 @@ from ftl_extract.matcher import FluentKey -class BeautyFluentSerializer(FluentSerializer): - """A serializer that formats the output FTL for better readability.""" - - def serialize_entry(self, entry: ast.EntryType, state: int = 0) -> str: # pragma: no cover - """Serialize an :class:`.ast.Entry` to a string.""" - if isinstance(entry, ast.Message): - return serialize_message(entry) - if isinstance(entry, ast.Term): - return serialize_term(entry) - if isinstance(entry, ast.Comment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "#")) - return "{}\n".format(serialize_comment(entry, "#")) - if isinstance(entry, ast.GroupComment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "##")) - return "{}\n".format(serialize_comment(entry, "##")) - if isinstance(entry, ast.ResourceComment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "###")) - return "{}\n".format(serialize_comment(entry, "###")) - if isinstance(entry, ast.Junk): - return serialize_junk(entry) - raise Exception(f"Unknown entry type: {type(entry)}") # noqa: TRY002, TRY003, EM102 - - -def serialize_comment( - comment: ast.Comment | ast.GroupComment | ast.ResourceComment, - prefix: str = "#", -) -> str: # pragma: no cover - if not comment.content: - return f"{prefix}" - - return "\n".join( - [prefix if len(line) == 0 else f"{prefix} {line}" for line in comment.content.split("\n")] - ) - - def generate_ftl( fluent_keys: Iterable[FluentKey], serializer: FluentSerializer, diff --git a/tests/test_extract/test_cli_extractor.py b/tests/test_extract/test_cli_extractor.py index c71a1ed..5978866 100644 --- a/tests/test_extract/test_cli_extractor.py +++ b/tests/test_extract/test_cli_extractor.py @@ -65,28 +65,6 @@ def mock_leave_as_is() -> list: ] -def test_extract_with_beauty_enabled( - setup_environment: tuple[Path, Path], - mock_fluent_key: FluentKey, -) -> None: - code_path, output_path = setup_environment - - with ( - patch( - "ftl_extract.ftl_extractor.extract_fluent_keys", return_value={"key-1": mock_fluent_key} - ), - patch( - "ftl_extract.ftl_extractor.import_ftl_from_dir", - return_value=({"key-1": mock_fluent_key}, []), - ), - patch( - "ftl_extract.ftl_extractor.generate_ftl", return_value=("key-1 = key-1", None) - ) as mock_generate_ftl, - ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=True) - mock_generate_ftl.assert_called() - - def test_extract_with_keys_to_comment_and_add( setup_environment: tuple[Path, Path], mock_fluent_key: FluentKey, @@ -110,7 +88,7 @@ def test_extract_with_keys_to_comment_and_add( "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) ) as mock_generate_ftl, ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=False) + extract(code_path, output_path, ("en",), ("i18n",)) mock_comment_ftl_key.assert_called() mock_generate_ftl.assert_called() @@ -136,7 +114,7 @@ def test_extract_with_keys_only_to_add( "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) ) as mock_generate_ftl, ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=False) + extract(code_path, output_path, ("en",), ("i18n",)) mock_generate_ftl.assert_called() @@ -174,22 +152,6 @@ def test_extraction_with_multiple_languages_handles_all( assert mock_extract_function.call_args[1]["language"] == ("en", "fr") -def test_extraction_with_beautify_option_enables_beautification( - runner: click.testing.CliRunner, - mock_extract_function: patch, - tmp_path: Path, -) -> None: - tmp_path.joinpath("path/to/code").mkdir(parents=True) - code_path = tmp_path.joinpath("path/to/code") - output_path = tmp_path.joinpath("path/to/output") - - result = runner.invoke( - cast(BaseCommand, cli_extract), [code_path.as_posix(), output_path.as_posix(), "--beauty"] - ) - assert result.exit_code == 0 - assert mock_extract_function.call_args[1]["beauty"] is True - - def test_extraction_with_nonexistent_code_path_fails(runner: click.testing.CliRunner) -> None: result = runner.invoke(cast(BaseCommand, cli_extract), ["nonexistent/path", "path/to/output"]) assert result.exit_code != 0 diff --git a/tests/test_extract/test_common_extract.py b/tests/test_extract/test_common_extract.py index 4c214ff..afcba60 100644 --- a/tests/test_extract/test_common_extract.py +++ b/tests/test_extract/test_common_extract.py @@ -2,6 +2,7 @@ from typing import Final from ftl_extract.code_extractor import extract_fluent_keys +from ftl_extract.const import IGNORE_ATTRIBUTES CONTENT: Final[str] = """ def test(i18n): @@ -25,15 +26,22 @@ def test(i18n): L("lazy-key-4", arg_1=arg1, arg_2=arg2) L("lazy-key-5", arg_1=obj.arg1, arg_2=obj.arg2) L("lazy-key-6", arg_1=obj.arg1(), arg_2=obj.arg2()) + + i18n.attr.key.one() + i18n.attr.key.two(_path="content/file.ftl") + i18n.attr.key.three(arg_1="arg-1", arg_2="arg-2", _path="content/file.ftl") + i18n.attr.key.four(arg_1=arg1, arg_2=arg2) + i18n.attr.key.five(arg_1=obj.arg1, arg_2=obj.arg2) + i18n.attr.key.six(arg_1=obj.arg1(), arg_2=obj.arg2()) """ def test_common_extract(tmp_path: Path) -> None: (tmp_path / "test.py").write_text(CONTENT) - fluent_keys_len = 18 # Number of keys in `CONTENT`. + fluent_keys_len = 24 # Number of keys in `CONTENT`. - fluent_keys = extract_fluent_keys(tmp_path, ("i18n", "L", "LF")) + fluent_keys = extract_fluent_keys(tmp_path, ("i18n", "L", "LF"), IGNORE_ATTRIBUTES) assert fluent_keys # Check if `fluent_keys` is not empty. assert len(fluent_keys) == fluent_keys_len # Check if `fluent_keys` has `fluent_keys_len` keys. assert "key-1" in fluent_keys @@ -173,7 +181,45 @@ def test_common_extract(tmp_path: Path) -> None: assert fluent_keys["lazy-key-6"].translation.value.elements[2].expression.id.name == "arg_2" assert fluent_keys["lazy-key-6"].code_path == tmp_path / "test.py" + assert fluent_keys["attr-key-one"].key == "attr-key-one" + assert fluent_keys["attr-key-one"].path == Path("_default.ftl") + assert fluent_keys["attr-key-one"].translation.value.elements[0].value == "attr-key-one" + assert fluent_keys["attr-key-one"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-two"].key == "attr-key-two" + assert fluent_keys["attr-key-two"].path == Path("content/file.ftl") + assert fluent_keys["attr-key-two"].translation.value.elements[0].value == "attr-key-two" + assert fluent_keys["attr-key-two"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-three"].key == "attr-key-three" + assert fluent_keys["attr-key-three"].path == Path("content/file.ftl") + assert fluent_keys["attr-key-three"].translation.value.elements[0].value == "attr-key-three" + assert fluent_keys["attr-key-three"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-three"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-three"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-four"].key == "attr-key-four" + assert fluent_keys["attr-key-four"].path == Path("_default.ftl") + assert fluent_keys["attr-key-four"].translation.value.elements[0].value == "attr-key-four" + assert fluent_keys["attr-key-four"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-four"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-four"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-five"].key == "attr-key-five" + assert fluent_keys["attr-key-five"].path == Path("_default.ftl") + assert fluent_keys["attr-key-five"].translation.value.elements[0].value == "attr-key-five" + assert fluent_keys["attr-key-five"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-five"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-five"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-six"].key == "attr-key-six" + assert fluent_keys["attr-key-six"].path == Path("_default.ftl") + assert fluent_keys["attr-key-six"].translation.value.elements[0].value == "attr-key-six" + assert fluent_keys["attr-key-six"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-six"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-six"].code_path == tmp_path / "test.py" + def test_extract_fluent_keys_no_files(tmp_path: Path) -> None: - fluent_keys = extract_fluent_keys(tmp_path, "i18n") + fluent_keys = extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) assert not fluent_keys diff --git a/tests/test_extract/test_extract_similar_keys.py b/tests/test_extract/test_extract_similar_keys.py index 73037bb..21d8a29 100644 --- a/tests/test_extract/test_extract_similar_keys.py +++ b/tests/test_extract/test_extract_similar_keys.py @@ -4,6 +4,7 @@ import pytest from ftl_extract.code_extractor import extract_fluent_keys +from ftl_extract.const import IGNORE_ATTRIBUTES from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, FTLExtractorDifferentTranslationError, @@ -51,7 +52,7 @@ def test_extract_similar_keys_in_different_paths_from_one_py_file(tmp_path: Path (tmp_path / "test.py").write_text(CONTENT_1) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( @@ -62,14 +63,14 @@ def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( (tmp_path / "test2.py").write_text(CONTENT_2_2) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_with_different_translation_one_py_file(tmp_path: Path) -> None: (tmp_path / "test.py").write_text(CONTENT_3) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_with_different_translation_different_py_files( @@ -80,4 +81,4 @@ def test_extract_similar_fluent_keys_with_different_translation_different_py_fil (tmp_path / "test2.py").write_text(CONTENT_4_2) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 59290d0..8045563 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -1,10 +1,10 @@ from pathlib import Path import pytest -from fluent.syntax import ast +from fluent.syntax import FluentSerializer, ast from ftl_extract.matcher import FluentKey -from ftl_extract.process.serializer import BeautyFluentSerializer, generate_ftl +from ftl_extract.process.serializer import generate_ftl @pytest.fixture() @@ -53,7 +53,7 @@ def test_custom_serializer_produces_correct_ftl_for_single_key( ) -> None: ftl_string, resource = generate_ftl( single_fluent_key, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert "greeting = Hello, world!" in ftl_string @@ -65,7 +65,7 @@ def test_custom_serializer_produces_correct_ftl_for_multiple_keys( ) -> None: ftl_string, resource = generate_ftl( multiple_fluent_keys, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert "greeting = Hello, world!" in ftl_string @@ -78,7 +78,7 @@ def test_custom_serializer_handles_empty_fluent_keys_list_properly( ) -> None: ftl_string, resource = generate_ftl( empty_fluent_keys, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert ftl_string == "" @@ -97,7 +97,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: ), ) ], - serializer=BeautyFluentSerializer(with_junk=True), + serializer=FluentSerializer(with_junk=True), leave_as_is=[ FluentKey( code_path=Path(),