diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 994badf..3676ab6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: - id: "check-json" - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.5.4 + rev: v0.5.5 hooks: - id: ruff args: [ "--fix" ] diff --git a/poetry.lock b/poetry.lock index e919311..1486be8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -518,46 +518,6 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "libcst" -version = "1.4.0" -description = "A concrete syntax tree with AST-like properties for Python 3.0 through 3.12 programs." -optional = false -python-versions = ">=3.9" -files = [ - {file = "libcst-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:279b54568ea1f25add50ea4ba3d76d4f5835500c82f24d54daae4c5095b986aa"}, - {file = "libcst-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3401dae41fe24565387a65baee3887e31a44e3e58066b0250bc3f3ccf85b1b5a"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1989fa12d3cd79118ebd29ebe2a6976d23d509b1a4226bc3d66fcb7cb50bd5d"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addc6d585141a7677591868886f6bda0577529401a59d210aa8112114340e129"}, - {file = "libcst-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17d71001cb25e94cfe8c3d997095741a8c4aa7a6d234c0f972bc42818c88dfaf"}, - {file = "libcst-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:2d47de16d105e7dd5f4e01a428d9f4dc1e71efd74f79766daf54528ce37f23c3"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6227562fc5c9c1efd15dfe90b0971ae254461b8b6b23c1b617139b6003de1c1"}, - {file = "libcst-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3399e6c95df89921511b44d8c5bf6a75bcbc2d51f1f6429763609ba005c10f6b"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48601e3e590e2d6a7ab8c019cf3937c70511a78d778ab3333764531253acdb33"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f42797309bb725f0f000510d5463175ccd7155395f09b5e7723971b0007a976d"}, - {file = "libcst-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb4e42ea107a37bff7f9fdbee9532d39f9ea77b89caa5c5112b37057b12e0838"}, - {file = "libcst-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:9d0cc3c5a2a51fa7e1d579a828c0a2e46b2170024fd8b1a0691c8a52f3abb2d9"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7ece51d935bc9bf60b528473d2e5cc67cbb88e2f8146297e40ee2c7d80be6f13"}, - {file = "libcst-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:81653dea1cdfa4c6520a7c5ffb95fa4d220cbd242e446c7a06d42d8636bfcbba"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6abce0e66bba2babfadc20530fd3688f672d565674336595b4623cd800b91ef"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5da9d7dc83801aba3b8d911f82dc1a375db0d508318bad79d9fb245374afe068"}, - {file = "libcst-1.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c54aa66c86d8ece9c93156a2cf5ca512b0dce40142fe9e072c86af2bf892411"}, - {file = "libcst-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:62e2682ee1567b6a89c91853865372bf34f178bfd237853d84df2b87b446e654"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b8ecdba8934632b4dadacb666cd3816627a6ead831b806336972ccc4ba7ca0e9"}, - {file = "libcst-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8e54c777b8d27339b70f304d16fc8bc8674ef1bd34ed05ea874bf4921eb5a313"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:061d6855ef30efe38b8a292b7e5d57c8e820e71fc9ec9846678b60a934b53bbb"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb0abf627ee14903d05d0ad9b2c6865f1b21eb4081e2c7bea1033f85db2b8bae"}, - {file = "libcst-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d024f44059a853b4b852cfc04fec33e346659d851371e46fc8e7c19de24d3da9"}, - {file = "libcst-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:3c6a8faab9da48c5b371557d0999b4ca51f4f2cbd37ee8c2c4df0ac01c781465"}, - {file = "libcst-1.4.0.tar.gz", hash = "sha256:449e0b16604f054fa7f27c3ffe86ea7ef6c409836fe68fe4e752a1894175db00"}, -] - -[package.dependencies] -pyyaml = ">=5.2" - -[package.extras] -dev = ["Sphinx (>=5.1.1)", "black (==23.12.1)", "build (>=0.10.0)", "coverage (>=4.5.4)", "fixit (==2.1.0)", "flake8 (==7.0.0)", "hypothesis (>=4.36.0)", "hypothesmith (>=0.0.4)", "jinja2 (==3.1.4)", "jupyter (>=1.0.0)", "maturin (>=0.8.3,<1.6)", "nbsphinx (>=0.4.2)", "prompt-toolkit (>=2.0.9)", "pyre-check (==0.9.18)", "setuptools-rust (>=1.5.2)", "setuptools-scm (>=6.0.1)", "slotscheck (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "ufmt (==2.6.0)", "usort (==1.0.8.post1)"] - [[package]] name = "markupsafe" version = "2.1.5" @@ -969,29 +929,29 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "ruff" -version = "0.5.4" +version = "0.5.5" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.5.4-py3-none-linux_armv6l.whl", hash = "sha256:82acef724fc639699b4d3177ed5cc14c2a5aacd92edd578a9e846d5b5ec18ddf"}, - {file = "ruff-0.5.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:da62e87637c8838b325e65beee485f71eb36202ce8e3cdbc24b9fcb8b99a37be"}, - {file = "ruff-0.5.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e98ad088edfe2f3b85a925ee96da652028f093d6b9b56b76fc242d8abb8e2059"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c55efbecc3152d614cfe6c2247a3054cfe358cefbf794f8c79c8575456efe19"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f9b85eaa1f653abd0a70603b8b7008d9e00c9fa1bbd0bf40dad3f0c0bdd06793"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cf497a47751be8c883059c4613ba2f50dd06ec672692de2811f039432875278"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:09c14ed6a72af9ccc8d2e313d7acf7037f0faff43cde4b507e66f14e812e37f7"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f6b8f97b8bad2490240aa84f3e68f390e13fabc9af5c0d3b96b485921cd60"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3520a00c0563d7a7a7c324ad7e2cde2355733dafa9592c671fb2e9e3cd8194c1"}, - {file = "ruff-0.5.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93789f14ca2244fb91ed481456f6d0bb8af1f75a330e133b67d08f06ad85b516"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:029454e2824eafa25b9df46882f7f7844d36fd8ce51c1b7f6d97e2615a57bbcc"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9492320eed573a13a0bc09a2957f17aa733fff9ce5bf00e66e6d4a88ec33813f"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a6e1f62a92c645e2919b65c02e79d1f61e78a58eddaebca6c23659e7c7cb4ac7"}, - {file = "ruff-0.5.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:768fa9208df2bec4b2ce61dbc7c2ddd6b1be9fb48f1f8d3b78b3332c7d71c1ff"}, - {file = "ruff-0.5.4-py3-none-win32.whl", hash = "sha256:e1e7393e9c56128e870b233c82ceb42164966f25b30f68acbb24ed69ce9c3a4e"}, - {file = "ruff-0.5.4-py3-none-win_amd64.whl", hash = "sha256:58b54459221fd3f661a7329f177f091eb35cf7a603f01d9eb3eb11cc348d38c4"}, - {file = "ruff-0.5.4-py3-none-win_arm64.whl", hash = "sha256:bd53da65f1085fb5b307c38fd3c0829e76acf7b2a912d8d79cadcdb4875c1eb7"}, - {file = "ruff-0.5.4.tar.gz", hash = "sha256:2795726d5f71c4f4e70653273d1c23a8182f07dd8e48c12de5d867bfb7557eed"}, + {file = "ruff-0.5.5-py3-none-linux_armv6l.whl", hash = "sha256:605d589ec35d1da9213a9d4d7e7a9c761d90bba78fc8790d1c5e65026c1b9eaf"}, + {file = "ruff-0.5.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:00817603822a3e42b80f7c3298c8269e09f889ee94640cd1fc7f9329788d7bf8"}, + {file = "ruff-0.5.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:187a60f555e9f865a2ff2c6984b9afeffa7158ba6e1eab56cb830404c942b0f3"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe26fc46fa8c6e0ae3f47ddccfbb136253c831c3289bba044befe68f467bfb16"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ad25dd9c5faac95c8e9efb13e15803cd8bbf7f4600645a60ffe17c73f60779b"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f70737c157d7edf749bcb952d13854e8f745cec695a01bdc6e29c29c288fc36e"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:cfd7de17cef6ab559e9f5ab859f0d3296393bc78f69030967ca4d87a541b97a0"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a09b43e02f76ac0145f86a08e045e2ea452066f7ba064fd6b0cdccb486f7c3e7"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d0b856cb19c60cd40198be5d8d4b556228e3dcd545b4f423d1ad812bfdca5884"}, + {file = "ruff-0.5.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3687d002f911e8a5faf977e619a034d159a8373514a587249cc00f211c67a091"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ac9dc814e510436e30d0ba535f435a7f3dc97f895f844f5b3f347ec8c228a523"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:af9bdf6c389b5add40d89b201425b531e0a5cceb3cfdcc69f04d3d531c6be74f"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d40a8533ed545390ef8315b8e25c4bb85739b90bd0f3fe1280a29ae364cc55d8"}, + {file = "ruff-0.5.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:cab904683bf9e2ecbbe9ff235bfe056f0eba754d0168ad5407832928d579e7ab"}, + {file = "ruff-0.5.5-py3-none-win32.whl", hash = "sha256:696f18463b47a94575db635ebb4c178188645636f05e934fdf361b74edf1bb2d"}, + {file = "ruff-0.5.5-py3-none-win_amd64.whl", hash = "sha256:50f36d77f52d4c9c2f1361ccbfbd09099a1b2ea5d2b2222c586ab08885cf3445"}, + {file = "ruff-0.5.5-py3-none-win_arm64.whl", hash = "sha256:3191317d967af701f1b73a31ed5788795936e423b7acce82a2b63e26eb3e89d6"}, + {file = "ruff-0.5.5.tar.gz", hash = "sha256:cc5516bdb4858d972fbc31d246bdb390eab8df1a26e2353be2dbc0c2d7f5421a"}, ] [[package]] @@ -1511,4 +1471,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9" -content-hash = "b91af9c4cf2d52ad6904bb0b429a0eebbb71c1ad33edf6fd066ad24e05875f70" +content-hash = "01c871c9d8d12d8363b757c9317a4515ceef6d222de323952ad88f084c8c1a29" diff --git a/pyproject.toml b/pyproject.toml index bc1dea2..74feb4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,6 @@ packages = [ [tool.poetry.dependencies] python = ">=3.9" fluent-syntax = "^0.19" -libcst = "^1.4" [tool.poetry.group.test.dependencies] pytest = "8.3.2" @@ -33,7 +32,7 @@ pytz = "2024.1" black = "24.4.2" isort = "5.13.2" pre-commit = "3.7.1" -ruff = "0.5.4" +ruff = "0.5.5" mypy = "1.11.0" typing-extensions = "4.12.2" diff --git a/src/ftl_extract/cli.py b/src/ftl_extract/cli.py index 9ec94df..89f7fbd 100644 --- a/src/ftl_extract/cli.py +++ b/src/ftl_extract/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from time import perf_counter_ns import click @@ -27,11 +28,17 @@ help="Names of function that is used to get translation.", ) @click.option( - "--beauty", - is_flag=True, - default=False, + "--ignore-attributes", + default=("set_locale", "use_locale", "use_context", "set_context"), + multiple=True, show_default=True, - help="Beautify output FTL files.", + help="Ignore attributes, like `i18n.set_locale`.", +) +@click.option( + "--expand-ignore-attributes", + "-a", + multiple=True, + help="Expand default|targeted ignore attributes.", ) @click.option( "--comment-junks", @@ -46,16 +53,21 @@ def cli_extract( output_path: Path, language: tuple[str, ...], i18n_keys: tuple[str, ...], - beauty: bool = False, + ignore_attributes: tuple[str, ...], + expand_ignore_attributes: tuple[str, ...] | None = None, comment_junks: bool = False, ) -> None: click.echo(f"Extracting from {code_path}...") + start_time = perf_counter_ns() extract( code_path=code_path, output_path=output_path, language=language, i18n_keys=i18n_keys, - beauty=beauty, + ignore_attributes=ignore_attributes, + expand_ignore_attributes=expand_ignore_attributes, comment_junks=comment_junks, ) + + click.echo(f"Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.") diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index 305abd0..fd5491c 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -1,10 +1,10 @@ from __future__ import annotations +import ast from pathlib import Path from typing import TYPE_CHECKING, cast -import libcst as cst -from fluent.syntax import ast +from fluent.syntax import ast as fluent_ast from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, @@ -13,7 +13,7 @@ from ftl_extract.matcher import I18nMatcher if TYPE_CHECKING: - from collections.abc import Iterator, Sequence + from collections.abc import Iterable, Iterator from ftl_extract.matcher import FluentKey @@ -30,7 +30,11 @@ def find_py_files(path: Path) -> Iterator[Path]: yield from path.rglob("[!{.}]*.py") if path.is_dir() else [path] -def parse_file(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]: +def parse_file( + path: Path, + i18n_keys: str | Iterable[str], + ignore_attributes: Iterable[str], +) -> dict[str, FluentKey]: """ Second step: parse given .py file and find all i18n calls. @@ -38,12 +42,14 @@ def parse_file(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKe :type path: Path :param i18n_keys: Names of function that is used to get translation. :type i18n_keys: str | Sequence[str] + :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. + :type ignore_attributes: Sequence[str] :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] """ - module = cst.parse_module(path.read_bytes()) - matcher = I18nMatcher(code_path=path, func_names=i18n_keys) - matcher.extract_matches(module) + node = ast.parse(path.read_bytes()) + matcher = I18nMatcher(code_path=path, func_names=i18n_keys, ignore_attributes=ignore_attributes) + matcher.visit(node) return matcher.fluent_keys @@ -90,12 +96,16 @@ def find_conflicts( if not current_fluent_keys[key].translation.equals(new_fluent_keys[key].translation): raise FTLExtractorDifferentTranslationError( key, - cast(ast.Message, current_fluent_keys[key].translation), - cast(ast.Message, new_fluent_keys[key].translation), + cast(fluent_ast.Message, current_fluent_keys[key].translation), + cast(fluent_ast.Message, new_fluent_keys[key].translation), ) -def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, FluentKey]: +def extract_fluent_keys( + path: Path, + i18n_keys: str | Iterable[str], + ignore_attributes: Iterable[str], +) -> dict[str, FluentKey]: """ Extract all `FluentKey`s from given path. @@ -103,6 +113,8 @@ def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, :type path: Path :param i18n_keys: Names of function that is used to get translation. :type i18n_keys: str | Sequence[str] + :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. + :type ignore_attributes: Sequence[str] :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] @@ -110,7 +122,7 @@ def extract_fluent_keys(path: Path, i18n_keys: str | Sequence[str]) -> dict[str, fluent_keys: dict[str, FluentKey] = {} for file in find_py_files(path): - keys = parse_file(file, i18n_keys) + keys = parse_file(path=file, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes) post_process_fluent_keys(keys) find_conflicts(fluent_keys, keys) fluent_keys.update(keys) diff --git a/src/ftl_extract/const.py b/src/ftl_extract/const.py new file mode 100644 index 0000000..1120bcb --- /dev/null +++ b/src/ftl_extract/const.py @@ -0,0 +1,10 @@ +from __future__ import annotations + +from typing import Literal + +I18N_LITERAL: Literal["i18n"] = "i18n" +GET_LITERAL: Literal["get"] = "get" +PATH_LITERAL: Literal["_path"] = "_path" +IGNORE_ATTRIBUTES: frozenset[str] = frozenset( + {"set_locale", "use_locale", "use_context", "set_context"} +) diff --git a/src/ftl_extract/ftl_extractor.py b/src/ftl_extract/ftl_extractor.py index e84930f..c525aa0 100644 --- a/src/ftl_extract/ftl_extractor.py +++ b/src/ftl_extract/ftl_extractor.py @@ -4,15 +4,18 @@ from click import echo from fluent.syntax import FluentSerializer +from fluent.syntax import ast as fl_ast from ftl_extract import extract_fluent_keys from ftl_extract.code_extractor import sort_fluent_keys_by_path +from ftl_extract.const import IGNORE_ATTRIBUTES from ftl_extract.ftl_importer import import_ftl_from_dir from ftl_extract.process.commentator import comment_ftl_key from ftl_extract.process.kwargs_extractor import extract_kwargs -from ftl_extract.process.serializer import BeautyFluentSerializer, generate_ftl +from ftl_extract.process.serializer import generate_ftl if TYPE_CHECKING: + from collections.abc import Iterable from pathlib import Path from ftl_extract.matcher import FluentKey @@ -23,18 +26,23 @@ def extract( output_path: Path, language: tuple[str, ...], i18n_keys: tuple[str, ...], - beauty: bool = False, + ignore_attributes: Iterable[str] = IGNORE_ATTRIBUTES, + expand_ignore_attributes: Iterable[str] | None = None, comment_junks: bool = False, + serializer: FluentSerializer | None = None, ) -> None: - serializer: FluentSerializer | BeautyFluentSerializer + if expand_ignore_attributes is not None: + ignore_attributes = frozenset(set(ignore_attributes) | set(expand_ignore_attributes or [])) - if beauty is True: - serializer = BeautyFluentSerializer(with_junk=True) - else: + if serializer is None: serializer = FluentSerializer(with_junk=True) # Extract fluent keys from code - in_code_fluent_keys = extract_fluent_keys(code_path, i18n_keys) + in_code_fluent_keys = extract_fluent_keys( + path=code_path, + i18n_keys=i18n_keys, + ignore_attributes=ignore_attributes, + ) for lang in language: # Import fluent keys from existing FTL files @@ -81,7 +89,8 @@ def extract( # Comment Junk elements if needed if comment_junks is True: for fluent_key in leave_as_is: - comment_ftl_key(fluent_key, serializer) + if isinstance(fluent_key.translation, fl_ast.Junk): + comment_ftl_key(fluent_key, serializer) sorted_fluent_keys = sort_fluent_keys_by_path(stored_fluent_keys) @@ -91,8 +100,17 @@ def extract( for path, keys in sort_fluent_keys_by_path(keys_to_comment).items(): sorted_fluent_keys.setdefault(path, []).extend(keys) + leave_as_is_with_path: dict[Path, list[FluentKey]] = {} + + for fluent_key in leave_as_is: + leave_as_is_with_path.setdefault( + fluent_key.path.relative_to(output_path / lang), [] + ).append(fluent_key) + for path, keys in sorted_fluent_keys.items(): - ftl, _ = generate_ftl(keys, serializer=serializer, leave_as_is=leave_as_is) + ftl, _ = generate_ftl( + keys, serializer=serializer, leave_as_is=leave_as_is_with_path.get(path, []) + ) (output_path / lang / path).parent.mkdir(parents=True, exist_ok=True) (output_path / lang / path).write_text(ftl, encoding="utf-8") echo(f"File {output_path / lang / path} has been saved. {len(keys)} keys updated.") diff --git a/src/ftl_extract/matcher.py b/src/ftl_extract/matcher.py index cd538f5..315d9d0 100644 --- a/src/ftl_extract/matcher.py +++ b/src/ftl_extract/matcher.py @@ -1,26 +1,21 @@ from __future__ import annotations -from collections.abc import Sequence +import ast from dataclasses import dataclass, field from math import inf from pathlib import Path -from typing import TYPE_CHECKING, Callable, cast +from typing import TYPE_CHECKING, cast -import libcst as cst -from fluent.syntax import ast -from libcst import matchers as m +from fluent.syntax import ast as fluent_ast +from ftl_extract.const import GET_LITERAL, I18N_LITERAL, IGNORE_ATTRIBUTES, PATH_LITERAL from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, FTLExtractorDifferentTranslationError, ) if TYPE_CHECKING: - from typing import Literal - -I18N_LITERAL: Literal["i18n"] = "i18n" -GET_LITERAL: Literal["get"] = "get" -PATH_LITERAL: Literal["_path"] = "_path" + from collections.abc import Iterable @dataclass @@ -42,14 +37,19 @@ class FluentKey: code_path: Path key: str - translation: ast.EntryType + translation: fluent_ast.EntryType path: Path = field(default=Path("_default.ftl")) locale: str | None = field(default=None) position: int | float = field(default=inf) -class I18nMatcher: - def __init__(self, code_path: Path, func_names: str | Sequence[str] = I18N_LITERAL) -> None: +class I18nMatcher(ast.NodeVisitor): + def __init__( + self, + code_path: Path, + func_names: str | Iterable[str] = I18N_LITERAL, + ignore_attributes: str | Iterable[str] = IGNORE_ATTRIBUTES, + ) -> None: """ :param code_path: Path to .py file where visitor will be used. @@ -58,130 +58,120 @@ def __init__(self, code_path: Path, func_names: str | Sequence[str] = I18N_LITER :type func_names: str | Sequence[str] """ self.code_path = code_path - self._func_names = {func_names} if isinstance(func_names, str) else set(func_names) + self.func_names = ( + frozenset({func_names}) if isinstance(func_names, str) else frozenset(func_names) + ) + self.ignore_attributes = ( + frozenset({ignore_attributes}) + if isinstance(ignore_attributes, str) + else frozenset(ignore_attributes) + ) self.fluent_keys: dict[str, FluentKey] = {} - self._matcher = m.OneOf( - m.Call( - func=m.Attribute( - value=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - attr=m.SaveMatchedNode(matcher=~m.Name(GET_LITERAL) & m.Name(), name="key"), - ), - args=[ - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + # Check if the call matches the pattern + if isinstance(node.func, ast.Attribute): + attr: ast.Attribute | ast.expr = node.func + attrs = [] + while isinstance(attr, ast.Attribute): + attrs.append(attr.attr) + attr = attr.value + + if isinstance(attr, ast.Name) and attr.id in self.func_names: + if len(attrs) == 1 and attrs[0] == GET_LITERAL: + # Check if the call has args + if not node.args: + return # Skip if no args + + # Add the first arg as the translation key + attrs.clear() + if isinstance(arg := node.args[0], ast.Constant): + key = cast(ast.Constant, arg).value + + else: + self.generic_visit(node) + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key=key, + keywords=node.keywords, ) - ], - ), - m.Call( - func=m.Attribute( - value=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - attr=m.Name(value=GET_LITERAL), - ), - args=[ - m.Arg( - value=m.SaveMatchedNode(matcher=m.SimpleString(), name="key"), keyword=None - ), - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", - ), - ], - ), - m.Call( - func=m.OneOf(*map(cast(Callable, m.Name), self._func_names)), - args=[ - m.Arg( - value=m.SaveMatchedNode(matcher=m.SimpleString(), name="key"), keyword=None - ), - m.SaveMatchedNode( - matcher=m.ZeroOrMore( - m.Arg( - value=m.DoNotCare(), - keyword=m.Name(), - ) - ), - name="kwargs", - ), - ], - ), - ) - - def extract_matches(self, module: cst.Module) -> None: - for match in m.extractall(module, self._matcher): - # Key - if isinstance(match["key"], cst.Name): - key = cast(cst.Name, match["key"]).value - translation = ast.Message( - id=ast.Identifier(name=key), - value=ast.Pattern( - elements=[ast.TextElement(value=cast(cst.Name, match["key"]).value)] - ), - ) - fluent_key = FluentKey( - code_path=self.code_path, - key=key, - translation=translation, - ) - elif isinstance(match["key"], cst.SimpleString): - key = cast(cst.SimpleString, match["key"]).raw_value - translation = ast.Message( - id=ast.Identifier(name=key), - value=ast.Pattern(elements=[ast.TextElement(value=key)]), - ) - fluent_key = FluentKey( - code_path=self.code_path, - key=key, - translation=translation, - ) - else: - msg = f"Unknown type of key: {type(match['key'])} | {match['key']}" - raise TypeError(msg) - # Kwargs - for kwarg in cast(Sequence[m.Arg], match["kwargs"]): - keyword = cast(cst.Name, kwarg.keyword) - if keyword.value == PATH_LITERAL: - fluent_key.path = Path(cast(cst.SimpleString, kwarg.value).raw_value) + process_fluent_key(self.fluent_keys, fluent_key) else: - if ( - isinstance(fluent_key.translation, ast.Message) - and fluent_key.translation.value is not None - ): - fluent_key.translation.value.elements.append( - ast.Placeable( - expression=ast.VariableReference( - id=ast.Identifier(name=keyword.value) - ) - ) - ) - - if fluent_key.key in self.fluent_keys: - if self.fluent_keys[fluent_key.key].path != fluent_key.path: - raise FTLExtractorDifferentPathsError( - fluent_key.key, - fluent_key.path, - self.fluent_keys[fluent_key.key].path, + if attrs[-1] in self.ignore_attributes: + self.generic_visit(node) + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key="-".join(reversed(attrs)), + keywords=node.keywords, ) - - if not self.fluent_keys[fluent_key.key].translation.equals(fluent_key.translation): - raise FTLExtractorDifferentTranslationError( - fluent_key.key, - cast(ast.Message, fluent_key.translation), - cast(ast.Message, self.fluent_keys[fluent_key.key].translation), - ) - + process_fluent_key(self.fluent_keys, fluent_key) else: - self.fluent_keys[fluent_key.key] = fluent_key + self.generic_visit(node) + + elif isinstance(node.func, ast.Name) and node.func.id in self.func_names: + if not node.args: + return + + fluent_key = create_fluent_key( + code_path=self.code_path, + key=cast(ast.Constant, node.args[0]).value, + keywords=node.keywords, + ) + process_fluent_key(self.fluent_keys, fluent_key) + + self.generic_visit(node) + + +def create_fluent_key( + code_path: Path, + key: str, + keywords: list[ast.keyword], +) -> FluentKey: + fluent_key = FluentKey( + code_path=code_path, + key=key, + translation=fluent_ast.Message( + id=fluent_ast.Identifier(name=key), + value=fluent_ast.Pattern(elements=[fluent_ast.TextElement(value=key)]), + ), + ) + + for kw in keywords: + if kw.arg == PATH_LITERAL: + if kw.value is not None and isinstance(kw.value, ast.Constant): + fluent_key.path = Path(kw.value.value) + elif isinstance(kw.arg, str): + cast( + fluent_ast.Pattern, cast(fluent_ast.Message, fluent_key.translation).value + ).elements.append( + fluent_ast.Placeable( + expression=fluent_ast.VariableReference(id=fluent_ast.Identifier(name=kw.arg)) + ) + ) + + return fluent_key + + +def process_fluent_key(fluent_keys: dict[str, FluentKey], new_fluent_key: FluentKey) -> None: + if new_fluent_key.key in fluent_keys: + if fluent_keys[new_fluent_key.key].path != new_fluent_key.path: + raise FTLExtractorDifferentPathsError( + new_fluent_key.key, + new_fluent_key.path, + fluent_keys[new_fluent_key.key].path, + ) + if not fluent_keys[new_fluent_key.key].translation.equals(new_fluent_key.translation): + raise FTLExtractorDifferentTranslationError( + new_fluent_key.key, + cast(fluent_ast.Message, new_fluent_key.translation), + cast(fluent_ast.Message, fluent_keys[new_fluent_key.key].translation), + ) + + else: + fluent_keys[new_fluent_key.key] = new_fluent_key diff --git a/src/ftl_extract/process/serializer.py b/src/ftl_extract/process/serializer.py index 1866a0e..d8c21f7 100644 --- a/src/ftl_extract/process/serializer.py +++ b/src/ftl_extract/process/serializer.py @@ -4,7 +4,6 @@ from typing import TYPE_CHECKING from fluent.syntax import FluentSerializer, ast -from fluent.syntax.serializer import serialize_junk, serialize_message, serialize_term if TYPE_CHECKING: from collections.abc import Iterable @@ -14,44 +13,6 @@ from ftl_extract.matcher import FluentKey -class BeautyFluentSerializer(FluentSerializer): - """A serializer that formats the output FTL for better readability.""" - - def serialize_entry(self, entry: ast.EntryType, state: int = 0) -> str: # pragma: no cover - """Serialize an :class:`.ast.Entry` to a string.""" - if isinstance(entry, ast.Message): - return serialize_message(entry) - if isinstance(entry, ast.Term): - return serialize_term(entry) - if isinstance(entry, ast.Comment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "#")) - return "{}\n".format(serialize_comment(entry, "#")) - if isinstance(entry, ast.GroupComment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "##")) - return "{}\n".format(serialize_comment(entry, "##")) - if isinstance(entry, ast.ResourceComment): - if state & self.HAS_ENTRIES: - return "\n{}\n".format(serialize_comment(entry, "###")) - return "{}\n".format(serialize_comment(entry, "###")) - if isinstance(entry, ast.Junk): - return serialize_junk(entry) - raise Exception(f"Unknown entry type: {type(entry)}") # noqa: TRY002, TRY003, EM102 - - -def serialize_comment( - comment: ast.Comment | ast.GroupComment | ast.ResourceComment, - prefix: str = "#", -) -> str: # pragma: no cover - if not comment.content: - return f"{prefix}" - - return "\n".join( - [prefix if len(line) == 0 else f"{prefix} {line}" for line in comment.content.split("\n")] - ) - - def generate_ftl( fluent_keys: Iterable[FluentKey], serializer: FluentSerializer, diff --git a/tests/test_extract/test_cli_extractor.py b/tests/test_extract/test_cli_extractor.py index c71a1ed..806bfa3 100644 --- a/tests/test_extract/test_cli_extractor.py +++ b/tests/test_extract/test_cli_extractor.py @@ -1,3 +1,4 @@ +import ast from pathlib import Path from typing import cast from unittest.mock import MagicMock, patch @@ -5,30 +6,31 @@ import click.testing import pytest from click import BaseCommand -from fluent.syntax import ast +from fluent.syntax import FluentSerializer +from fluent.syntax import ast as fl_ast from ftl_extract.cli import cli_extract from ftl_extract.ftl_extractor import extract -from ftl_extract.matcher import FluentKey +from ftl_extract.matcher import FluentKey, I18nMatcher @pytest.fixture() def mock_fluent_key(tmp_path: Path) -> FluentKey: mock = MagicMock(spec=FluentKey) mock.code_path = tmp_path / "code" - mock.translation = MagicMock(spec=ast.Message) + mock.translation = MagicMock(spec=fl_ast.Message) - mock.translation.id = MagicMock(spec=ast.Identifier) + mock.translation.id = MagicMock(spec=fl_ast.Identifier) mock.translation.id.name = "key-1" - text_element = MagicMock(spec=ast.TextElement) + text_element = MagicMock(spec=fl_ast.TextElement) text_element.value = "key-1" - mock.translation.value = MagicMock(spec=ast.Pattern) + mock.translation.value = MagicMock(spec=fl_ast.Pattern) mock.translation.value.elements = [text_element] mock.translation.attributes = [] - mock.translation.comment = MagicMock(spec=ast.Comment) + mock.translation.comment = MagicMock(spec=fl_ast.Comment) mock.translation.comment.content = "Comment" return mock @@ -57,36 +59,14 @@ def mock_extract_function() -> patch: @pytest.fixture() def mock_leave_as_is() -> list: return [ - MagicMock(spec=ast.Comment), - MagicMock(spec=ast.GroupComment), - MagicMock(spec=ast.ResourceComment), - MagicMock(spec=ast.Term), - MagicMock(spec=ast.Junk), + MagicMock(spec=fl_ast.Comment), + MagicMock(spec=fl_ast.GroupComment), + MagicMock(spec=fl_ast.ResourceComment), + MagicMock(spec=fl_ast.Term), + MagicMock(spec=fl_ast.Junk), ] -def test_extract_with_beauty_enabled( - setup_environment: tuple[Path, Path], - mock_fluent_key: FluentKey, -) -> None: - code_path, output_path = setup_environment - - with ( - patch( - "ftl_extract.ftl_extractor.extract_fluent_keys", return_value={"key-1": mock_fluent_key} - ), - patch( - "ftl_extract.ftl_extractor.import_ftl_from_dir", - return_value=({"key-1": mock_fluent_key}, []), - ), - patch( - "ftl_extract.ftl_extractor.generate_ftl", return_value=("key-1 = key-1", None) - ) as mock_generate_ftl, - ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=True) - mock_generate_ftl.assert_called() - - def test_extract_with_keys_to_comment_and_add( setup_environment: tuple[Path, Path], mock_fluent_key: FluentKey, @@ -110,7 +90,7 @@ def test_extract_with_keys_to_comment_and_add( "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) ) as mock_generate_ftl, ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=False) + extract(code_path, output_path, ("en",), ("i18n",)) mock_comment_ftl_key.assert_called() mock_generate_ftl.assert_called() @@ -136,7 +116,7 @@ def test_extract_with_keys_only_to_add( "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) ) as mock_generate_ftl, ): - extract(code_path, output_path, ("en",), ("i18n",), beauty=False) + extract(code_path, output_path, ("en",), ("i18n",)) mock_generate_ftl.assert_called() @@ -174,22 +154,6 @@ def test_extraction_with_multiple_languages_handles_all( assert mock_extract_function.call_args[1]["language"] == ("en", "fr") -def test_extraction_with_beautify_option_enables_beautification( - runner: click.testing.CliRunner, - mock_extract_function: patch, - tmp_path: Path, -) -> None: - tmp_path.joinpath("path/to/code").mkdir(parents=True) - code_path = tmp_path.joinpath("path/to/code") - output_path = tmp_path.joinpath("path/to/output") - - result = runner.invoke( - cast(BaseCommand, cli_extract), [code_path.as_posix(), output_path.as_posix(), "--beauty"] - ) - assert result.exit_code == 0 - assert mock_extract_function.call_args[1]["beauty"] is True - - def test_extraction_with_nonexistent_code_path_fails(runner: click.testing.CliRunner) -> None: result = runner.invoke(cast(BaseCommand, cli_extract), ["nonexistent/path", "path/to/output"]) assert result.exit_code != 0 @@ -213,16 +177,173 @@ def test_extraction_with_invalid_i18n_keys_ignores_them( assert mock_extract_function.call_args[1]["i18n_keys"] == ("nonexistent_key",) -def test_extract_comments_junk_elements_if_needed( +def test_comment_junk_elements_if_needed(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + + mock_junk_key = MagicMock(spec=FluentKey) + mock_junk_key.translation = MagicMock(spec=fl_ast.Junk) + mock_serializer = MagicMock(spec=FluentSerializer) + + with ( + patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value={}), + patch("ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({}, [mock_junk_key])), + patch("ftl_extract.ftl_extractor.comment_ftl_key") as mock_comment_ftl_key, + patch("fluent.syntax.serializer.FluentSerializer", return_value=mock_serializer), + ): + extract( + code_path, + output_path, + ("en",), + ("i18n",), + comment_junks=True, + serializer=mock_serializer, + ) + mock_comment_ftl_key.assert_called_once_with(mock_junk_key, mock_serializer) + + +def test_expand_ignore_attributes_updates_ignore_attributes( setup_environment: tuple[Path, Path], - mock_leave_as_is: list, ) -> None: code_path, output_path = setup_environment + initial_ignore_attributes = ["attr1", "attr2"] + expand_ignore_attributes = ["attr3", "attr4"] + expected_ignore_attributes = frozenset({"attr1", "attr2", "attr3", "attr4"}) with ( patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value={}), - patch("ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({}, mock_leave_as_is)), - patch("ftl_extract.ftl_extractor.comment_ftl_key") as mock_comment_ftl_key, + patch("ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({}, [])), + patch("ftl_extract.ftl_extractor.comment_ftl_key"), + patch("ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None)), ): - extract(code_path, output_path, ("en",), ("i18n",), comment_junks=True) - assert mock_comment_ftl_key.call_count == len(mock_leave_as_is) + extract( + code_path, + output_path, + ("en",), + ("i18n",), + ignore_attributes=initial_ignore_attributes, + expand_ignore_attributes=expand_ignore_attributes, + ) + + assert ( + frozenset(initial_ignore_attributes) | frozenset(expand_ignore_attributes) + == expected_ignore_attributes + ) + + +def test_stored_fluent_keys_code_path_update(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + mock_fluent_key = MagicMock(spec=FluentKey) + mock_fluent_key.path = Path("_default.ftl") + mock_fluent_key.code_path = code_path / "some_code_path.py" + + stored_fluent_key = MagicMock(spec=FluentKey) + stored_fluent_key.path = Path(output_path / "en" / "_default.ftl") + stored_fluent_key.code_path = None + + in_code_fluent_keys = {"key-1": mock_fluent_key} + stored_fluent_keys = {"key-1": stored_fluent_key} + + with ( + patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value=in_code_fluent_keys), + patch( + "ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=(stored_fluent_keys, []) + ), + patch("ftl_extract.ftl_extractor.extract_kwargs", return_value=set()), + patch("ftl_extract.ftl_extractor.comment_ftl_key"), + patch("ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None)), + ): + extract( + code_path, + output_path, + ("en",), + ("i18n",), + ) + + assert stored_fluent_keys["key-1"].code_path == mock_fluent_key.code_path + + +def test_keys_to_comment_and_add_on_different_kwargs(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + mock_fluent_key = MagicMock(spec=FluentKey) + mock_fluent_key.path = Path("_default.ftl") + mock_fluent_key.code_path = code_path / "some_code_path.py" + + stored_fluent_key = MagicMock(spec=FluentKey) + stored_fluent_key.path = Path(output_path / "en" / "_default.ftl") + stored_fluent_key.code_path = None + + in_code_fluent_keys = {"key-1": mock_fluent_key} + stored_fluent_keys = {"key-1": stored_fluent_key} + + with ( + patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value=in_code_fluent_keys), + patch( + "ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=(stored_fluent_keys, []) + ), + patch("ftl_extract.ftl_extractor.extract_kwargs", side_effect=[{"arg1"}, {"arg2"}]), + patch("ftl_extract.ftl_extractor.comment_ftl_key"), + patch("ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None)), + ): + extract( + code_path, + output_path, + ("en",), + ("i18n",), + ) + + assert "key-1" not in stored_fluent_keys + assert "key-1" in in_code_fluent_keys + assert in_code_fluent_keys["key-1"] == mock_fluent_key + + +def test_i18n_matcher_skips_call_with_no_args(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + matcher = I18nMatcher(code_path) + + node = ast.Call(func=ast.Attribute(value=ast.Name(id="i18n"), attr="get"), args=[]) + matcher.visit_Call(node) + + assert len(matcher.fluent_keys) == 0 + + +def test_generic_visit_called_on_else_block(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + matcher = I18nMatcher(code_path) + + node = ast.Call(func=ast.Attribute(value=ast.Name(id="i18n"), attr="get"), args=[ast.Name()]) + + with patch.object(matcher, "generic_visit", wraps=matcher.generic_visit) as mock_generic_visit: + matcher.visit_Call(node) + mock_generic_visit.assert_called_with(node.args[0]) + + +def test_generic_visit_called_when_attr_in_ignore_attributes( + setup_environment: tuple[Path, Path], +) -> None: + code_path, output_path = setup_environment + matcher = I18nMatcher(code_path, ignore_attributes={"ignore_this"}) + + # Create a mock AST node for a function call with an attribute in ignore_attributes + node = ast.Call( + func=ast.Attribute( + value=ast.Name(id="i18n", ctx=ast.Load()), attr="ignore_this", ctx=ast.Load() + ), + args=[ast.Constant(value="key")], + keywords=[], + ) + + with patch.object(matcher, "generic_visit", wraps=matcher.generic_visit) as mock_generic_visit: + matcher.visit_Call(node) + mock_generic_visit.assert_called_with(node.args[0]) + + assert len(matcher.fluent_keys) == 0 + + +def test_i18n_matcher_skips_call_with_no_args_in_elif(setup_environment: tuple[Path, Path]) -> None: + code_path, output_path = setup_environment + matcher = I18nMatcher(code_path) + + node = ast.Call(func=ast.Name(id="i18n", ctx=ast.Load()), args=[], keywords=[]) + matcher.visit_Call(node) + + assert len(matcher.fluent_keys) == 0 diff --git a/tests/test_extract/test_common_extract.py b/tests/test_extract/test_common_extract.py index 4c214ff..afcba60 100644 --- a/tests/test_extract/test_common_extract.py +++ b/tests/test_extract/test_common_extract.py @@ -2,6 +2,7 @@ from typing import Final from ftl_extract.code_extractor import extract_fluent_keys +from ftl_extract.const import IGNORE_ATTRIBUTES CONTENT: Final[str] = """ def test(i18n): @@ -25,15 +26,22 @@ def test(i18n): L("lazy-key-4", arg_1=arg1, arg_2=arg2) L("lazy-key-5", arg_1=obj.arg1, arg_2=obj.arg2) L("lazy-key-6", arg_1=obj.arg1(), arg_2=obj.arg2()) + + i18n.attr.key.one() + i18n.attr.key.two(_path="content/file.ftl") + i18n.attr.key.three(arg_1="arg-1", arg_2="arg-2", _path="content/file.ftl") + i18n.attr.key.four(arg_1=arg1, arg_2=arg2) + i18n.attr.key.five(arg_1=obj.arg1, arg_2=obj.arg2) + i18n.attr.key.six(arg_1=obj.arg1(), arg_2=obj.arg2()) """ def test_common_extract(tmp_path: Path) -> None: (tmp_path / "test.py").write_text(CONTENT) - fluent_keys_len = 18 # Number of keys in `CONTENT`. + fluent_keys_len = 24 # Number of keys in `CONTENT`. - fluent_keys = extract_fluent_keys(tmp_path, ("i18n", "L", "LF")) + fluent_keys = extract_fluent_keys(tmp_path, ("i18n", "L", "LF"), IGNORE_ATTRIBUTES) assert fluent_keys # Check if `fluent_keys` is not empty. assert len(fluent_keys) == fluent_keys_len # Check if `fluent_keys` has `fluent_keys_len` keys. assert "key-1" in fluent_keys @@ -173,7 +181,45 @@ def test_common_extract(tmp_path: Path) -> None: assert fluent_keys["lazy-key-6"].translation.value.elements[2].expression.id.name == "arg_2" assert fluent_keys["lazy-key-6"].code_path == tmp_path / "test.py" + assert fluent_keys["attr-key-one"].key == "attr-key-one" + assert fluent_keys["attr-key-one"].path == Path("_default.ftl") + assert fluent_keys["attr-key-one"].translation.value.elements[0].value == "attr-key-one" + assert fluent_keys["attr-key-one"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-two"].key == "attr-key-two" + assert fluent_keys["attr-key-two"].path == Path("content/file.ftl") + assert fluent_keys["attr-key-two"].translation.value.elements[0].value == "attr-key-two" + assert fluent_keys["attr-key-two"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-three"].key == "attr-key-three" + assert fluent_keys["attr-key-three"].path == Path("content/file.ftl") + assert fluent_keys["attr-key-three"].translation.value.elements[0].value == "attr-key-three" + assert fluent_keys["attr-key-three"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-three"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-three"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-four"].key == "attr-key-four" + assert fluent_keys["attr-key-four"].path == Path("_default.ftl") + assert fluent_keys["attr-key-four"].translation.value.elements[0].value == "attr-key-four" + assert fluent_keys["attr-key-four"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-four"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-four"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-five"].key == "attr-key-five" + assert fluent_keys["attr-key-five"].path == Path("_default.ftl") + assert fluent_keys["attr-key-five"].translation.value.elements[0].value == "attr-key-five" + assert fluent_keys["attr-key-five"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-five"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-five"].code_path == tmp_path / "test.py" + + assert fluent_keys["attr-key-six"].key == "attr-key-six" + assert fluent_keys["attr-key-six"].path == Path("_default.ftl") + assert fluent_keys["attr-key-six"].translation.value.elements[0].value == "attr-key-six" + assert fluent_keys["attr-key-six"].translation.value.elements[1].expression.id.name == "arg_1" + assert fluent_keys["attr-key-six"].translation.value.elements[2].expression.id.name == "arg_2" + assert fluent_keys["attr-key-six"].code_path == tmp_path / "test.py" + def test_extract_fluent_keys_no_files(tmp_path: Path) -> None: - fluent_keys = extract_fluent_keys(tmp_path, "i18n") + fluent_keys = extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) assert not fluent_keys diff --git a/tests/test_extract/test_extract_similar_keys.py b/tests/test_extract/test_extract_similar_keys.py index 73037bb..21d8a29 100644 --- a/tests/test_extract/test_extract_similar_keys.py +++ b/tests/test_extract/test_extract_similar_keys.py @@ -4,6 +4,7 @@ import pytest from ftl_extract.code_extractor import extract_fluent_keys +from ftl_extract.const import IGNORE_ATTRIBUTES from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, FTLExtractorDifferentTranslationError, @@ -51,7 +52,7 @@ def test_extract_similar_keys_in_different_paths_from_one_py_file(tmp_path: Path (tmp_path / "test.py").write_text(CONTENT_1) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( @@ -62,14 +63,14 @@ def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( (tmp_path / "test2.py").write_text(CONTENT_2_2) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_with_different_translation_one_py_file(tmp_path: Path) -> None: (tmp_path / "test.py").write_text(CONTENT_3) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) def test_extract_similar_fluent_keys_with_different_translation_different_py_files( @@ -80,4 +81,4 @@ def test_extract_similar_fluent_keys_with_different_translation_different_py_fil (tmp_path / "test2.py").write_text(CONTENT_4_2) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n") + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 59290d0..8045563 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -1,10 +1,10 @@ from pathlib import Path import pytest -from fluent.syntax import ast +from fluent.syntax import FluentSerializer, ast from ftl_extract.matcher import FluentKey -from ftl_extract.process.serializer import BeautyFluentSerializer, generate_ftl +from ftl_extract.process.serializer import generate_ftl @pytest.fixture() @@ -53,7 +53,7 @@ def test_custom_serializer_produces_correct_ftl_for_single_key( ) -> None: ftl_string, resource = generate_ftl( single_fluent_key, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert "greeting = Hello, world!" in ftl_string @@ -65,7 +65,7 @@ def test_custom_serializer_produces_correct_ftl_for_multiple_keys( ) -> None: ftl_string, resource = generate_ftl( multiple_fluent_keys, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert "greeting = Hello, world!" in ftl_string @@ -78,7 +78,7 @@ def test_custom_serializer_handles_empty_fluent_keys_list_properly( ) -> None: ftl_string, resource = generate_ftl( empty_fluent_keys, - serializer=BeautyFluentSerializer(), + serializer=FluentSerializer(), leave_as_is=[], ) assert ftl_string == "" @@ -97,7 +97,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: ), ) ], - serializer=BeautyFluentSerializer(with_junk=True), + serializer=FluentSerializer(with_junk=True), leave_as_is=[ FluentKey( code_path=Path(),