From 0b7400d935cca614e78e90e6aa7974c8694c6f53 Mon Sep 17 00:00:00 2001 From: andrew000 <11490628+andrew000@users.noreply.github.com> Date: Sun, 27 Oct 2024 22:55:27 +0200 Subject: [PATCH 1/3] Add `default-ftl-file` --- pyproject.toml | 79 +++++++++------ src/ftl_extract/cli.py | 8 ++ src/ftl_extract/code_extractor.py | 11 ++- src/ftl_extract/const.py | 17 ++-- src/ftl_extract/exceptions.py | 4 +- src/ftl_extract/ftl_extractor.py | 11 ++- src/ftl_extract/ftl_importer.py | 5 +- src/ftl_extract/matcher.py | 9 +- src/ftl_extract/process/kwargs_extractor.py | 7 +- tests/test_extract/__init__.py | 0 tests/test_extract/test_cli_extractor.py | 25 +++-- tests/test_extract/test_common_extract.py | 16 ++- .../test_extract/test_extract_similar_keys.py | 10 +- tests/test_kwargs_extractor.py | 98 +++++++++---------- tests/test_post_process_fluent_keys.py | 5 +- tests/test_serializer.py | 4 +- uv.lock | 2 +- 17 files changed, 189 insertions(+), 122 deletions(-) create mode 100644 tests/test_extract/__init__.py diff --git a/pyproject.toml b/pyproject.toml index e8e7779..ae926f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,14 +54,6 @@ build-backend = "hatchling.build" [tool.pytest.ini_options] testpaths = "tests" -[tool.mypy] -packages = ["src/ftl_extract"] -exclude = [ - "\\.?venv", - "\\.idea", - "\\.tests?", -] - [tool.coverage.report] exclude_lines = [ "pragma: no cover", @@ -88,28 +80,17 @@ src = ["src", "tests"] target-version = "py39" line-length = 100 exclude = [ - ".bzr", - ".direnv", - ".eggs", ".git", - ".hg", ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", ".ruff_cache", - ".svn", - ".tox", - ".venv", "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", + "__pycache__", + "*.pyi", "venv", ".venv", - "tests/.data_for_testing", + "tests/.files", + "dist", + "build", ] [tool.ruff.lint] @@ -117,19 +98,55 @@ select = ["ALL"] ignore = [ "A003", "ANN002", "ANN003", "ANN101", "ANN102", "ANN401", - "COM812", "C901", "D100", "D101", "D102", "D103", "D104", "D105", "D106", "D107", "D203", "D205", "D212", - "DTZ003", "ERA001", - "F841", "FA100", "FA102", "FBT001", "FBT002", "FIX002", - "INP001", "ISC001", + "I001", "PLR0911", "PLR0912", "PLR0913", "PLR0915", "PLR5501", "PLW0120", - "RUF", - "S101", "S311", - "TD002", "TD003" + "RUF001", + "S101", + "TD002", "TD003", +] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" + +[tool.mypy] +python_version = "3.9" +packages = ["src/ftl_extract"] +plugins = [] +allow_redefinition = true +check_untyped_defs = true +disallow_any_generics = true +disallow_incomplete_defs = true +disallow_untyped_calls = true +disallow_untyped_defs = true +extra_checks = true +follow_imports = "skip" +follow_imports_for_stubs = false +ignore_missing_imports = false +namespace_packages = true +no_implicit_optional = true +no_implicit_reexport = true +pretty = true +show_absolute_path = true +show_error_codes = true +show_error_context = true +warn_redundant_casts = true +warn_unused_configs = true +warn_unused_ignores = true +disable_error_code = [ + "no-redef", +] +exclude = [ + "\\.?venv", + "\\.idea", + "\\.tests?", ] diff --git a/src/ftl_extract/cli.py b/src/ftl_extract/cli.py index 89f7fbd..870d921 100644 --- a/src/ftl_extract/cli.py +++ b/src/ftl_extract/cli.py @@ -5,6 +5,7 @@ import click +from ftl_extract.const import DEFAULT_FTL_FILE from ftl_extract.ftl_extractor import extract @@ -47,6 +48,11 @@ show_default=True, help="Comments Junk elements.", ) +@click.option( + "--default-ftl-file", + default=DEFAULT_FTL_FILE, + show_default=True, +) @click.version_option() def cli_extract( code_path: Path, @@ -56,6 +62,7 @@ def cli_extract( ignore_attributes: tuple[str, ...], expand_ignore_attributes: tuple[str, ...] | None = None, comment_junks: bool = False, + default_ftl_file: str = DEFAULT_FTL_FILE, ) -> None: click.echo(f"Extracting from {code_path}...") start_time = perf_counter_ns() @@ -68,6 +75,7 @@ def cli_extract( ignore_attributes=ignore_attributes, expand_ignore_attributes=expand_ignore_attributes, comment_junks=comment_junks, + default_ftl_file=default_ftl_file, ) click.echo(f"Done in {(perf_counter_ns() - start_time) * 1e-9:.3f}s.") diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index fd5491c..8f9edf1 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -53,19 +53,21 @@ def parse_file( return matcher.fluent_keys -def post_process_fluent_keys(fluent_keys: dict[str, FluentKey]) -> None: +def post_process_fluent_keys(fluent_keys: dict[str, FluentKey], default_ftl_file: str) -> None: """ Third step: post-process parsed `FluentKey`. :param fluent_keys: Dict with `key` and `FluentKey` that will be post-processed. :type fluent_keys: dict[str, FluentKey] + :param default_ftl_file: Default name of FTL file. + :type default_ftl_file: str """ for fluent_key in fluent_keys.values(): if not isinstance(fluent_key.path, Path): fluent_key.path = Path(fluent_key.path) if not fluent_key.path.suffix: # if path looks like directory (no suffix) - fluent_key.path /= "_default.ftl" + fluent_key.path /= default_ftl_file def find_conflicts( @@ -105,6 +107,7 @@ def extract_fluent_keys( path: Path, i18n_keys: str | Iterable[str], ignore_attributes: Iterable[str], + default_ftl_file: str, ) -> dict[str, FluentKey]: """ Extract all `FluentKey`s from given path. @@ -115,6 +118,8 @@ def extract_fluent_keys( :type i18n_keys: str | Sequence[str] :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. :type ignore_attributes: Sequence[str] + :param default_ftl_file: Default name of FTL file. + :type default_ftl_file: str :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] @@ -123,7 +128,7 @@ def extract_fluent_keys( for file in find_py_files(path): keys = parse_file(path=file, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes) - post_process_fluent_keys(keys) + post_process_fluent_keys(keys, default_ftl_file) find_conflicts(fluent_keys, keys) fluent_keys.update(keys) diff --git a/src/ftl_extract/const.py b/src/ftl_extract/const.py index 1120bcb..4479f7d 100644 --- a/src/ftl_extract/const.py +++ b/src/ftl_extract/const.py @@ -1,10 +1,15 @@ from __future__ import annotations -from typing import Literal +from typing import TYPE_CHECKING -I18N_LITERAL: Literal["i18n"] = "i18n" -GET_LITERAL: Literal["get"] = "get" -PATH_LITERAL: Literal["_path"] = "_path" -IGNORE_ATTRIBUTES: frozenset[str] = frozenset( - {"set_locale", "use_locale", "use_context", "set_context"} +if TYPE_CHECKING: + from typing import Final, Literal + + +I18N_LITERAL: Final[Literal["i18n"]] = "i18n" +GET_LITERAL: Final[Literal["get"]] = "get" +PATH_LITERAL: Final[Literal["_path"]] = "_path" +IGNORE_ATTRIBUTES: Final[frozenset[str]] = frozenset( + {"set_locale", "use_locale", "use_context", "set_context"}, ) +DEFAULT_FTL_FILE: Final[str] = "_default.ftl" diff --git a/src/ftl_extract/exceptions.py b/src/ftl_extract/exceptions.py index 189d20c..a12684b 100644 --- a/src/ftl_extract/exceptions.py +++ b/src/ftl_extract/exceptions.py @@ -18,7 +18,7 @@ def __init__(self, key: str, current_path: Path, new_path: Path) -> None: self.new_path = new_path super().__init__( f"Key {key!r} already exists with different path: " - f"{self.current_path} != {self.new_path}" + f"{self.current_path} != {self.new_path}", ) @@ -33,5 +33,5 @@ def __init__( self.new_translation = new_translation super().__init__( f"Translation {key!r} already exists with different elements: " - f"{self.current_translation} != {self.new_translation}" + f"{self.current_translation} != {self.new_translation}", ) diff --git a/src/ftl_extract/ftl_extractor.py b/src/ftl_extract/ftl_extractor.py index c525aa0..c9ecd49 100644 --- a/src/ftl_extract/ftl_extractor.py +++ b/src/ftl_extract/ftl_extractor.py @@ -8,7 +8,7 @@ from ftl_extract import extract_fluent_keys from ftl_extract.code_extractor import sort_fluent_keys_by_path -from ftl_extract.const import IGNORE_ATTRIBUTES +from ftl_extract.const import DEFAULT_FTL_FILE, IGNORE_ATTRIBUTES from ftl_extract.ftl_importer import import_ftl_from_dir from ftl_extract.process.commentator import comment_ftl_key from ftl_extract.process.kwargs_extractor import extract_kwargs @@ -29,6 +29,7 @@ def extract( ignore_attributes: Iterable[str] = IGNORE_ATTRIBUTES, expand_ignore_attributes: Iterable[str] | None = None, comment_junks: bool = False, + default_ftl_file: str = DEFAULT_FTL_FILE, serializer: FluentSerializer | None = None, ) -> None: if expand_ignore_attributes is not None: @@ -42,6 +43,7 @@ def extract( path=code_path, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes, + default_ftl_file=default_ftl_file, ) for lang in language: @@ -104,12 +106,15 @@ def extract( for fluent_key in leave_as_is: leave_as_is_with_path.setdefault( - fluent_key.path.relative_to(output_path / lang), [] + fluent_key.path.relative_to(output_path / lang), + [], ).append(fluent_key) for path, keys in sorted_fluent_keys.items(): ftl, _ = generate_ftl( - keys, serializer=serializer, leave_as_is=leave_as_is_with_path.get(path, []) + keys, + serializer=serializer, + leave_as_is=leave_as_is_with_path.get(path, []), ) (output_path / lang / path).parent.mkdir(parents=True, exist_ok=True) (output_path / lang / path).write_text(ftl, encoding="utf-8") diff --git a/src/ftl_extract/ftl_importer.py b/src/ftl_extract/ftl_importer.py index 056ec90..cbe8008 100644 --- a/src/ftl_extract/ftl_importer.py +++ b/src/ftl_extract/ftl_importer.py @@ -12,7 +12,8 @@ def import_from_ftl( - path: Path, locale: str + path: Path, + locale: str, ) -> tuple[dict[str, FluentKey], Resource, list[FluentKey]]: """Import `FluentKey`s from FTL.""" ftl_keys = {} @@ -39,7 +40,7 @@ def import_from_ftl( path=path, locale=locale, position=position, - ) + ), ) return ftl_keys, resource, leave_as_is diff --git a/src/ftl_extract/matcher.py b/src/ftl_extract/matcher.py index 315d9d0..60ea990 100644 --- a/src/ftl_extract/matcher.py +++ b/src/ftl_extract/matcher.py @@ -145,14 +145,15 @@ def create_fluent_key( for kw in keywords: if kw.arg == PATH_LITERAL: if kw.value is not None and isinstance(kw.value, ast.Constant): - fluent_key.path = Path(kw.value.value) + fluent_key.path = Path(cast(ast.Constant, kw.value).value) elif isinstance(kw.arg, str): cast( - fluent_ast.Pattern, cast(fluent_ast.Message, fluent_key.translation).value + fluent_ast.Pattern, + cast(fluent_ast.Message, fluent_key.translation).value, ).elements.append( fluent_ast.Placeable( - expression=fluent_ast.VariableReference(id=fluent_ast.Identifier(name=kw.arg)) - ) + expression=fluent_ast.VariableReference(id=fluent_ast.Identifier(name=kw.arg)), + ), ) return fluent_key diff --git a/src/ftl_extract/process/kwargs_extractor.py b/src/ftl_extract/process/kwargs_extractor.py index 61e1a0f..e2c835e 100644 --- a/src/ftl_extract/process/kwargs_extractor.py +++ b/src/ftl_extract/process/kwargs_extractor.py @@ -1,6 +1,11 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + from fluent.syntax import ast -from ftl_extract.matcher import FluentKey +if TYPE_CHECKING: + from ftl_extract.matcher import FluentKey def _extract_kwargs_from_variable_reference( diff --git a/tests/test_extract/__init__.py b/tests/test_extract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_extract/test_cli_extractor.py b/tests/test_extract/test_cli_extractor.py index e2e14cd..3bca553 100644 --- a/tests/test_extract/test_cli_extractor.py +++ b/tests/test_extract/test_cli_extractor.py @@ -79,7 +79,8 @@ def test_extract_with_keys_to_comment_and_add( with ( patch( - "ftl_extract.ftl_extractor.extract_fluent_keys", return_value={"key-1": mock_fluent_key} + "ftl_extract.ftl_extractor.extract_fluent_keys", + return_value={"key-1": mock_fluent_key}, ), patch( "ftl_extract.ftl_extractor.import_ftl_from_dir", @@ -87,7 +88,8 @@ def test_extract_with_keys_to_comment_and_add( ), patch("ftl_extract.ftl_extractor.comment_ftl_key") as mock_comment_ftl_key, patch( - "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) + "ftl_extract.ftl_extractor.generate_ftl", + return_value=("generated ftl", None), ) as mock_generate_ftl, ): extract(code_path, output_path, ("en",), ("i18n",)) @@ -106,14 +108,16 @@ def test_extract_with_keys_only_to_add( with ( patch( - "ftl_extract.ftl_extractor.extract_fluent_keys", return_value={"key-2": mock_fluent_key} + "ftl_extract.ftl_extractor.extract_fluent_keys", + return_value={"key-2": mock_fluent_key}, ), patch( "ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=({"key-1": mock_fluent_key}, []), ), patch( - "ftl_extract.ftl_extractor.generate_ftl", return_value=("generated ftl", None) + "ftl_extract.ftl_extractor.generate_ftl", + return_value=("generated ftl", None), ) as mock_generate_ftl, ): extract(code_path, output_path, ("en",), ("i18n",)) @@ -130,7 +134,8 @@ def test_extraction_with_valid_paths_succeeds( output_path = tmp_path.joinpath("path/to/output") result = runner.invoke( - cast(BaseCommand, cli_extract), [code_path.as_posix(), output_path.as_posix()] + cast(BaseCommand, cli_extract), + [code_path.as_posix(), output_path.as_posix()], ) assert result.exit_code == 0 assert f"Extracting from {code_path}..." in result.output @@ -246,7 +251,8 @@ def test_stored_fluent_keys_code_path_update(setup_environment: tuple[Path, Path with ( patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value=in_code_fluent_keys), patch( - "ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=(stored_fluent_keys, []) + "ftl_extract.ftl_extractor.import_ftl_from_dir", + return_value=(stored_fluent_keys, []), ), patch("ftl_extract.ftl_extractor.extract_kwargs", return_value=set()), patch("ftl_extract.ftl_extractor.comment_ftl_key"), @@ -278,7 +284,8 @@ def test_keys_to_comment_and_add_on_different_kwargs(setup_environment: tuple[Pa with ( patch("ftl_extract.ftl_extractor.extract_fluent_keys", return_value=in_code_fluent_keys), patch( - "ftl_extract.ftl_extractor.import_ftl_from_dir", return_value=(stored_fluent_keys, []) + "ftl_extract.ftl_extractor.import_ftl_from_dir", + return_value=(stored_fluent_keys, []), ), patch("ftl_extract.ftl_extractor.extract_kwargs", side_effect=[{"arg1"}, {"arg2"}]), patch("ftl_extract.ftl_extractor.comment_ftl_key"), @@ -330,7 +337,9 @@ def test_generic_visit_called_when_attr_in_ignore_attributes( # Create a mock AST node for a function call with an attribute in ignore_attributes node = ast.Call( func=ast.Attribute( - value=ast.Name(id="i18n", ctx=ast.Load()), attr="ignore_this", ctx=ast.Load() + value=ast.Name(id="i18n", ctx=ast.Load()), + attr="ignore_this", + ctx=ast.Load(), ), args=[ast.Constant(value="key")], keywords=[], diff --git a/tests/test_extract/test_common_extract.py b/tests/test_extract/test_common_extract.py index afcba60..803592a 100644 --- a/tests/test_extract/test_common_extract.py +++ b/tests/test_extract/test_common_extract.py @@ -2,7 +2,7 @@ from typing import Final from ftl_extract.code_extractor import extract_fluent_keys -from ftl_extract.const import IGNORE_ATTRIBUTES +from ftl_extract.const import DEFAULT_FTL_FILE, IGNORE_ATTRIBUTES CONTENT: Final[str] = """ def test(i18n): @@ -41,7 +41,12 @@ def test_common_extract(tmp_path: Path) -> None: fluent_keys_len = 24 # Number of keys in `CONTENT`. - fluent_keys = extract_fluent_keys(tmp_path, ("i18n", "L", "LF"), IGNORE_ATTRIBUTES) + fluent_keys = extract_fluent_keys( + tmp_path, + ("i18n", "L", "LF"), + IGNORE_ATTRIBUTES, + default_ftl_file=DEFAULT_FTL_FILE, + ) assert fluent_keys # Check if `fluent_keys` is not empty. assert len(fluent_keys) == fluent_keys_len # Check if `fluent_keys` has `fluent_keys_len` keys. assert "key-1" in fluent_keys @@ -221,5 +226,10 @@ def test_common_extract(tmp_path: Path) -> None: def test_extract_fluent_keys_no_files(tmp_path: Path) -> None: - fluent_keys = extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) + fluent_keys = extract_fluent_keys( + tmp_path, + "i18n", + IGNORE_ATTRIBUTES, + default_ftl_file=DEFAULT_FTL_FILE, + ) assert not fluent_keys diff --git a/tests/test_extract/test_extract_similar_keys.py b/tests/test_extract/test_extract_similar_keys.py index 21d8a29..dd6a714 100644 --- a/tests/test_extract/test_extract_similar_keys.py +++ b/tests/test_extract/test_extract_similar_keys.py @@ -4,7 +4,7 @@ import pytest from ftl_extract.code_extractor import extract_fluent_keys -from ftl_extract.const import IGNORE_ATTRIBUTES +from ftl_extract.const import DEFAULT_FTL_FILE, IGNORE_ATTRIBUTES from ftl_extract.exceptions import ( FTLExtractorDifferentPathsError, FTLExtractorDifferentTranslationError, @@ -52,7 +52,7 @@ def test_extract_similar_keys_in_different_paths_from_one_py_file(tmp_path: Path (tmp_path / "test.py").write_text(CONTENT_1) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES, default_ftl_file=DEFAULT_FTL_FILE) def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( @@ -63,14 +63,14 @@ def test_extract_similar_fluent_keys_in_different_paths_from_different_py_files( (tmp_path / "test2.py").write_text(CONTENT_2_2) with pytest.raises(FTLExtractorDifferentPathsError): - extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES, default_ftl_file=DEFAULT_FTL_FILE) def test_extract_similar_fluent_keys_with_different_translation_one_py_file(tmp_path: Path) -> None: (tmp_path / "test.py").write_text(CONTENT_3) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES, default_ftl_file=DEFAULT_FTL_FILE) def test_extract_similar_fluent_keys_with_different_translation_different_py_files( @@ -81,4 +81,4 @@ def test_extract_similar_fluent_keys_with_different_translation_different_py_fil (tmp_path / "test2.py").write_text(CONTENT_4_2) with pytest.raises(FTLExtractorDifferentTranslationError): - extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES) + extract_fluent_keys(tmp_path, "i18n", IGNORE_ATTRIBUTES, default_ftl_file=DEFAULT_FTL_FILE) diff --git a/tests/test_kwargs_extractor.py b/tests/test_kwargs_extractor.py index ed3a7fa..48269d6 100644 --- a/tests/test_kwargs_extractor.py +++ b/tests/test_kwargs_extractor.py @@ -17,12 +17,12 @@ def extracts_variable_names_from_simple_variable() -> None: elements=[ ast.TextElement("Hello, "), ast.Placeable( - expression=ast.VariableReference(id=ast.Identifier("username")) + expression=ast.VariableReference(id=ast.Identifier("username")), ), - ] + ], ), ), - ) + ), ) assert kwargs == {"username"} @@ -51,12 +51,12 @@ def extracts_variable_names_from_select_expression() -> None: default=False, ), ], - ) - ) - ] + ), + ), + ], ), ), - ) + ), ) assert kwargs == {"gender"} @@ -70,7 +70,7 @@ def returns_empty_set_for_messages_without_variables() -> None: id=ast.Identifier("no_variables"), value=ast.Pattern(elements=[ast.TextElement("Just a text message.")]), ), - ) + ), ) assert kwargs == set() @@ -81,7 +81,7 @@ def test_returns_empty_set_for_comment_translation() -> None: code_path=Path("test.py"), key="key-1", translation=ast.Comment(content="This is a comment"), - ) + ), ) assert kwargs == set() @@ -95,7 +95,7 @@ def test_returns_empty_set_for_translation_without_value() -> None: id=ast.Identifier("message_no_value"), value=None, # Explicitly setting value to None ), - ) + ), ) assert kwargs == set() @@ -111,16 +111,16 @@ def test_extracts_variable_names_from_mixed_elements() -> None: elements=[ ast.TextElement("Hello, "), ast.Placeable( - expression=ast.VariableReference(id=ast.Identifier("username")) + expression=ast.VariableReference(id=ast.Identifier("username")), ), ast.TextElement(" your balance is "), ast.Placeable( - expression=ast.VariableReference(id=ast.Identifier("balance")) + expression=ast.VariableReference(id=ast.Identifier("balance")), ), - ] + ], ), ), - ) + ), ) assert kwargs == {"username", "balance"} @@ -141,24 +141,24 @@ def test_extracts_selector_variable_name_from_select_expression() -> None: ast.Variant( key=ast.Identifier("active"), value=ast.Pattern( - elements=[ast.TextElement("Active User")] + elements=[ast.TextElement("Active User")], ), default=False, ), ast.Variant( key=ast.Identifier("inactive"), value=ast.Pattern( - elements=[ast.TextElement("Inactive User")] + elements=[ast.TextElement("Inactive User")], ), default=False, ), ], - ) - ) - ] + ), + ), + ], ), ), - ) + ), ) assert kwargs == {"user_status"} @@ -176,7 +176,7 @@ def test_nested_extraction() -> None: ast.Placeable( expression=ast.SelectExpression( selector=ast.VariableReference( - id=ast.Identifier(name="first_level_key") + id=ast.Identifier(name="first_level_key"), ), variants=[ ast.Variant( @@ -188,24 +188,24 @@ def test_nested_extraction() -> None: expression=ast.SelectExpression( selector=ast.VariableReference( id=ast.Identifier( - name="second_level_key" - ) + name="second_level_key", + ), ), variants=[ ast.Variant( key=ast.NumberLiteral(value="1"), value=ast.Pattern( elements=[ - ast.TextElement(value="OK") - ] + ast.TextElement(value="OK"), + ], ), ), ast.Variant( key=ast.NumberLiteral(value="2"), value=ast.Pattern( elements=[ - ast.TextElement(value="NO") - ] + ast.TextElement(value="NO"), + ], ), ), ast.Variant( @@ -213,16 +213,16 @@ def test_nested_extraction() -> None: value=ast.Pattern( elements=[ ast.TextElement( - value="ANOTHER" - ) - ] + value="ANOTHER", + ), + ], ), default=True, ), ], - ) + ), ), - ] + ], ), ), ast.Variant( @@ -234,24 +234,24 @@ def test_nested_extraction() -> None: expression=ast.SelectExpression( selector=ast.VariableReference( id=ast.Identifier( - name="second_level_key" - ) + name="second_level_key", + ), ), variants=[ ast.Variant( key=ast.NumberLiteral(value="1"), value=ast.Pattern( elements=[ - ast.TextElement(value="OK") - ] + ast.TextElement(value="OK"), + ], ), ), ast.Variant( key=ast.NumberLiteral(value="2"), value=ast.Pattern( elements=[ - ast.TextElement(value="NO") - ] + ast.TextElement(value="NO"), + ], ), ), ast.Variant( @@ -259,16 +259,16 @@ def test_nested_extraction() -> None: value=ast.Pattern( elements=[ ast.TextElement( - value="ANOTHER" - ) - ] + value="ANOTHER", + ), + ], ), default=True, ), ], - ) + ), ), - ] + ], ), ), ast.Variant( @@ -278,23 +278,23 @@ def test_nested_extraction() -> None: ast.TextElement(value="⏳ "), ast.Placeable( expression=ast.VariableReference( - id=ast.Identifier(name="second_level_key") - ) + id=ast.Identifier(name="second_level_key"), + ), ), ast.TextElement(value=" ANOTHER"), - ] + ], ), default=True, ), ], - ) + ), ), - ] + ], ), ), locale="en", position=0, - ) + ), ) assert kwargs == {"first_level_key", "second_level_key"} diff --git a/tests/test_post_process_fluent_keys.py b/tests/test_post_process_fluent_keys.py index ef26b67..1e656d9 100644 --- a/tests/test_post_process_fluent_keys.py +++ b/tests/test_post_process_fluent_keys.py @@ -2,6 +2,7 @@ from unittest.mock import Mock from ftl_extract.code_extractor import post_process_fluent_keys +from ftl_extract.const import DEFAULT_FTL_FILE from ftl_extract.matcher import FluentKey @@ -10,7 +11,7 @@ def test_process_fluent_key() -> None: fluent_mock.path = "test.ftl" fluent_keys = {"key-1": fluent_mock} - post_process_fluent_keys(fluent_keys) + post_process_fluent_keys(fluent_keys, default_ftl_file=DEFAULT_FTL_FILE) assert fluent_mock.path == Path("test.ftl") @@ -19,5 +20,5 @@ def test_process_fluent_key_default() -> None: fluent_mock.path = Path("test") fluent_keys = {"key-1": fluent_mock} - post_process_fluent_keys(fluent_keys) + post_process_fluent_keys(fluent_keys, default_ftl_file=DEFAULT_FTL_FILE) assert fluent_mock.path == Path("test/_default.ftl") diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 4d97d15..3b53f41 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -17,7 +17,7 @@ def single_fluent_key() -> list[FluentKey]: id=ast.Identifier("greeting"), value=ast.Pattern(elements=[ast.TextElement("Hello, world!")]), ), - ) + ), ] @@ -95,7 +95,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: id=ast.Identifier("test_message"), value=ast.Pattern(elements=[ast.TextElement("Test message content")]), ), - ) + ), ], serializer=FluentSerializer(with_junk=True), leave_as_is=[ diff --git a/uv.lock b/uv.lock index c4a96e9..5de5169 100644 --- a/uv.lock +++ b/uv.lock @@ -278,7 +278,7 @@ wheels = [ [[package]] name = "ftl-extract" -version = "0.4.3" +version = "0.4.4" source = { editable = "." } dependencies = [ { name = "click" }, From b50edc9d2f8ac63700627d0e544b64043ef723df Mon Sep 17 00:00:00 2001 From: andrew000 <11490628+andrew000@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:00:06 +0200 Subject: [PATCH 2/3] Change input type of `default-ftl-file` --- src/ftl_extract/cli.py | 3 ++- src/ftl_extract/code_extractor.py | 4 ++-- src/ftl_extract/const.py | 3 ++- src/ftl_extract/ftl_extractor.py | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ftl_extract/cli.py b/src/ftl_extract/cli.py index 870d921..08b0b93 100644 --- a/src/ftl_extract/cli.py +++ b/src/ftl_extract/cli.py @@ -52,6 +52,7 @@ "--default-ftl-file", default=DEFAULT_FTL_FILE, show_default=True, + type=click.Path(path_type=Path), ) @click.version_option() def cli_extract( @@ -62,7 +63,7 @@ def cli_extract( ignore_attributes: tuple[str, ...], expand_ignore_attributes: tuple[str, ...] | None = None, comment_junks: bool = False, - default_ftl_file: str = DEFAULT_FTL_FILE, + default_ftl_file: Path = DEFAULT_FTL_FILE, ) -> None: click.echo(f"Extracting from {code_path}...") start_time = perf_counter_ns() diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index 8f9edf1..4dd2079 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -53,7 +53,7 @@ def parse_file( return matcher.fluent_keys -def post_process_fluent_keys(fluent_keys: dict[str, FluentKey], default_ftl_file: str) -> None: +def post_process_fluent_keys(fluent_keys: dict[str, FluentKey], default_ftl_file: Path) -> None: """ Third step: post-process parsed `FluentKey`. @@ -107,7 +107,7 @@ def extract_fluent_keys( path: Path, i18n_keys: str | Iterable[str], ignore_attributes: Iterable[str], - default_ftl_file: str, + default_ftl_file: Path, ) -> dict[str, FluentKey]: """ Extract all `FluentKey`s from given path. diff --git a/src/ftl_extract/const.py b/src/ftl_extract/const.py index 4479f7d..e974860 100644 --- a/src/ftl_extract/const.py +++ b/src/ftl_extract/const.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -12,4 +13,4 @@ IGNORE_ATTRIBUTES: Final[frozenset[str]] = frozenset( {"set_locale", "use_locale", "use_context", "set_context"}, ) -DEFAULT_FTL_FILE: Final[str] = "_default.ftl" +DEFAULT_FTL_FILE: Final[Path] = Path("_default.ftl") diff --git a/src/ftl_extract/ftl_extractor.py b/src/ftl_extract/ftl_extractor.py index c9ecd49..c5afda2 100644 --- a/src/ftl_extract/ftl_extractor.py +++ b/src/ftl_extract/ftl_extractor.py @@ -29,7 +29,7 @@ def extract( ignore_attributes: Iterable[str] = IGNORE_ATTRIBUTES, expand_ignore_attributes: Iterable[str] | None = None, comment_junks: bool = False, - default_ftl_file: str = DEFAULT_FTL_FILE, + default_ftl_file: Path = DEFAULT_FTL_FILE, serializer: FluentSerializer | None = None, ) -> None: if expand_ignore_attributes is not None: From 1b34030a8baac00165652e254a26d3040567589e Mon Sep 17 00:00:00 2001 From: andrew000 <11490628+andrew000@users.noreply.github.com> Date: Sun, 27 Oct 2024 23:16:36 +0200 Subject: [PATCH 3/3] Remove default field `path` of `FluentKey` --- pyproject.toml | 1 - src/ftl_extract/code_extractor.py | 21 +++++++++++++++++---- src/ftl_extract/matcher.py | 9 ++++++++- tests/test_extract/test_cli_extractor.py | 14 ++++++++++---- tests/test_kwargs_extractor.py | 9 +++++++++ tests/test_serializer.py | 10 ++++++++++ 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae926f4..4468dfc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,6 @@ pretty = true show_absolute_path = true show_error_codes = true show_error_context = true -warn_redundant_casts = true warn_unused_configs = true warn_unused_ignores = true disable_error_code = [ diff --git a/src/ftl_extract/code_extractor.py b/src/ftl_extract/code_extractor.py index 4dd2079..6975fa8 100644 --- a/src/ftl_extract/code_extractor.py +++ b/src/ftl_extract/code_extractor.py @@ -34,6 +34,7 @@ def parse_file( path: Path, i18n_keys: str | Iterable[str], ignore_attributes: Iterable[str], + default_ftl_file: Path, ) -> dict[str, FluentKey]: """ Second step: parse given .py file and find all i18n calls. @@ -44,11 +45,18 @@ def parse_file( :type i18n_keys: str | Sequence[str] :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. :type ignore_attributes: Sequence[str] + :param default_ftl_file: Default name of FTL file. + :type default_ftl_file: Path :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] """ node = ast.parse(path.read_bytes()) - matcher = I18nMatcher(code_path=path, func_names=i18n_keys, ignore_attributes=ignore_attributes) + matcher = I18nMatcher( + code_path=path, + default_ftl_file=default_ftl_file, + func_names=i18n_keys, + ignore_attributes=ignore_attributes, + ) matcher.visit(node) return matcher.fluent_keys @@ -60,7 +68,7 @@ def post_process_fluent_keys(fluent_keys: dict[str, FluentKey], default_ftl_file :param fluent_keys: Dict with `key` and `FluentKey` that will be post-processed. :type fluent_keys: dict[str, FluentKey] :param default_ftl_file: Default name of FTL file. - :type default_ftl_file: str + :type default_ftl_file: Path """ for fluent_key in fluent_keys.values(): if not isinstance(fluent_key.path, Path): @@ -119,7 +127,7 @@ def extract_fluent_keys( :param ignore_attributes: Ignore attributes, like `i18n.set_locale`. :type ignore_attributes: Sequence[str] :param default_ftl_file: Default name of FTL file. - :type default_ftl_file: str + :type default_ftl_file: Path :return: Dict with `key` and `FluentKey`. :rtype: dict[str, FluentKey] @@ -127,7 +135,12 @@ def extract_fluent_keys( fluent_keys: dict[str, FluentKey] = {} for file in find_py_files(path): - keys = parse_file(path=file, i18n_keys=i18n_keys, ignore_attributes=ignore_attributes) + keys = parse_file( + path=file, + i18n_keys=i18n_keys, + ignore_attributes=ignore_attributes, + default_ftl_file=default_ftl_file, + ) post_process_fluent_keys(keys, default_ftl_file) find_conflicts(fluent_keys, keys) fluent_keys.update(keys) diff --git a/src/ftl_extract/matcher.py b/src/ftl_extract/matcher.py index 60ea990..73afdfc 100644 --- a/src/ftl_extract/matcher.py +++ b/src/ftl_extract/matcher.py @@ -38,7 +38,7 @@ class FluentKey: code_path: Path key: str translation: fluent_ast.EntryType - path: Path = field(default=Path("_default.ftl")) + path: Path locale: str | None = field(default=None) position: int | float = field(default=inf) @@ -47,6 +47,7 @@ class I18nMatcher(ast.NodeVisitor): def __init__( self, code_path: Path, + default_ftl_file: Path, func_names: str | Iterable[str] = I18N_LITERAL, ignore_attributes: str | Iterable[str] = IGNORE_ATTRIBUTES, ) -> None: @@ -66,6 +67,7 @@ def __init__( if isinstance(ignore_attributes, str) else frozenset(ignore_attributes) ) + self.default_ftl_file = default_ftl_file self.fluent_keys: dict[str, FluentKey] = {} def visit_Call(self, node: ast.Call) -> None: # noqa: N802 @@ -96,6 +98,7 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 code_path=self.code_path, key=key, keywords=node.keywords, + default_ftl_file=self.default_ftl_file, ) process_fluent_key(self.fluent_keys, fluent_key) @@ -109,6 +112,7 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 code_path=self.code_path, key="-".join(reversed(attrs)), keywords=node.keywords, + default_ftl_file=self.default_ftl_file, ) process_fluent_key(self.fluent_keys, fluent_key) else: @@ -122,6 +126,7 @@ def visit_Call(self, node: ast.Call) -> None: # noqa: N802 code_path=self.code_path, key=cast(ast.Constant, node.args[0]).value, keywords=node.keywords, + default_ftl_file=self.default_ftl_file, ) process_fluent_key(self.fluent_keys, fluent_key) @@ -132,6 +137,7 @@ def create_fluent_key( code_path: Path, key: str, keywords: list[ast.keyword], + default_ftl_file: Path, ) -> FluentKey: fluent_key = FluentKey( code_path=code_path, @@ -140,6 +146,7 @@ def create_fluent_key( id=fluent_ast.Identifier(name=key), value=fluent_ast.Pattern(elements=[fluent_ast.TextElement(value=key)]), ), + path=default_ftl_file, ) for kw in keywords: diff --git a/tests/test_extract/test_cli_extractor.py b/tests/test_extract/test_cli_extractor.py index 3bca553..f669858 100644 --- a/tests/test_extract/test_cli_extractor.py +++ b/tests/test_extract/test_cli_extractor.py @@ -10,6 +10,7 @@ from fluent.syntax import ast as fl_ast from ftl_extract.cli import cli_extract +from ftl_extract.const import DEFAULT_FTL_FILE from ftl_extract.ftl_extractor import extract from ftl_extract.matcher import FluentKey, I18nMatcher @@ -187,6 +188,7 @@ def test_comment_junk_elements_if_needed(setup_environment: tuple[Path, Path]) - mock_junk_key = MagicMock(spec=FluentKey) mock_junk_key.translation = MagicMock(spec=fl_ast.Junk) + mock_junk_key.path = MagicMock(spec=Path) mock_serializer = MagicMock(spec=FluentSerializer) with ( @@ -305,7 +307,7 @@ def test_keys_to_comment_and_add_on_different_kwargs(setup_environment: tuple[Pa def test_i18n_matcher_skips_call_with_no_args(setup_environment: tuple[Path, Path]) -> None: code_path, output_path = setup_environment - matcher = I18nMatcher(code_path) + matcher = I18nMatcher(code_path, default_ftl_file=DEFAULT_FTL_FILE) node = ast.Call(func=ast.Attribute(value=ast.Name(id="i18n"), attr="get"), args=[], keywords=[]) matcher.visit_Call(node) @@ -315,7 +317,7 @@ def test_i18n_matcher_skips_call_with_no_args(setup_environment: tuple[Path, Pat def test_generic_visit_called_on_else_block(setup_environment: tuple[Path, Path]) -> None: code_path, output_path = setup_environment - matcher = I18nMatcher(code_path) + matcher = I18nMatcher(code_path, default_ftl_file=DEFAULT_FTL_FILE) node = ast.Call( func=ast.Attribute(value=ast.Name(id="i18n"), attr="get"), @@ -332,7 +334,11 @@ def test_generic_visit_called_when_attr_in_ignore_attributes( setup_environment: tuple[Path, Path], ) -> None: code_path, output_path = setup_environment - matcher = I18nMatcher(code_path, ignore_attributes={"ignore_this"}) + matcher = I18nMatcher( + code_path, + default_ftl_file=DEFAULT_FTL_FILE, + ignore_attributes={"ignore_this"}, + ) # Create a mock AST node for a function call with an attribute in ignore_attributes node = ast.Call( @@ -354,7 +360,7 @@ def test_generic_visit_called_when_attr_in_ignore_attributes( def test_i18n_matcher_skips_call_with_no_args_in_elif(setup_environment: tuple[Path, Path]) -> None: code_path, output_path = setup_environment - matcher = I18nMatcher(code_path) + matcher = I18nMatcher(code_path, default_ftl_file=DEFAULT_FTL_FILE) node = ast.Call(func=ast.Name(id="i18n", ctx=ast.Load()), args=[], keywords=[]) matcher.visit_Call(node) diff --git a/tests/test_kwargs_extractor.py b/tests/test_kwargs_extractor.py index 48269d6..4cb4952 100644 --- a/tests/test_kwargs_extractor.py +++ b/tests/test_kwargs_extractor.py @@ -2,6 +2,7 @@ from fluent.syntax import ast +from ftl_extract.const import DEFAULT_FTL_FILE from ftl_extract.matcher import FluentKey from ftl_extract.process.kwargs_extractor import extract_kwargs @@ -22,6 +23,7 @@ def extracts_variable_names_from_simple_variable() -> None: ], ), ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == {"username"} @@ -56,6 +58,7 @@ def extracts_variable_names_from_select_expression() -> None: ], ), ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == {"gender"} @@ -70,6 +73,7 @@ def returns_empty_set_for_messages_without_variables() -> None: id=ast.Identifier("no_variables"), value=ast.Pattern(elements=[ast.TextElement("Just a text message.")]), ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == set() @@ -81,6 +85,7 @@ def test_returns_empty_set_for_comment_translation() -> None: code_path=Path("test.py"), key="key-1", translation=ast.Comment(content="This is a comment"), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == set() @@ -95,6 +100,7 @@ def test_returns_empty_set_for_translation_without_value() -> None: id=ast.Identifier("message_no_value"), value=None, # Explicitly setting value to None ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == set() @@ -120,6 +126,7 @@ def test_extracts_variable_names_from_mixed_elements() -> None: ], ), ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == {"username", "balance"} @@ -158,6 +165,7 @@ def test_extracts_selector_variable_name_from_select_expression() -> None: ], ), ), + path=DEFAULT_FTL_FILE, ), ) assert kwargs == {"user_status"} @@ -292,6 +300,7 @@ def test_nested_extraction() -> None: ], ), ), + path=DEFAULT_FTL_FILE, locale="en", position=0, ), diff --git a/tests/test_serializer.py b/tests/test_serializer.py index 3b53f41..4eba00a 100644 --- a/tests/test_serializer.py +++ b/tests/test_serializer.py @@ -3,6 +3,7 @@ import pytest from fluent.syntax import FluentSerializer, ast +from ftl_extract.const import DEFAULT_FTL_FILE from ftl_extract.matcher import FluentKey from ftl_extract.process.serializer import generate_ftl @@ -17,6 +18,7 @@ def single_fluent_key() -> list[FluentKey]: id=ast.Identifier("greeting"), value=ast.Pattern(elements=[ast.TextElement("Hello, world!")]), ), + path=DEFAULT_FTL_FILE, ), ] @@ -31,6 +33,7 @@ def multiple_fluent_keys() -> list[FluentKey]: id=ast.Identifier("greeting"), value=ast.Pattern(elements=[ast.TextElement("Hello, world!")]), ), + path=DEFAULT_FTL_FILE, ), FluentKey( code_path=Path("test.py"), @@ -39,6 +42,7 @@ def multiple_fluent_keys() -> list[FluentKey]: id=ast.Identifier("farewell"), value=ast.Pattern(elements=[ast.TextElement("Goodbye, world!")]), ), + path=DEFAULT_FTL_FILE, ), ] @@ -95,6 +99,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: id=ast.Identifier("test_message"), value=ast.Pattern(elements=[ast.TextElement("Test message content")]), ), + path=DEFAULT_FTL_FILE, ), ], serializer=FluentSerializer(with_junk=True), @@ -103,16 +108,19 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: code_path=Path(), key="", translation=ast.Comment(content="This is a comment"), + path=DEFAULT_FTL_FILE, ), FluentKey( code_path=Path(), key="", translation=ast.GroupComment(content="This is a group comment"), + path=DEFAULT_FTL_FILE, ), FluentKey( code_path=Path(), key="", translation=ast.ResourceComment(content="This is a resource comment"), + path=DEFAULT_FTL_FILE, ), FluentKey( code_path=Path(), @@ -121,6 +129,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: id=ast.Identifier("test_term"), value=ast.Pattern(elements=[ast.TextElement("Test term content")]), ), + path=DEFAULT_FTL_FILE, ), FluentKey( code_path=Path(), @@ -128,6 +137,7 @@ def test_generate_ftl_includes_leave_as_is_elements() -> None: translation=ast.Junk( content="This is junk content", ), + path=DEFAULT_FTL_FILE, ), ], )