From 726ec39d8eb46cfa848a2f39a6e827da6301f339 Mon Sep 17 00:00:00 2001 From: Michael Sasser Date: Mon, 11 Dec 2023 17:03:18 +0100 Subject: [PATCH] fix: make linters happy --- .pre-commit-config.yaml | 14 ++--- matrixctl/addon_manager.py | 2 +- matrixctl/handlers/api.py | 116 ++++++++++++++++++++++--------------- matrixctl/handlers/ssh.py | 2 +- matrixctl/handlers/yaml.py | 4 +- pyproject.toml | 11 +++- 6 files changed, 90 insertions(+), 59 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 598a37fc..23835feb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,14 +10,14 @@ repos: # ruff - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.272' + rev: 'v0.1.7' hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] # pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: # Prevent giant files from being committed. - id: check-added-large-files @@ -126,7 +126,7 @@ repos: # format: black - repo: https://github.com/ambv/black - rev: 23.3.0 + rev: 23.11.0 hooks: - id: black @@ -142,7 +142,7 @@ repos: # static type checking with mypy - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.7.1 hooks: - id: mypy additional_dependencies: @@ -162,20 +162,20 @@ repos: # vulture - repo: https://github.com/jendrikseipp/vulture - rev: "v2.7" + rev: "v2.10" hooks: - id: vulture # pyupgrade - repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 + rev: v3.15.0 hooks: - id: pyupgrade exclude: docs/source/conf.py # pycln - repo: https://github.com/hadialqattan/pycln - rev: v2.1.5 + rev: v2.4.0 hooks: - id: pycln args: [--config=pyproject.toml] diff --git a/matrixctl/addon_manager.py b/matrixctl/addon_manager.py index 9ac49c88..e9c24dd3 100644 --- a/matrixctl/addon_manager.py +++ b/matrixctl/addon_manager.py @@ -38,7 +38,7 @@ # skipcq: PYL-W0212 # noqa: ERA001 # pyright: reportPrivateUsage=false SubParsersAction = argparse._SubParsersAction # noqa: SLF001 -SubParserType = Callable[[SubParsersAction], None] +SubParserType = Callable[[SubParsersAction], None] # type: ignore # noqa: PGH003 ParserSetupType = Callable[[], argparse.ArgumentParser] # global diff --git a/matrixctl/handlers/api.py b/matrixctl/handlers/api.py index 995dcdb0..fe3cc701 100644 --- a/matrixctl/handlers/api.py +++ b/matrixctl/handlers/api.py @@ -25,6 +25,7 @@ import typing as t import urllib.parse + from collections.abc import Generator from collections.abc import Iterable from contextlib import suppress @@ -41,6 +42,9 @@ __author__: str = "Michael Sasser" __email__: str = "Michael@MichaelSasser.org" +HTTP_RETURN_CODE_302: int = 302 +HTTP_RETURN_CODE_404: int = 404 + logger = logging.getLogger(__name__) @@ -48,12 +52,11 @@ InputQueueType = asyncio.Queue[tuple[int, "RequestBuilder", httpx.AsyncClient]] OutputQueueType = asyncio.Queue[ - t.Union[tuple[int, httpx.Response], tuple[int, Exception]] + tuple[int, httpx.Response] | tuple[int, Exception] ] class RequestStrategy(t.NamedTuple): - """Use this NamedTuple as request strategy data. This NamedTuple is only used in this module. @@ -69,7 +72,6 @@ class RequestStrategy(t.NamedTuple): @attr.s(slots=True, auto_attribs=True, repr=False) class RequestBuilder: - """Build the URL for an API request.""" token: str = attr.ib() @@ -79,12 +81,13 @@ class RequestBuilder: subdomain: str = "matrix" api_path: str = "_synapse/admin" api_version: str = "v2" - data: t.Optional[dict[str, t.Any]] = None # just key/value store - json: t.Optional[dict[str, t.Any]] = None # json - content: t.Optional[t.Union[str, bytes, Iterable[bytes]]] = None # bytes + data: dict[str, t.Any] | None = None # just key/value store + json: dict[str, t.Any] | None = None # json + content: str | bytes | Iterable[bytes] | None = None # bytes method: str = "GET" - params: dict[str, t.Union[str, int]] = {} - headers: dict[str, str] = {} # Cannot be none with MatrixCtl + params: dict[str, str | int] = {} # noqa: RUF012 + # Cannot be none with MatrixCtl + headers: dict[str, str] = {} # noqa: RUF012 concurrent_limit: int = 4 timeout: float = 5.0 # seconds success_codes: tuple[int, ...] = ( @@ -171,7 +174,9 @@ def __repr__(self) -> str: def preplan_request_strategy( - limit: int, concurrent_limit: int | float, max_step_size: int = 100 + limit: int, + concurrent_limit: float, + max_step_size: int = 100, ) -> RequestStrategy: """Use this functiona as helper for optimizing asynchronous requests. @@ -231,15 +236,22 @@ def preplan_request_strategy( logger.debug("offset = %s (negative not allowed)", offset) if offset < 0: - raise InternalResponseError("The offset must always be positive.") + msg: str = "The offset must always be positive." + raise InternalResponseError(msg) return RequestStrategy( - new_limit, new_step_size, new_limit, offset, new_iterations + new_limit, + new_step_size, + new_limit, + offset, + new_iterations, ) def generate_worker_configs( - request_config: RequestBuilder, next_token: int, limit: int + request_config: RequestBuilder, + next_token: int, + limit: int, ) -> Generator[RequestBuilder, None, None]: """Create workers for async requests (minus the already done sync request). @@ -271,11 +283,12 @@ def generate_worker_configs( """ if limit - next_token < 0: - raise InternalResponseError( + msg: str = ( f"limit - next_token is negative ({limit - next_token}). " "Make sure that you not use generate_worker_configs() if it " "isn't necessary. For example with total > 100." ) + raise InternalResponseError(msg) strategy: RequestStrategy = preplan_request_strategy( limit - next_token, # minus the already queried concurrent_limit=request_config.concurrent_limit, @@ -296,7 +309,9 @@ def generate_worker_configs( strategy.step_size, ) for i in range( - next_token + 1, strategy.limit + next_token + 1, strategy.step_size + next_token + 1, + strategy.limit + next_token + 1, + strategy.step_size, ): worker_config = deepcopy(request_config) # deepcopy needed worker_config.params["from"] = i @@ -304,7 +319,8 @@ def generate_worker_configs( async def async_worker( - input_queue: InputQueueType, output_queue: OutputQueueType + input_queue: InputQueueType, + output_queue: OutputQueueType, ) -> None: """Use this coro as worker to make (a)synchronous request. @@ -333,7 +349,7 @@ async def async_worker( await output_queue.put((idx, output)) # Capture all exceptions and put them into the output queue - except Exception as err: # skipcq: PYL-W0703 + except Exception as err: # skipcq: PYL-W0703 # noqa: BLE001 await output_queue.put((idx, err)) finally: @@ -341,7 +357,8 @@ async def async_worker( async def group_async_results( - input_size: int, output_queue: OutputQueueType + input_size: int, + output_queue: OutputQueueType, ) -> list[Exception | httpx.Response]: """Use this coro to group the requests afterwards in a single list. @@ -376,16 +393,18 @@ async def group_async_results( async def exec_async_request( request_config: Generator[RequestBuilder, None, None], ) -> list[httpx.Response]: - """Overload for request.""" + ... @t.overload -async def exec_async_request(request_config: RequestBuilder) -> httpx.Response: - """Overload for request.""" +async def exec_async_request( + request_config: RequestBuilder, +) -> httpx.Response: + ... async def exec_async_request( - request_config: RequestBuilder | Generator[RequestBuilder, None, None] + request_config: RequestBuilder | Generator[RequestBuilder, None, None], ) -> httpx.Response | list[httpx.Response]: """Use this coro to generate and run workers and group the responses. @@ -435,7 +454,7 @@ async def exec_async_request( # Generate task pool, and start collecting data. output_queue: OutputQueueType = asyncio.Queue() result_task = asyncio.create_task( - group_async_results(input_size, output_queue) + group_async_results(input_size, output_queue), ) tasks = [ asyncio.create_task(async_worker(input_queue, output_queue)) @@ -455,10 +474,10 @@ async def exec_async_request( errors: list[Exception] if isinstance(results, Iterable): if errors := [err for err in results if isinstance(err, Exception)]: - raise Exception(errors) + raise Exception(errors) # noqa: TRY002 return t.cast(list[httpx.Response], results) if isinstance(results, Exception): # Not concurrent - raise Exception(results) + raise Exception(results) # noqa: TRY004,TRY002 return t.cast(httpx.Response, results) @@ -466,12 +485,14 @@ async def exec_async_request( def request( request_config: Generator[RequestBuilder, None, None], ) -> list[httpx.Response]: - """Overload for request.""" + ... @t.overload -def request(request_config: RequestBuilder) -> httpx.Response: - """Overload for request.""" +def request( + request_config: RequestBuilder, +) -> httpx.Response: + ... # flake8: noqa: C901 @@ -540,16 +561,16 @@ def _request(request_config: RequestBuilder) -> httpx.Response | t.NoReturn: with httpx.Client(http2=True, timeout=request_config.timeout) as client: response: httpx.Response = client.request( method=request_config.method, - data=request_config.data, # type: ignore + data=request_config.data, # type: ignore # noqa: PGH003 json=request_config.json, - content=request_config.content, # type: ignore + content=request_config.content, # type: ignore # noqa: PGH003 url=str(request_config), params=request_config.params, headers=request_config.headers_with_auth, follow_redirects=False, ) - if response.status_code == 302: + if response.status_code == HTTP_RETURN_CODE_302: logger.critical( "The api request resulted in an redirect (302). " "This indicates, that the API might have changed, or your " @@ -557,19 +578,19 @@ def _request(request_config: RequestBuilder) -> httpx.Response | t.NoReturn: "Please make sure your installation of matrixctl is " "up-to-date and your vars.yml contains:\n\n" "matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to" - '_domain: ""' + '_domain: ""', ) sys.exit(1) - if response.status_code == 404: + if response.status_code == HTTP_RETURN_CODE_404: logger.critical( "The server returned an 404 error. This can have multiple causes." " One of them is, you try to request a resource, which does not or" " no longer exist. Another one is, your API endpoint is disabled." " Make sure, that your vars.yml contains the following excessive" " long" - " line:\n\nmatrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled:" - " true" + " line:\n\nmatrix_nginx_proxy_proxy_matrix_client_api_forwarded" + "_location_synapse_admin_api_enabled: true", ) sys.exit(1) @@ -583,7 +604,7 @@ def _request(request_config: RequestBuilder) -> httpx.Response | t.NoReturn: "The server rejected your access-token. " "Please make sure, your access-token is correct " "and up-to-date. Your access-token will change every " - "time, you log out." + "time, you log out.", ) sys.exit(1) raise InternalResponseError(payload=response) @@ -592,7 +613,8 @@ def _request(request_config: RequestBuilder) -> httpx.Response | t.NoReturn: async def _arequest( - request_config: RequestBuilder, client: httpx.AsyncClient + request_config: RequestBuilder, + client: httpx.AsyncClient, ) -> httpx.Response: """Send an asynchronous request to the synapse API and receive a response. @@ -613,9 +635,9 @@ async def _arequest( # There is some weird stuff going on in httpx. It is set to None by default response: httpx.Response = await client.request( method=request_config.method, - data=request_config.data, # type: ignore + data=request_config.data, # type: ignore # noqa: PGH003 json=request_config.json, - content=request_config.content, # type: ignore + content=request_config.content, # type: ignore # noqa: PGH003 url=str(request_config), params=request_config.params, headers=request_config.headers_with_auth, @@ -623,7 +645,7 @@ async def _arequest( follow_redirects=False, ) - if response.status_code == 302: + if response.status_code == HTTP_RETURN_CODE_302: logger.critical( "The api request resulted in an redirect (302). " "This indicates, that the API might have changed, or your " @@ -631,20 +653,20 @@ async def _arequest( "Please make sure your installation of matrixctl is " "up-to-date and your vars.yml contains:\n\n" "matrix_nginx_proxy_proxy_matrix_client_redirect_root_uri_to" - '_domain: ""' + '_domain: ""', ) - raise QWorkerExit() # TODO - if response.status_code == 404: + raise QWorkerExit # TODO + if response.status_code == HTTP_RETURN_CODE_404: logger.critical( "The server returned an 404 error. This can have multiple causes." " One of them is, you try to request a resource, which does not or" " no longer exist. Another one is, your API endpoint is disabled." " Make sure, that your vars.yml contains the following excessive" " long" - " line:\n\nmatrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled:" - " true" + " line:\n\nmatrix_nginx_proxy_proxy_matrix_client_api_forwarded" + "_location_synapse_admin_api_enabled: true", ) - raise QWorkerExit() # TODO + raise QWorkerExit # TODO logger.debug("JSON response: %s", response.json()) @@ -656,9 +678,9 @@ async def _arequest( "The server rejected your access-token. " "Please make sure, your access-token is correct " "and up-to-date. Your access-token will change every " - "time, you log out." + "time, you log out.", ) - raise QWorkerExit() # TODO + raise QWorkerExit # TODO raise InternalResponseError(payload=response) return response diff --git a/matrixctl/handlers/ssh.py b/matrixctl/handlers/ssh.py index 00c47f01..3cd9dd11 100644 --- a/matrixctl/handlers/ssh.py +++ b/matrixctl/handlers/ssh.py @@ -146,7 +146,7 @@ def run_cmd(self: SSH, cmd: str) -> SSHResponse: return response - def __enter__(self: SSH) -> SSH: + def __enter__(self: SSH) -> SSH: # noqa: PYI034 """Connect to the SSH server with the "with" statement. Parameters diff --git a/matrixctl/handlers/yaml.py b/matrixctl/handlers/yaml.py index c7b13f0b..2855a014 100644 --- a/matrixctl/handlers/yaml.py +++ b/matrixctl/handlers/yaml.py @@ -127,11 +127,11 @@ def __getattr__(self: JinjaUndefined, _: str) -> t.Any: class YAML: """Use the YAML class to read and parse the configuration file(s).""" - DEFAULT_PATHS: list[Path] = [ + DEFAULT_PATHS: t.ClassVar[list[Path]] = [ Path("/etc/matrixctl/config"), Path.home() / ".config/matrixctl/config", ] - JINJA_PREDEFINED: dict[str, str | int] = { + JINJA_PREDEFINED: t.ClassVar[dict[str, str | int]] = { "home": str(Path.home()), "user": getuser(), "default_ssh_port": 22, diff --git a/pyproject.toml b/pyproject.toml index f263b50c..666061f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ fix = true show-fixes = true ignore-init-module-imports = true src = ["matrixctl", "docs"] -ignore = ["D202", "TCH003", "TCH001", "ANN401"] +ignore = ["D202", "TCH003", "TCH001", "ANN401", "ANN101"] select = [ "E", # pycodestyle (Errors) "W", # pycodestyle (Warnings) @@ -377,6 +377,8 @@ ignore-magic = true fail-under = 100 exclude = ["setup.py", "docs", "build"] # ignore-regex = ["^get$", "^mock_.*", ".*BaseClass.*"] +# ignore-regex see: https://github.com/econchick/interrogate/issues?q=is%3Aissue+overload +ignore-regex = ["request", "exec_async_request"] verbose = 1 # quiet = false # whitelist-regex = [] @@ -389,6 +391,9 @@ min_confidence = 80 paths = ["matrixctl"] sort_by_size = true +[tool.coverage.report] +exclude_also = ["@t.overload", "pragma: no cover"] + [tool.towncrier] package = "matrixctl" package_dir = "matrixctl" @@ -465,6 +470,10 @@ envlist = precommit,py310,py311,docs,changelog, # py310, isolated_build = True +[coverage:report] +exclude_also = + @t.overload + pragma: no cover [testenv] allowlist_externals = poetry