From 85f54cb8a07560613ab872a820d1d59b1ee52162 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 21:29:21 +0900 Subject: [PATCH 01/10] Update import paths for CapsuleParams in capsula files --- capsula/_capsule.py | 2 +- capsula/_context/_file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/capsula/_capsule.py b/capsula/_capsule.py index 092ef68a..d180dca1 100644 --- a/capsula/_capsule.py +++ b/capsula/_capsule.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - from capsula._decorator import CapsuleParams + from capsula._run import CapsuleParams from capsula.utils import ExceptionInfo from ._backport import Self, TypeAlias diff --git a/capsula/_context/_file.py b/capsula/_context/_file.py index d4083878..5b3a494a 100644 --- a/capsula/_context/_file.py +++ b/capsula/_context/_file.py @@ -11,7 +11,7 @@ from ._base import ContextBase if TYPE_CHECKING: - from capsula._decorator import CapsuleParams + from capsula._run import CapsuleParams logger = logging.getLogger(__name__) From 1dfd53d8c84f0198bc94500ef125649d454e8cc1 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 21:35:13 +0900 Subject: [PATCH 02/10] Update import paths in various capsula modules --- capsula/_context/_git.py | 2 +- capsula/_reporter/_base.py | 2 +- capsula/_reporter/_json.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/capsula/_context/_git.py b/capsula/_context/_git.py index 94ff405e..f8d3becc 100644 --- a/capsula/_context/_git.py +++ b/capsula/_context/_git.py @@ -12,7 +12,7 @@ from ._base import ContextBase if TYPE_CHECKING: - from capsula._decorator import CapsuleParams + from capsula._run import CapsuleParams logger = logging.getLogger(__name__) diff --git a/capsula/_reporter/_base.py b/capsula/_reporter/_base.py index e1926e9d..8dc187a8 100644 --- a/capsula/_reporter/_base.py +++ b/capsula/_reporter/_base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from capsula.encapsulator import Capsule +from capsula._capsule import Capsule class ReporterBase(ABC): diff --git a/capsula/_reporter/_json.py b/capsula/_reporter/_json.py index 84131047..1aecfb5e 100644 --- a/capsula/_reporter/_json.py +++ b/capsula/_reporter/_json.py @@ -12,8 +12,8 @@ from capsula.utils import to_nested_dict if TYPE_CHECKING: - from capsula._decorator import CapsuleParams - from capsula.encapsulator import Capsule + from capsula._capsule import Capsule + from capsula._run import CapsuleParams from ._base import ReporterBase From bd4a9eca97e32ba1b6f4872e51d0944aeecc65f0 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:11:54 +0900 Subject: [PATCH 03/10] improve typing so mypy --strict succeeds --- capsula/_context/_command.py | 12 ++++++++++-- capsula/_context/_cpu.py | 8 ++++++-- capsula/_context/_file.py | 25 +++++++++++++++++-------- capsula/_context/_function.py | 13 ++++++++++--- capsula/_context/_git.py | 15 ++++++++++++--- capsula/_context/_platform.py | 16 +++++++++++++++- capsula/_root.py | 2 +- capsula/_run.py | 17 +++++++++++------ capsula/encapsulator.py | 6 +++--- 9 files changed, 85 insertions(+), 29 deletions(-) diff --git a/capsula/_context/_command.py b/capsula/_context/_command.py index 59c1367b..0a8bc3db 100644 --- a/capsula/_context/_command.py +++ b/capsula/_context/_command.py @@ -2,7 +2,7 @@ import logging import subprocess -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, TypedDict if TYPE_CHECKING: from pathlib import Path @@ -12,13 +12,21 @@ logger = logging.getLogger(__name__) +class _CommandContextData(TypedDict): + command: str + cwd: Path | None + returncode: int + stdout: str + stderr: str + + class CommandContext(ContextBase): def __init__(self, command: str, *, cwd: Path | None = None, check: bool = False) -> None: self.command = command self.cwd = cwd self.check = check - def encapsulate(self) -> dict: + def encapsulate(self) -> _CommandContextData: logger.debug(f"Running command: {self.command}") output = subprocess.run( self.command, diff --git a/capsula/_context/_cpu.py b/capsula/_context/_cpu.py index 38fb1a9d..ab99ac28 100644 --- a/capsula/_context/_cpu.py +++ b/capsula/_context/_cpu.py @@ -1,11 +1,15 @@ +from __future__ import annotations + +from typing import Any + from cpuinfo import get_cpu_info from ._base import ContextBase class CpuContext(ContextBase): - def encapsulate(self) -> dict: - return get_cpu_info() + def encapsulate(self) -> dict[str, Any]: + return get_cpu_info() # type: ignore[no-any-return] def default_key(self) -> str: return "cpu" diff --git a/capsula/_context/_file.py b/capsula/_context/_file.py index 5b3a494a..cf0e749b 100644 --- a/capsula/_context/_file.py +++ b/capsula/_context/_file.py @@ -4,7 +4,7 @@ import warnings from pathlib import Path from shutil import copyfile, move -from typing import TYPE_CHECKING, Callable, Iterable +from typing import TYPE_CHECKING, Callable, Iterable, TypedDict from capsula._backport import file_digest @@ -16,6 +16,12 @@ logger = logging.getLogger(__name__) +class _FileContextData(TypedDict): + copied_to: tuple[Path, ...] + moved_to: Path | None + hash: dict[str, str] | None + + class FileContext(ContextBase): _default_hash_algorithm = "sha256" @@ -46,21 +52,24 @@ def _normalize_copy_dst_path(self, p: Path) -> Path: else: return p - def encapsulate(self) -> dict: + def encapsulate(self) -> _FileContextData: self.copy_to = tuple(self._normalize_copy_dst_path(p) for p in self.copy_to) - info: dict = { - "copied_to": self.copy_to, - "moved_to": self.move_to, - } - if self.compute_hash: with self.path.open("rb") as f: digest = file_digest(f, self.hash_algorithm).hexdigest() - info["hash"] = { + hash_data = { "algorithm": self.hash_algorithm, "digest": digest, } + else: + hash_data = None + + info: _FileContextData = { + "copied_to": self.copy_to, + "moved_to": self.move_to, + "hash": hash_data, + } for path in self.copy_to: copyfile(self.path, path) diff --git a/capsula/_context/_function.py b/capsula/_context/_function.py index c2a525c1..03e2d02c 100644 --- a/capsula/_context/_function.py +++ b/capsula/_context/_function.py @@ -2,18 +2,25 @@ import inspect from pathlib import Path -from typing import Any, Callable, Mapping, Sequence +from typing import Any, Callable, Mapping, Sequence, TypedDict from ._base import ContextBase +class _FunctionCallContextData(TypedDict): + file_path: Path + first_line_no: int + args: Sequence[Any] + kwargs: Mapping[str, Any] + + class FunctionCallContext(ContextBase): - def __init__(self, function: Callable, args: Sequence[Any], kwargs: Mapping[str, Any]) -> None: + def __init__(self, function: Callable[..., Any], args: Sequence[Any], kwargs: Mapping[str, Any]) -> None: self.function = function self.args = args self.kwargs = kwargs - def encapsulate(self) -> dict: + def encapsulate(self) -> _FunctionCallContextData: file_path = Path(inspect.getfile(self.function)) _, first_line_no = inspect.getsourcelines(self.function) return { diff --git a/capsula/_context/_git.py b/capsula/_context/_git.py index f8d3becc..11b38d35 100644 --- a/capsula/_context/_git.py +++ b/capsula/_context/_git.py @@ -2,8 +2,9 @@ import inspect import logging +from os import PathLike from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING, Callable, TypedDict from git.repo import Repo @@ -23,6 +24,14 @@ def __init__(self, repo: Repo) -> None: super().__init__(f"Repository {repo.working_dir} is dirty") +class _GitRepositoryContextData(TypedDict): + working_dir: PathLike[str] | str + sha: str + remotes: dict[str, str] + branch: str + is_dirty: bool + + class GitRepositoryContext(ContextBase): def __init__( self, @@ -39,12 +48,12 @@ def __init__( self.allow_dirty = allow_dirty self.diff_file = None if diff_file is None else Path(diff_file) - def encapsulate(self) -> dict: + def encapsulate(self) -> _GitRepositoryContextData: repo = Repo(self.path, search_parent_directories=self.search_parent_directories) if not self.allow_dirty and repo.is_dirty(): raise GitRepositoryDirtyError(repo) - info = { + info: _GitRepositoryContextData = { "working_dir": repo.working_dir, "sha": repo.head.commit.hexsha, "remotes": {remote.name: remote.url for remote in repo.remotes}, diff --git a/capsula/_context/_platform.py b/capsula/_context/_platform.py index d401fdfd..4b0178e5 100644 --- a/capsula/_context/_platform.py +++ b/capsula/_context/_platform.py @@ -1,10 +1,24 @@ +from __future__ import annotations + import platform as pf +from typing import TypedDict from ._base import ContextBase +class _PlatformContextData(TypedDict): + machine: str + node: str + platform: str + release: str + version: str + system: str + processor: str + python: dict[str, str | dict[str, str]] + + class PlatformContext(ContextBase): - def encapsulate(self) -> dict: + def encapsulate(self) -> _PlatformContextData: return { "machine": pf.machine(), "node": pf.node(), diff --git a/capsula/_root.py b/capsula/_root.py index a59017e5..017c3407 100644 --- a/capsula/_root.py +++ b/capsula/_root.py @@ -15,7 +15,7 @@ def record(key: _CapsuleItemKey, value: Any) -> None: def current_run_name() -> str: - run: Run | None = Run.get_current() + run: Run[Any, Any] | None = Run.get_current() if run is None: msg = "No active run found." raise RuntimeError(msg) diff --git a/capsula/_run.py b/capsula/_run.py index 30332d7b..7de4b3e0 100644 --- a/capsula/_run.py +++ b/capsula/_run.py @@ -3,7 +3,7 @@ import queue import threading from pathlib import Path -from typing import TYPE_CHECKING, Callable, Generic, Literal, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar, Unpack, overload from pydantic import BaseModel @@ -24,9 +24,9 @@ class FuncInfo(BaseModel): - func: Callable - args: tuple - kwargs: dict + func: Callable[..., Any] + args: tuple[Any, ...] + kwargs: dict[str, Any] class CapsuleParams(FuncInfo): @@ -41,7 +41,7 @@ class Run(Generic[_P, _T]): def _get_run_stack(cls) -> queue.LifoQueue[Self]: if not hasattr(cls._thread_local, "run_stack"): cls._thread_local.run_stack = queue.LifoQueue() - return cls._thread_local.run_stack + return cls._thread_local.run_stack # type: ignore[no-any-return] @classmethod def get_current(cls) -> Self | None: @@ -58,7 +58,12 @@ def __init__(self, func: Callable[_P, _T], *, pass_pre_run_capsule: Literal[Fals def __init__(self, func: Callable[Concatenate[Capsule, _P], _T], *, pass_pre_run_capsule: Literal[True]) -> None: ... - def __init__(self, func, *, pass_pre_run_capsule: bool = False) -> None: + def __init__( + self, + func: Callable[_P, _T] | Callable[Concatenate[Capsule, _P], _T], + *, + pass_pre_run_capsule: bool = False, + ) -> None: self.pre_run_context_generators: list[Callable[[CapsuleParams], ContextBase]] = [] self.in_run_watcher_generators: list[Callable[[CapsuleParams], WatcherBase]] = [] self.post_run_context_generators: list[Callable[[CapsuleParams], ContextBase]] = [] diff --git a/capsula/encapsulator.py b/capsula/encapsulator.py index 6030cb99..f0b83dbd 100644 --- a/capsula/encapsulator.py +++ b/capsula/encapsulator.py @@ -40,10 +40,10 @@ def encapsulate(self) -> Any: _V = TypeVar("_V", bound=WatcherBase) -class WatcherGroup(AbstractContextManager, Generic[_K, _V]): +class WatcherGroup(AbstractContextManager[dict[_K, Any]], Generic[_K, _V]): def __init__(self, watchers: OrderedDict[_K, _V]) -> None: self.watchers = watchers - self.context_manager_stack: queue.LifoQueue[AbstractContextManager] = queue.LifoQueue() + self.context_manager_stack: queue.LifoQueue[AbstractContextManager[None]] = queue.LifoQueue() def __enter__(self) -> dict[_K, Any]: self.context_manager_stack = queue.LifoQueue() @@ -83,7 +83,7 @@ class Encapsulator: def _get_context_stack(cls) -> queue.LifoQueue[Self]: if not hasattr(cls._thread_local, "context_stack"): cls._thread_local.context_stack = queue.LifoQueue() - return cls._thread_local.context_stack + return cls._thread_local.context_stack # type: ignore[no-any-return] @classmethod def get_current(cls) -> Self | None: From 8c6ea63a3ef91f56de8986a28fcc03f8b884bf38 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:12:15 +0900 Subject: [PATCH 04/10] Update mypy command to run in strict mode --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e91610e..f94d1349 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: run: poetry install --no-interaction - name: Run mypy - run: poetry run --no-interaction mypy . + run: poetry run --no-interaction mypy --strict . tox: if: github.event.pull_request.draft == false From 4833c7f1415b210ec833962876d65043364178f9 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:13:21 +0900 Subject: [PATCH 05/10] Fix ruff errors --- capsula/_context/_git.py | 3 ++- capsula/_run.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/capsula/_context/_git.py b/capsula/_context/_git.py index 11b38d35..5b42a126 100644 --- a/capsula/_context/_git.py +++ b/capsula/_context/_git.py @@ -2,7 +2,6 @@ import inspect import logging -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Callable, TypedDict @@ -13,6 +12,8 @@ from ._base import ContextBase if TYPE_CHECKING: + from os import PathLike + from capsula._run import CapsuleParams logger = logging.getLogger(__name__) diff --git a/capsula/_run.py b/capsula/_run.py index 7de4b3e0..8f5485a4 100644 --- a/capsula/_run.py +++ b/capsula/_run.py @@ -3,7 +3,7 @@ import queue import threading from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar, Unpack, overload +from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar, overload from pydantic import BaseModel From 7b3793e576c639d933a16e0720650eae68fe8bbc Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:24:09 +0900 Subject: [PATCH 06/10] resolve mypy error in Python 3.8 --- capsula/_backport.py | 6 +++--- capsula/_context/_file.py | 2 +- capsula/encapsulator.py | 4 ++-- examples/enc_context_manager.py | 4 ++-- examples/low_level.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/capsula/_backport.py b/capsula/_backport.py index a7c1f256..47a6e0b6 100644 --- a/capsula/_backport.py +++ b/capsula/_backport.py @@ -21,11 +21,11 @@ file_digest = hashlib.file_digest else: if TYPE_CHECKING: - import io + from typing_extensions import Buffer from typing import Protocol class _BytesIOLike(Protocol): - def getbuffer(self) -> io.ReadableBuffer: + def getbuffer(self) -> Buffer: ... class _FileDigestFileObj(Protocol): @@ -41,7 +41,7 @@ def file_digest( /, *, _bufsize: int = 2**18, - ): + ) -> hashlib._Hash: """Hash the contents of a file-like object. Returns a digest object. *fileobj* must be a file-like object opened for reading in binary mode. diff --git a/capsula/_context/_file.py b/capsula/_context/_file.py index cf0e749b..7ceb318a 100644 --- a/capsula/_context/_file.py +++ b/capsula/_context/_file.py @@ -74,7 +74,7 @@ def encapsulate(self) -> _FileContextData: for path in self.copy_to: copyfile(self.path, path) if self.move_to is not None: - move(self.path, self.move_to) + move(str(self.path), self.move_to) return info diff --git a/capsula/encapsulator.py b/capsula/encapsulator.py index f0b83dbd..cc6ee403 100644 --- a/capsula/encapsulator.py +++ b/capsula/encapsulator.py @@ -6,7 +6,7 @@ from collections.abc import Hashable from contextlib import AbstractContextManager from itertools import chain -from typing import TYPE_CHECKING, Any, Generic, Tuple, TypeVar, Union +from typing import TYPE_CHECKING, Any, Dict, Generic, Tuple, TypeVar, Union from capsula.utils import ExceptionInfo @@ -40,7 +40,7 @@ def encapsulate(self) -> Any: _V = TypeVar("_V", bound=WatcherBase) -class WatcherGroup(AbstractContextManager[dict[_K, Any]], Generic[_K, _V]): +class WatcherGroup(AbstractContextManager[Dict[_K, Any]], Generic[_K, _V]): def __init__(self, watchers: OrderedDict[_K, _V]) -> None: self.watchers = watchers self.context_manager_stack: queue.LifoQueue[AbstractContextManager[None]] = queue.LifoQueue() diff --git a/examples/enc_context_manager.py b/examples/enc_context_manager.py index c443487f..e2e62520 100644 --- a/examples/enc_context_manager.py +++ b/examples/enc_context_manager.py @@ -1,6 +1,6 @@ import logging import random -from datetime import UTC, datetime +from datetime import datetime, timezone from pathlib import Path import orjson @@ -29,7 +29,7 @@ def calc_pi(n_samples: int, seed: int) -> float: def main(n_samples: int, seed: int) -> None: # Define the run name and create the capsule directory - run_name = datetime.now(UTC).astimezone().strftime(r"%Y%m%d_%H%M%S") + run_name = datetime.now(timezone.utc).astimezone().strftime(r"%Y%m%d_%H%M%S") capsule_directory = Path(__file__).parents[1] / "vault" / run_name with capsula.Encapsulator() as enc: diff --git a/examples/low_level.py b/examples/low_level.py index 91cd20c1..3a41dcd6 100644 --- a/examples/low_level.py +++ b/examples/low_level.py @@ -1,6 +1,6 @@ import logging import random -from datetime import UTC, datetime +from datetime import datetime, timezone from pathlib import Path import orjson @@ -16,7 +16,7 @@ # Define the run name and create the capsule directory -run_name = datetime.now(UTC).astimezone().strftime(r"%Y%m%d_%H%M%S") +run_name = datetime.now(timezone.utc).astimezone().strftime(r"%Y%m%d_%H%M%S") capsule_directory = Path(__file__).parents[1] / "vault" / run_name capsule_directory.mkdir(parents=True, exist_ok=True) From 7a0cbd33f1f89e524235ec3890687ad97b3bc174 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:40:44 +0900 Subject: [PATCH 07/10] Add AbstractContextManager to _backport.py --- capsula/_backport.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/capsula/_backport.py b/capsula/_backport.py index 47a6e0b6..ffdd177c 100644 --- a/capsula/_backport.py +++ b/capsula/_backport.py @@ -1,6 +1,6 @@ from __future__ import annotations -__all__ = ["Concatenate", "ParamSpec", "Self", "TypeAlias", "file_digest"] +__all__ = ["AbstractContextManager", "Concatenate", "ParamSpec", "Self", "TypeAlias", "file_digest"] import hashlib import sys @@ -16,6 +16,11 @@ else: from typing_extensions import Concatenate, ParamSpec, TypeAlias +if sys.version_info >= (3, 9): + from contextlib import AbstractContextManager +else: + from typing import ContextManager as AbstractContextManager + if sys.version_info >= (3, 11): file_digest = hashlib.file_digest From ccaf01003347a597c07b565b8c59e8d35b33e84a Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:41:28 +0900 Subject: [PATCH 08/10] Update type hints in capsula/_run.py --- capsula/_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/capsula/_run.py b/capsula/_run.py index 8f5485a4..959ad799 100644 --- a/capsula/_run.py +++ b/capsula/_run.py @@ -3,7 +3,7 @@ import queue import threading from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, Dict, Generic, Literal, Tuple, TypeVar, overload from pydantic import BaseModel @@ -25,8 +25,8 @@ class FuncInfo(BaseModel): func: Callable[..., Any] - args: tuple[Any, ...] - kwargs: dict[str, Any] + args: Tuple[Any, ...] + kwargs: Dict[str, Any] class CapsuleParams(FuncInfo): From 51723118ef53e38f0ab25f5d41c4c72b1f8c90d6 Mon Sep 17 00:00:00 2001 From: Shunichiro Nomura Date: Sun, 4 Feb 2024 22:41:39 +0900 Subject: [PATCH 09/10] Refactor AbstractContextManager import and usage --- capsula/encapsulator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capsula/encapsulator.py b/capsula/encapsulator.py index cc6ee403..2eb58b2c 100644 --- a/capsula/encapsulator.py +++ b/capsula/encapsulator.py @@ -4,12 +4,12 @@ import threading from collections import OrderedDict from collections.abc import Hashable -from contextlib import AbstractContextManager from itertools import chain from typing import TYPE_CHECKING, Any, Dict, Generic, Tuple, TypeVar, Union from capsula.utils import ExceptionInfo +from ._backport import AbstractContextManager from ._capsule import Capsule from ._context import ContextBase from ._watcher import WatcherBase @@ -40,7 +40,7 @@ def encapsulate(self) -> Any: _V = TypeVar("_V", bound=WatcherBase) -class WatcherGroup(AbstractContextManager[Dict[_K, Any]], Generic[_K, _V]): +class WatcherGroup(Generic[_K, _V], AbstractContextManager[Dict[_K, Any]]): def __init__(self, watchers: OrderedDict[_K, _V]) -> None: self.watchers = watchers self.context_manager_stack: queue.LifoQueue[AbstractContextManager[None]] = queue.LifoQueue() From a883a6da415201390b2047a8eaa0d9dbb31bc76c Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 13:44:44 +0000 Subject: [PATCH 10/10] Update tox report --- coverage/badge.svg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage/badge.svg b/coverage/badge.svg index 1b5968e7..e2b73409 100644 --- a/coverage/badge.svg +++ b/coverage/badge.svg @@ -1 +1 @@ -coverage: 46.32%coverage46.32% \ No newline at end of file +coverage: 48.92%coverage48.92% \ No newline at end of file