diff --git a/.github/workflows/update-expected-output.yml b/.github/workflows/update-expected-output.yml index b3b2672..71bef16 100644 --- a/.github/workflows/update-expected-output.yml +++ b/.github/workflows/update-expected-output.yml @@ -2,9 +2,10 @@ name: Update expected output on: pull_request: - # Don't update PRs on every push. PRs can be closed and - # reopened if the update action should be run again. - types: [opened, reopened] + # Don't update PRs on every push. PRs can be switched + # to draft status and back (or closed and reopened), + # if the update action should be run again. + types: [opened, reopened, ready_for_review] branches: - "**" paths: diff --git a/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst b/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst new file mode 100644 index 0000000..55d7fbf --- /dev/null +++ b/changelog.d/20241108_162952_ncoghlan_fix_automatic_layer_versioning.rst @@ -0,0 +1,9 @@ +Added +----- + +- Setting ``versioned = True`` in a layer definition will append a + lock version number to the layer name that automatically increments + each time the locked requirements change for that layer (``layer@1``, + ``layer@2``, etc). Layer dependency declarations and build environments, + use the unversioned name, but deployed environments and their metadata + will use the versioned name (implemented in :issue:`24`). diff --git a/docs/api/stacks/venvstacks.stacks.ApplicationEnv.rst b/docs/api/stacks/venvstacks.stacks.ApplicationEnv.rst index 8fa92f4..dca1c1e 100644 --- a/docs/api/stacks/venvstacks.stacks.ApplicationEnv.rst +++ b/docs/api/stacks/venvstacks.stacks.ApplicationEnv.rst @@ -15,7 +15,7 @@ venvstacks.stacks.ApplicationEnv ~ApplicationEnv.export_environment ~ApplicationEnv.get_constraint_paths ~ApplicationEnv.install_requirements - ~ApplicationEnv.link_base_runtime_paths + ~ApplicationEnv.link_base_runtime ~ApplicationEnv.link_layered_environments ~ApplicationEnv.lock_requirements ~ApplicationEnv.report_python_site_details @@ -26,6 +26,7 @@ venvstacks.stacks.ApplicationEnv .. autosummary:: + ~FrameworkEnv.base_runtime ~ApplicationEnv.category ~ApplicationEnv.env_name ~ApplicationEnv.env_spec @@ -33,9 +34,7 @@ venvstacks.stacks.ApplicationEnv ~ApplicationEnv.kind ~ApplicationEnv.launch_module_name ~ApplicationEnv.linked_constraints_paths - ~ApplicationEnv.linked_dynlib_paths ~ApplicationEnv.linked_frameworks - ~ApplicationEnv.linked_pylib_paths ~ApplicationEnv.want_build ~ApplicationEnv.want_lock ~ApplicationEnv.want_publish diff --git a/docs/api/stacks/venvstacks.stacks.FrameworkEnv.rst b/docs/api/stacks/venvstacks.stacks.FrameworkEnv.rst index 14e21ee..d0aa889 100644 --- a/docs/api/stacks/venvstacks.stacks.FrameworkEnv.rst +++ b/docs/api/stacks/venvstacks.stacks.FrameworkEnv.rst @@ -16,7 +16,7 @@ venvstacks.stacks.FrameworkEnv ~FrameworkEnv.export_environment ~FrameworkEnv.get_constraint_paths ~FrameworkEnv.install_requirements - ~FrameworkEnv.link_base_runtime_paths + ~FrameworkEnv.link_base_runtime ~FrameworkEnv.lock_requirements ~FrameworkEnv.report_python_site_details ~FrameworkEnv.request_export @@ -26,6 +26,7 @@ venvstacks.stacks.FrameworkEnv .. autosummary:: + ~FrameworkEnv.base_runtime ~FrameworkEnv.category ~FrameworkEnv.env_name ~FrameworkEnv.env_spec diff --git a/src/venvstacks/stacks.py b/src/venvstacks/stacks.py index 8c02313..82a68fe 100755 --- a/src/venvstacks/stacks.py +++ b/src/venvstacks/stacks.py @@ -27,6 +27,7 @@ Any, ClassVar, Iterable, + Iterator, Literal, Mapping, MutableMapping, @@ -955,6 +956,9 @@ def get_build_platform() -> TargetPlatform: @dataclass class _PythonEnvironment(ABC): + # Python environment used to run tools like `uv` and `pip` + tools_python_path: ClassVar[Path] = Path(sys.executable) + # Specified in concrete subclasses kind: ClassVar[LayerVariants] category: ClassVar[LayerCategories] @@ -973,11 +977,12 @@ class _PythonEnvironment(ABC): python_path: Path = field(init=False, repr=False) env_lock: EnvironmentLock = field(init=False) - # Set in subclass or externally after creation - base_python_path: Path | None = field(init=False, repr=False) - tools_python_path: Path | None = field(init=False, repr=False) + # Derived from layer spec in subclass __post_init__ py_version: str = field(init=False, repr=False) + # Set in subclass __post_init__, or when build environments are created + base_python_path: Path | None = field(init=False, repr=False) + # Operation flags allow for requested commands to be applied only to selected layers # Notes: # - the "build if needed" (want_build=None) option is fairly ineffective, since @@ -1010,7 +1015,16 @@ def env_name(self) -> EnvNameBuild: def install_target(self) -> EnvNameDeploy: return self.env_lock.get_deployed_name(self.env_spec.env_name) + def get_deployed_path(self, build_path: Path) -> str: + """Get relative deployment location for a build env path""" + env_deployed_path = Path(self.install_target) + relative_path = build_path.relative_to(self.env_path) + return str(env_deployed_path / relative_path) + def __post_init__(self) -> None: + # Concrete subclasses must set the version before finishing the base initialisation + # Assert its existence here to make failures to do so easier to diagnose + assert self.py_version is not None, "Subclass failed to set 'py_version'" self.env_path = self.build_path / self.env_name self.pylib_path = self._get_py_scheme_path("purelib") self.executables_path = self._get_py_scheme_path("scripts") @@ -1041,8 +1055,8 @@ def get_deployed_config(self) -> postinstall.LayerConfig: def _get_deployed_config( self, - pylib_paths: Iterable[Path], - dynlib_paths: Iterable[Path], + pylib_dirs: Iterable[str], + dynlib_dirs: Iterable[str], link_external_base: bool = True, ) -> postinstall.LayerConfig: # Helper for subclass get_deployed_config implementations @@ -1053,18 +1067,13 @@ def _get_deployed_config( build_env_name = build_env_path.name build_path = build_env_path.parent - def from_internal_path(target_build_path: Path) -> str: - # Input path is an absolute path inside the environment - # Output path is relative to the base of the environment - return str(target_build_path.relative_to(build_env_path)) - - def from_relative_path(relative_build_path: Path) -> str: + def relative_to_env(relative_build_path: Path) -> str: # Input path is relative to the base of the build directory # Output path is relative to the base of the environment # Note: we avoid `walk_up=True` here, firstly to maintain # Python 3.11 compatibility, but also to limit the # the relative paths to *peer* environments, rather - # than all potentially value relative path calculations + # than all potentially valid relative path calculations if relative_build_path.is_absolute(): self._fail_build(f"{relative_build_path} is not a relative path") if relative_build_path.parts[0] == build_env_name: @@ -1073,19 +1082,23 @@ def from_relative_path(relative_build_path: Path) -> str: # Emit relative reference to peer environment return str(Path("..", *relative_build_path.parts)) + def from_internal_path(target_build_path: Path) -> str: + # Input path is an absolute path inside the environment + # Output path is relative to the base of the environment + return str(target_build_path.relative_to(build_env_path)) + def from_external_path(target_build_path: Path) -> str: # Input path is an absolute path, potentially from a peer environment # Output path is relative to the base of the environment relative_build_path = target_build_path.relative_to(build_path) - return from_relative_path(relative_build_path) + return relative_to_env(relative_build_path) layer_python = from_internal_path(self.python_path) if link_external_base: base_python = from_external_path(base_python_path) else: - # "base_python" in the runtime layer refers solely to - # the external environment used to set up the base - # runtime layer, rather than being a linked environment + # "base_python" in a runtime layer refers to the layer itself + assert layer_python == from_internal_path(base_python_path) base_python = layer_python return postinstall.LayerConfig( @@ -1093,8 +1106,8 @@ def from_external_path(target_build_path: Path) -> str: py_version=self.py_version, base_python=base_python, site_dir=from_internal_path(self.pylib_path), - pylib_dirs=[from_relative_path(p) for p in pylib_paths], - dynlib_dirs=[from_relative_path(p) for p in dynlib_paths], + pylib_dirs=[relative_to_env(Path(d)) for d in pylib_dirs], + dynlib_dirs=[relative_to_env(Path(d)) for d in dynlib_dirs], ) def _write_deployed_config(self) -> None: @@ -1401,7 +1414,10 @@ def export_environment( # Define the input metadata that gets published in the export manifest export_request = self.request_export(output_path, previous_metadata, force) - return export_request.export_environment(env_path, previous_metadata) + return export_request.export_environment( + env_path, + previous_metadata, + ) class RuntimeEnv(_PythonEnvironment): @@ -1417,20 +1433,11 @@ def _get_python_dir_path(self) -> Path: return super()._get_python_dir_path() def __post_init__(self) -> None: - self.py_version = py_version = self.env_spec.py_version + # Ensure Python version is set before finishing base class initialisation + self.py_version = self.env_spec.py_version super().__post_init__() - tools_env_path = self.build_path / "build-tools" - if tools_env_path.exists(): - tools_bin_path = Path( - _get_py_scheme_path("scripts", tools_env_path, py_version) - ) - tools_python_path = tools_bin_path / _binary_with_extension("python") - else: - # No build tools environment created by wrapper script, so use the running Python - tools_python_path = Path(sys.executable) - # Runtimes have no base Python other than the build tools Python - self.base_python_path = tools_python_path - self.tools_python_path = tools_python_path + # Runtimes are their own base Python + self.base_python_path = self.python_path @property def env_spec(self) -> RuntimeSpec: @@ -1481,16 +1488,16 @@ def create_build_environment(self, *, clean: bool = False) -> None: class _VirtualEnvironment(_PythonEnvironment): + base_runtime: RuntimeEnv | None = field(init=False, repr=False) linked_constraints_paths: list[Path] = field(init=False, repr=False) - linked_pylib_paths: list[Path] = field(init=False, repr=False) - linked_dynlib_paths: list[Path] = field(init=False, repr=False) def __post_init__(self) -> None: + # Ensure Python version is set before finishing base class initialisation self.py_version = self.env_spec.runtime.py_version super().__post_init__() + # Base runtime env will be linked when creating the build environments + self.base_runtime = None self.linked_constraints_paths = [] - self.linked_pylib_paths = [] - self.linked_dynlib_paths = [] @property def env_spec(self) -> _VirtualEnvironmentSpec: @@ -1498,18 +1505,51 @@ def env_spec(self) -> _VirtualEnvironmentSpec: assert isinstance(self._env_spec, _VirtualEnvironmentSpec) return self._env_spec - def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: + def _linked_environments(self) -> Iterator[_PythonEnvironment]: + runtime_env = self.base_runtime + # This is only ever invoked *after* the environment has been linked + assert runtime_env is not None + yield runtime_env + + def _iter_build_pylib_dirs(self) -> Iterator[str]: + for env in self._linked_environments(): + yield str(env.pylib_path.relative_to(self.build_path)) + + def _iter_build_dynlib_dirs(self) -> Iterator[str]: + for env in self._linked_environments(): + dynlib_path = env.dynlib_path + if dynlib_path is not None: + yield str(dynlib_path.relative_to(self.build_path)) + + def _iter_deployed_pylib_dirs(self) -> Iterator[str]: + for env in self._linked_environments(): + yield env.get_deployed_path(env.pylib_path) + + def _iter_deployed_dynlib_dirs(self) -> Iterator[str]: + for env in self._linked_environments(): + dynlib_path = env.dynlib_path + if dynlib_path is not None: + yield env.get_deployed_path(dynlib_path) + + def link_base_runtime(self, runtime: RuntimeEnv) -> None: + if self.base_runtime is not None: + raise BuildEnvError( + f"Layered environment base runtime already linked {self}" + ) + # Link the runtime environment + self.base_runtime = runtime # Link executable paths self.base_python_path = runtime.python_path - self.tools_python_path = runtime.tools_python_path + # Link runtime layer dependency constraints if self.linked_constraints_paths: self._fail_build("Layered environment base runtime already linked") self.linked_constraints_paths[:] = [runtime.requirements_path] + print(f"Linked {self}") def get_deployed_config(self) -> postinstall.LayerConfig: """Layer config to be published in `venvstacks_layer.json`""" return self._get_deployed_config( - self.linked_pylib_paths, self.linked_dynlib_paths + self._iter_deployed_pylib_dirs(), self._iter_deployed_dynlib_dirs() ) def get_constraint_paths(self) -> list[Path]: @@ -1544,8 +1584,8 @@ def _ensure_virtual_environment(self) -> subprocess.CompletedProcess[str]: def _link_build_environment(self) -> None: # Create sitecustomize file for the build environment build_path = self.build_path - build_pylib_paths = [build_path / p for p in self.linked_pylib_paths] - build_dynlib_paths = [build_path / p for p in self.linked_dynlib_paths] + build_pylib_paths = [build_path / d for d in self._iter_build_pylib_dirs()] + build_dynlib_paths = [build_path / d for d in self._iter_build_dynlib_dirs()] sc_contents = postinstall.generate_sitecustomize( build_pylib_paths, build_dynlib_paths ) @@ -1591,21 +1631,6 @@ def env_spec(self) -> FrameworkSpec: assert isinstance(self._env_spec, FrameworkSpec) return self._env_spec - def link_base_runtime_paths(self, runtime: RuntimeEnv) -> None: - super().link_base_runtime_paths(runtime) - # TODO: Reduce code duplication with ApplicationEnv - runtime_target_path = Path(runtime.install_target) - - def _runtime_path(build_path: Path) -> Path: - relative_path = build_path.relative_to(runtime.env_path) - return runtime_target_path / relative_path - - pylib_paths = self.linked_pylib_paths - dynlib_paths = self.linked_dynlib_paths - pylib_paths.append(_runtime_path(runtime.pylib_path)) - if runtime.dynlib_path is not None: - dynlib_paths.append(_runtime_path(runtime.dynlib_path)) - class ApplicationEnv(_VirtualEnvironment): """Application layer build environment""" @@ -1627,42 +1652,28 @@ def __post_init__(self) -> None: self.launch_module_name = self.env_spec.launch_module_path.stem self.linked_frameworks = [] + def _linked_environments(self) -> Iterator[_PythonEnvironment]: + # Linked frameworks are emitted before the base runtime layer + for fw_env in self.linked_frameworks: + yield fw_env + yield from super()._linked_environments() + def link_layered_environments( self, runtime: RuntimeEnv, frameworks: Mapping[LayerBaseName, FrameworkEnv] ) -> None: - self.link_base_runtime_paths(runtime) + self.link_base_runtime(runtime) constraints_paths = self.linked_constraints_paths if not constraints_paths: self._fail_build("Failed to add base environment constraints path") # The runtime site-packages folder is added here rather than via pyvenv.cfg # to ensure it appears in sys.path after the framework site-packages folders - pylib_paths = self.linked_pylib_paths - dynlib_paths = self.linked_dynlib_paths fw_envs = self.linked_frameworks - if pylib_paths or dynlib_paths or fw_envs: + if fw_envs: self._fail_build("Layered application environment already linked") for env_spec in self.env_spec.frameworks: env = frameworks[env_spec.name] fw_envs.append(env) constraints_paths.append(env.requirements_path) - install_target_path = Path(env.install_target) - - def _fw_env_path(build_path: Path) -> Path: - relative_path = build_path.relative_to(env.env_path) - return install_target_path / relative_path - - pylib_paths.append(_fw_env_path(env.pylib_path)) - if env.dynlib_path is not None: - dynlib_paths.append(_fw_env_path(env.pylib_path)) - runtime_target_path = Path(runtime.install_target) - - def _runtime_path(build_path: Path) -> Path: - relative_path = build_path.relative_to(runtime.env_path) - return runtime_target_path / relative_path - - pylib_paths.append(_runtime_path(runtime.pylib_path)) - if runtime.dynlib_path is not None: - dynlib_paths.append(_runtime_path(runtime.dynlib_path)) def _update_existing_environment(self, *, lock_only: bool = False) -> None: super()._update_existing_environment(lock_only=lock_only) @@ -1850,7 +1861,7 @@ def define_build_environment( ) for fw_env in frameworks.values(): runtime = runtimes[fw_env.env_spec.runtime.name] - fw_env.link_base_runtime_paths(runtime) + fw_env.link_base_runtime(runtime) print("Defining application environments:") applications = self._define_envs( build_path, index_config, ApplicationEnv, self.applications diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json index 8a70e77..351ad8f 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-client.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "bb594ea66705fe5a122e930e2c48f1aaf6f720d568e37e0b590db95fb7261dea" + "sha256": "d932e554c35c9ce2cd1d594cb20e8a37c0195a5b82ffcb50627a96fc122e79a2" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 3004, + "archive_size": 3008, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:23:43.205589+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:fb8a843c694d03d7ee74b457cdac2bd82b6b439de0ed308d72fe698c6c9c6cf4", diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json index 64b962f..91213b7 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/app-scipy-import.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "8d69cfe0f408ba396dde664cd58c18b4541e6937f470e6e4f06a03d6ec069e46" + "sha256": "ec5be5ebc608e24bf32642cf0e071ce975f4650dac69c233c36124e53f4c10f9" }, - "archive_name": "app-scipy-import.tar.xz", + "archive_name": "app-scipy-import@1.tar.xz", "archive_size": 2912, - "install_target": "app-scipy-import", + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:23:43.173589+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:36b0dbfec94b7de6507f348f0823cd02fdca2ea79eeafe92d571c26ae347d150", "runtime_name": "cpython@3.11", diff --git a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json index ae73caf..91978f2 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/env_metadata/framework-scipy.json @@ -1,11 +1,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "e8ef68225cfa6a016539a4db3c5dc1b984f53de0df51181eefeb0e9449771ea9" + "sha256": "622e3056ccf54d8723bf983c92f35bf4138d2fee5da3cdd56d25391f18906aab" }, - "archive_name": "framework-scipy.tar.xz", - "archive_size": 23957800, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.tar.xz", + "archive_size": 23961760, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:23:42.825586+00:00", diff --git a/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json b/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json index c9818e3..973d2e5 100644 --- a/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json +++ b/tests/sample_project/expected_manifests/linux_x86_64/venvstacks.json @@ -6,16 +6,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "8d69cfe0f408ba396dde664cd58c18b4541e6937f470e6e4f06a03d6ec069e46" + "sha256": "ec5be5ebc608e24bf32642cf0e071ce975f4650dac69c233c36124e53f4c10f9" }, - "archive_name": "app-scipy-import.tar.xz", + "archive_name": "app-scipy-import@1.tar.xz", "archive_size": 2912, - "install_target": "app-scipy-import", + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:23:43.173589+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:36b0dbfec94b7de6507f348f0823cd02fdca2ea79eeafe92d571c26ae347d150", "runtime_name": "cpython@3.11", @@ -26,16 +26,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "bb594ea66705fe5a122e930e2c48f1aaf6f720d568e37e0b590db95fb7261dea" + "sha256": "d932e554c35c9ce2cd1d594cb20e8a37c0195a5b82ffcb50627a96fc122e79a2" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 3004, + "archive_size": 3008, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:23:43.205589+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:fb8a843c694d03d7ee74b457cdac2bd82b6b439de0ed308d72fe698c6c9c6cf4", @@ -67,11 +67,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "e8ef68225cfa6a016539a4db3c5dc1b984f53de0df51181eefeb0e9449771ea9" + "sha256": "622e3056ccf54d8723bf983c92f35bf4138d2fee5da3cdd56d25391f18906aab" }, - "archive_name": "framework-scipy.tar.xz", - "archive_size": 23957800, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.tar.xz", + "archive_size": 23961760, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:23:42.825586+00:00", diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json index 2282818..290b09b 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-client.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "bf065fe724e53e03886117ff0a7d61e29910da9c46b4a14799581cfc1238c77f" + "sha256": "f0338c45382c53320d30dc99e4370075de52593adb46a969f0692e772dcfb3ca" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 2980, + "archive_size": 2984, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:23:33.147967+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:fb8a843c694d03d7ee74b457cdac2bd82b6b439de0ed308d72fe698c6c9c6cf4", diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json index 40a1486..cd4588b 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/app-scipy-import.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "3ab7e8458a233e96790809ad0437209bcbf1823d7883220a44554b1c3fd51afa" + "sha256": "3ef2198496159dca41b2fa5cdea6ec0afb40b6060f95f23dc3da02e99e2d435f" }, - "archive_name": "app-scipy-import.tar.xz", + "archive_name": "app-scipy-import@1.tar.xz", "archive_size": 2896, - "install_target": "app-scipy-import", + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:23:33.121208+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:36b0dbfec94b7de6507f348f0823cd02fdca2ea79eeafe92d571c26ae347d150", "runtime_name": "cpython@3.11", diff --git a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json index 996c5bd..0735cfd 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/env_metadata/framework-scipy.json @@ -1,11 +1,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "6872599275f0b5448926033968ba5f3467b07bf09ac19af8538975a39bf7b712" + "sha256": "32f6135ea3b938e47c8897b0da7a51c6992397edbefe2b27358665b1c83d29aa" }, - "archive_name": "framework-scipy.tar.xz", - "archive_size": 15078848, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.tar.xz", + "archive_size": 15076180, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:23:32.960668+00:00", diff --git a/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json b/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json index 1006d12..08d663a 100644 --- a/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json +++ b/tests/sample_project/expected_manifests/macosx_arm64/venvstacks.json @@ -6,16 +6,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "3ab7e8458a233e96790809ad0437209bcbf1823d7883220a44554b1c3fd51afa" + "sha256": "3ef2198496159dca41b2fa5cdea6ec0afb40b6060f95f23dc3da02e99e2d435f" }, - "archive_name": "app-scipy-import.tar.xz", + "archive_name": "app-scipy-import@1.tar.xz", "archive_size": 2896, - "install_target": "app-scipy-import", + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:23:33.121208+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:36b0dbfec94b7de6507f348f0823cd02fdca2ea79eeafe92d571c26ae347d150", "runtime_name": "cpython@3.11", @@ -26,16 +26,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "bf065fe724e53e03886117ff0a7d61e29910da9c46b4a14799581cfc1238c77f" + "sha256": "f0338c45382c53320d30dc99e4370075de52593adb46a969f0692e772dcfb3ca" }, "archive_name": "app-scipy-client.tar.xz", - "archive_size": 2980, + "archive_size": 2984, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:23:33.147967+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:fb8a843c694d03d7ee74b457cdac2bd82b6b439de0ed308d72fe698c6c9c6cf4", @@ -47,11 +47,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "6872599275f0b5448926033968ba5f3467b07bf09ac19af8538975a39bf7b712" + "sha256": "32f6135ea3b938e47c8897b0da7a51c6992397edbefe2b27358665b1c83d29aa" }, - "archive_name": "framework-scipy.tar.xz", - "archive_size": 15078848, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.tar.xz", + "archive_size": 15076180, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:23:32.960668+00:00", diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json index 0c350fb..4bf772b 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-client.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "9f3de2bf483797a9a93629feca94756de57d82e62b157a4836b96fa14a289180" + "sha256": "ca15fb796933f031340de069d53f6d3a821e481b1372eb4ff001368ac92a214c" }, "archive_name": "app-scipy-client.zip", - "archive_size": 257112, + "archive_size": 257117, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:24:36.468633+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:3bff0428616a2f1724732e78e7788e753dd5f1aa10aa5d3b87707b8dbde121de", diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json index f679796..b68d0a2 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/app-scipy-import.json @@ -3,16 +3,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "3f4e2a19a1611db1139f9e68a268a963dd30139690ee6d8325896265007bf823" + "sha256": "9eae1f6e63422d58ccef53ae199ac55884c50f19e1faf5ecd98c2dfc651ce723" }, - "archive_name": "app-scipy-import.zip", - "archive_size": 256621, - "install_target": "app-scipy-import", + "archive_name": "app-scipy-import@1.zip", + "archive_size": 256677, + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:24:36.386938+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:9aba38b5efe287f35d58825dce6b1c47ed556b930056e6edc00ca9e1a165796b", "runtime_name": "cpython@3.11.10", diff --git a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json index 34fa1a6..827d795 100644 --- a/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json +++ b/tests/sample_project/expected_manifests/win_amd64/env_metadata/framework-scipy.json @@ -1,11 +1,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "71146182f2bcf36ed9af674ac146ee32228e8d0a2890fb9c51ebd87e7fe58b45" + "sha256": "ac2c6a48192ac3657c5cbec001aeddf48ddd45881ac86a0cfa9eb5b839d2e3d4" }, - "archive_name": "framework-scipy.zip", - "archive_size": 45080726, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.zip", + "archive_size": 45087226, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:24:35.999197+00:00", diff --git a/tests/sample_project/expected_manifests/win_amd64/venvstacks.json b/tests/sample_project/expected_manifests/win_amd64/venvstacks.json index b21d903..73a7469 100644 --- a/tests/sample_project/expected_manifests/win_amd64/venvstacks.json +++ b/tests/sample_project/expected_manifests/win_amd64/venvstacks.json @@ -6,16 +6,16 @@ "app_launch_module_hash": "sha256:d806d778921ad216c1f950886d27b4b77e5561fe3467046fec258805980cc6d1", "archive_build": 1, "archive_hashes": { - "sha256": "3f4e2a19a1611db1139f9e68a268a963dd30139690ee6d8325896265007bf823" + "sha256": "9eae1f6e63422d58ccef53ae199ac55884c50f19e1faf5ecd98c2dfc651ce723" }, - "archive_name": "app-scipy-import.zip", - "archive_size": 256621, - "install_target": "app-scipy-import", + "archive_name": "app-scipy-import@1.zip", + "archive_size": 256677, + "install_target": "app-scipy-import@1", "layer_name": "app-scipy-import", "lock_version": 1, "locked_at": "2024-10-15T10:24:36.386938+00:00", "required_layers": [ - "framework-scipy" + "framework-scipy@1" ], "requirements_hash": "sha256:9aba38b5efe287f35d58825dce6b1c47ed556b930056e6edc00ca9e1a165796b", "runtime_name": "cpython@3.11.10", @@ -26,16 +26,16 @@ "app_launch_module_hash": "sha256/bbe4da6de13a8f13a05cdd2bb3b90884861a6636b1450248d03aea799a7fc828", "archive_build": 1, "archive_hashes": { - "sha256": "9f3de2bf483797a9a93629feca94756de57d82e62b157a4836b96fa14a289180" + "sha256": "ca15fb796933f031340de069d53f6d3a821e481b1372eb4ff001368ac92a214c" }, "archive_name": "app-scipy-client.zip", - "archive_size": 257112, + "archive_size": 257117, "install_target": "app-scipy-client", "layer_name": "app-scipy-client", "lock_version": 1, "locked_at": "2024-10-15T10:24:36.468633+00:00", "required_layers": [ - "framework-scipy", + "framework-scipy@1", "framework-http-client" ], "requirements_hash": "sha256:3bff0428616a2f1724732e78e7788e753dd5f1aa10aa5d3b87707b8dbde121de", @@ -47,11 +47,11 @@ { "archive_build": 1, "archive_hashes": { - "sha256": "71146182f2bcf36ed9af674ac146ee32228e8d0a2890fb9c51ebd87e7fe58b45" + "sha256": "ac2c6a48192ac3657c5cbec001aeddf48ddd45881ac86a0cfa9eb5b839d2e3d4" }, - "archive_name": "framework-scipy.zip", - "archive_size": 45080726, - "install_target": "framework-scipy", + "archive_name": "framework-scipy@1.zip", + "archive_size": 45087226, + "install_target": "framework-scipy@1", "layer_name": "framework-scipy", "lock_version": 1, "locked_at": "2024-10-15T10:24:35.999197+00:00", diff --git a/tests/sample_project/venvstacks.toml b/tests/sample_project/venvstacks.toml index 9ceb843..17cb920 100644 --- a/tests/sample_project/venvstacks.toml +++ b/tests/sample_project/venvstacks.toml @@ -65,9 +65,7 @@ requirements = [ [[frameworks]] name = "scipy" -# Automatic versioning currently breaks venv layering -# https://github.com/lmstudio-ai/venvstacks/issues/24 -# versioned = true +versioned = true runtime = "cpython@3.11" requirements = [ "scipy", @@ -96,9 +94,7 @@ requirements = [ [[applications]] name = "scipy-import" -# Automatic versioning currently breaks venv layering -# https://github.com/lmstudio-ai/venvstacks/issues/24 -# versioned = true +versioned = true launch_module = "launch_modules/scipy_import.py" frameworks = ["scipy"] requirements = [ diff --git a/tests/support.py b/tests/support.py index 8b790f6..d26fc25 100644 --- a/tests/support.py +++ b/tests/support.py @@ -243,6 +243,12 @@ class DeploymentTestCase(unittest.TestCase): def assertPathExists(self, expected_path: Path) -> None: self.assertTrue(expected_path.exists(), f"No such path: {str(expected_path)}") + def assertPathContains(self, containing_path: Path, contained_path: Path) -> None: + self.assertTrue( + contained_path.is_relative_to(containing_path), + f"{str(containing_path)!r} is not a parent folder of {str(contained_path)!r}", + ) + def assertSysPathEntry(self, expected: str, env_sys_path: Sequence[str]) -> None: self.assertTrue( any(expected in path_entry for path_entry in env_sys_path), @@ -364,13 +370,16 @@ def check_deployed_environments( # Nothing at all should be emitted on stderr self.assertEqual(launch_result.stderr, "") - def check_environment_exports(self, export_paths: ExportedEnvironmentPaths) -> None: + def check_environment_exports( + self, export_path: Path, export_paths: ExportedEnvironmentPaths + ) -> None: metadata_path, snippet_paths, env_paths = export_paths exported_manifests = ManifestData(metadata_path, snippet_paths) env_name_to_path: dict[str, Path] = {} for env_metadata, env_path in zip(exported_manifests.snippet_data, env_paths): # TODO: Check more details regarding expected metadata contents - self.assertTrue(env_path.exists()) + self.assertPathExists(env_path) + self.assertPathContains(export_path, env_path) env_name = EnvNameDeploy(env_metadata["install_target"]) self.assertEqual(env_path.name, env_name) env_name_to_path[env_name] = env_path diff --git a/tests/test_minimal_project.py b/tests/test_minimal_project.py index 98f1f70..7bd2160 100644 --- a/tests/test_minimal_project.py +++ b/tests/test_minimal_project.py @@ -489,7 +489,7 @@ def get_deployed_env_details( def test_create_environments(self) -> None: # Fast test to check the links between build envs are set up correctly - # (if this fails, there's no point even trying to full slow test case) + # (if this fails, there's no point even trying the full slow test case) build_env = self.build_env build_env.create_environments() self.check_build_environments(self.build_env.all_environments()) @@ -596,7 +596,7 @@ def test_locking_and_publishing(self) -> None: with self.subTest("Check environment export"): export_path = self.working_path / "_export🦎" export_result = build_env.export_environments(export_path) - self.check_environment_exports(export_result) + self.check_environment_exports(export_path, export_result) subtests_passed += 1 # Test stage: ensure published archives and manifests have the expected name # and that unpacking them allows launch module execution diff --git a/tests/test_sample_project.py b/tests/test_sample_project.py index 94a1b93..1e3b184 100644 --- a/tests/test_sample_project.py +++ b/tests/test_sample_project.py @@ -267,7 +267,7 @@ def setUp(self) -> None: def test_create_environments(self) -> None: # Fast test to check the links between build envs are set up correctly - # (if this fails, there's no point even trying to full slow test case) + # (if this fails, there's no point even trying the full slow test case) build_env = self.build_env build_env.create_environments() self.check_build_environments(self.build_env.all_environments()) @@ -381,7 +381,7 @@ def test_build_is_reproducible(self) -> None: with self.subTest("Check environment export"): export_path = self.working_path / "_export🦎" export_result = build_env.export_environments(export_path) - self.check_environment_exports(export_result) + self.check_environment_exports(export_path, export_result) subtests_passed += 1 # Work aroung pytest-subtests not failing the test case when subtests fail