From d59459c1c97b68832c263640c2a875744d53aba0 Mon Sep 17 00:00:00 2001 From: "r.jaepel" Date: Wed, 20 Nov 2024 22:24:13 +0100 Subject: [PATCH] Add a CadetDockerRunner and tests --- cadet/cadet.py | 19 ++- cadet/runner.py | 183 ++++++++++++++++++++++++- pyproject.toml | 4 + tests/docker_resources/Dockerfile | 26 ++++ tests/docker_resources/environment.yml | 6 + tests/test_docker_integration.py | 67 +++++++++ 6 files changed, 302 insertions(+), 3 deletions(-) create mode 100644 tests/docker_resources/Dockerfile create mode 100644 tests/docker_resources/environment.yml create mode 100644 tests/test_docker_integration.py diff --git a/cadet/cadet.py b/cadet/cadet.py index b749fc6..e8aed53 100644 --- a/cadet/cadet.py +++ b/cadet/cadet.py @@ -9,7 +9,7 @@ from addict import Dict from cadet.h5 import H5 -from cadet.runner import CadetRunnerBase, CadetCLIRunner, ReturnInformation +from cadet.runner import CadetRunnerBase, CadetCLIRunner, ReturnInformation, CadetDockerRunner from cadet.cadet_dll import CadetDLLRunner @@ -184,7 +184,10 @@ class Cadet(H5, metaclass=CadetMeta): Stores the information returned after a simulation run. """ - def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, *data): + def __init__( + self, install_path: Optional[Path] = None, use_dll: bool = False, + docker_container: Optional[str] = None, *data + ): """ Initialize a new instance of the Cadet class. Priority order of install_paths is: @@ -194,6 +197,12 @@ def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, * Parameters ---------- + install_path : Optional[Path] + The root directory of the CADET installation. + use_dll : Optional[bool] + Indicates whether to use a DLL or a CLI executable. + docker_container : Optional[str] + Label of the docker container to use. *data : tuple Additional data to be passed to the H5 base class initialization. """ @@ -208,6 +217,12 @@ def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, * self.install_path = install_path # This will set _cadet_dll_runner and _cadet_cli_runner return + # If we get a docker_container, we use the docker container + if docker_container is not None: + self._cadet_docker_runner: Optional[CadetDockerRunner] = CadetDockerRunner( + docker_container + ) + # If _cadet_cli_runner_class has been set in the Meta Class, use them, else instantiate Nones if hasattr(self, "cadet_cli_path") and self.cadet_cli_path is not None: self._cadet_cli_runner: Optional[CadetCLIRunner] = CadetCLIRunner( diff --git a/cadet/runner.py b/cadet/runner.py index fff6304..4520b89 100644 --- a/cadet/runner.py +++ b/cadet/runner.py @@ -1,5 +1,4 @@ import os -import pathlib import re import subprocess from abc import ABC, abstractmethod @@ -7,6 +6,13 @@ from pathlib import Path from typing import Optional +try: + import docker + from docker.types import Mount + from docker.errors import ContainerError +except ImportError: + pass + @dataclass class ReturnInformation: @@ -255,3 +261,178 @@ def cadet_commit_hash(self) -> str: @property def cadet_path(self) -> os.PathLike: return self._cadet_path + + +class CadetDockerRunner(CadetRunnerBase): + """ + Docker-based CADET runner. + + This class runs CADET simulations using a command-line interface (CLI) executable that is + contained inside a Docker container. This expects the Docker container to have the + root of the installation as the working folder. This gives us access to the cadet-cli and + tools such as createLWE. + """ + + def __init__(self, image_name: str) -> None: + """ + Initialize the CadetFileRunner. + + Parameters + ---------- + image_name : os.PathLike + Name of the docker image to use for CADET + """ + + self.docker_container = image_name + try: + self.client = docker.from_env() + except NameError as e: + raise NameError("Could not import Docker.") from e + + self.image = self.client.images.get(name=image_name) + self._get_cadet_version() + + def run( + self, + simulation: "Cadet", + timeout: Optional[int] = None, + ) -> ReturnInformation: + """ + Run a CADET simulation using the CLI executable. + + Parameters + ---------- + simulation : Cadet + Not used in this runner. + timeout : Optional[int] + Maximum time allowed for the simulation to run, in seconds. + + Raises + ------ + RuntimeError + If the simulation process returns a non-zero exit code. + + Returns + ------- + ReturnInformation + Information about the simulation run. + """ + if simulation.filename is None: + raise ValueError("Filename must be set before run can be used") + + filename = Path(simulation.filename) + + mount = Mount( + source=filename.parent.absolute().as_posix(), + type="bind", + target="/data" + ) + + try: + log = self.client.containers.run( + image=self.image, + mounts=[mount], + command=f"cadet-cli /data/{filename.name}", + stdout=True, + stderr=False + ) + + return_info = ReturnInformation( + return_code=0, + error_message="", + log=log + ) + except ContainerError as e: + return_info = ReturnInformation( + return_code=-1, + error_message=e.stderr, + log="" + ) + + return return_info + + def clear(self) -> None: + """ + Clear the simulation data. + + This method can be extended if any cleanup is required. + """ + pass + + def load_results(self, sim: "Cadet") -> None: + """ + Load the results of the simulation into the provided object. + + Parameters + ---------- + sim : Cadet + The simulation object where results will be loaded. + """ + sim.load(paths=["/meta", "/output"], update=True) + + def _get_cadet_version(self) -> dict: + """ + Get version and branch name of the currently instanced CADET build. + Returns + ------- + dict + Dictionary containing: cadet_version as x.x.x, cadet_branch, cadet_build_type, cadet_commit_hash + Raises + ------ + ValueError + If version and branch name cannot be found in the output string. + RuntimeError + If any unhandled event during running the subprocess occurs. + """ + return_info = self.client.containers.run( + image=self.image, + command="cadet-cli --version" + ) + + version_output = return_info.decode()[:10000] + + version_match = re.search( + r'cadet-cli version ([\d.]+) \((.*) branch\)\n', + version_output + ) + + commit_hash_match = re.search( + "Built from commit (.*)\n", + version_output + ) + + build_variant_match = re.search( + "Build variant (.*)\n", + version_output + ) + + if version_match: + self._cadet_version = version_match.group(1) + self._cadet_branch = version_match.group(2) + self._cadet_commit_hash = commit_hash_match.group(1) + if build_variant_match: + self._cadet_build_type = build_variant_match.group(1) + else: + self._cadet_build_type = None + else: + raise ValueError("CADET version or branch name missing from output.") + + @property + def cadet_version(self) -> str: + return self._cadet_version + + @property + def cadet_branch(self) -> str: + return self._cadet_branch + + @property + def cadet_build_type(self) -> str: + return self._cadet_build_type + + @property + def cadet_commit_hash(self) -> str: + return self._cadet_commit_hash + + @property + def cadet_path(self) -> os.PathLike: + return self.image.tags[0] diff --git a/pyproject.toml b/pyproject.toml index 9419c06..8143388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ testing = [ "pytest", "joblib" ] +docker = [ + "docker" +] [project.urls] "homepage" = "https://github.com/cadet/CADET-Python" @@ -57,4 +60,5 @@ testpaths = ["tests"] markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "local: marks tests as only useful on local installs (deselect with '-m \"not local\"')", + "docker: marks tests as requiring a docker installation (deselect with '-m \"not docker\"')", ] \ No newline at end of file diff --git a/tests/docker_resources/Dockerfile b/tests/docker_resources/Dockerfile new file mode 100644 index 0000000..15fcf28 --- /dev/null +++ b/tests/docker_resources/Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 + +# Comments are provided throughout this file to help you get started. +# If you need more help, visit the Dockerfile reference guide at +# https://docs.docker.com/go/dockerfile-reference/ + +# Want to help us make this template better? Share your feedback here: https://forms.gle/ybq9Krt8jtBL3iCk7 + +ARG MAMBA_VERSION=1.5.8 +FROM mambaorg/micromamba:${MAMBA_VERSION}-focal AS base + +# Prevents Python from writing pyc files. +ENV PYTHONDONTWRITEBYTECODE=1 + +# Keeps Python from buffering stdout and stderr to avoid situations where +# the application crashes without emitting any logs due to buffering. +ENV PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml + +RUN micromamba install -y -n base -f /tmp/environment.yml && \ + micromamba clean --all --yes + +WORKDIR /opt/conda/bin diff --git a/tests/docker_resources/environment.yml b/tests/docker_resources/environment.yml new file mode 100644 index 0000000..740e785 --- /dev/null +++ b/tests/docker_resources/environment.yml @@ -0,0 +1,6 @@ +name: cadet +channels: + - conda-forge +dependencies: + - python + - cadet diff --git a/tests/test_docker_integration.py b/tests/test_docker_integration.py new file mode 100644 index 0000000..df9ee2a --- /dev/null +++ b/tests/test_docker_integration.py @@ -0,0 +1,67 @@ +import os +import docker +import pytest + +from cadet import Cadet +from cadet.runner import CadetDockerRunner +from tests.test_dll import setup_model + +os.chdir("docker_resources") + + +@pytest.fixture +def my_docker_image_tag(): + client = docker.from_env() + + image, logs = client.images.build( + path=".", + tag="cadet-docker-test", + quiet=False, + ) + + # for log in logs: + # print(log) + + return_info = client.containers.run( + image=image, + command="cadet-cli --version" + ) + + if len(return_info.decode()[:10000]) < 1: + raise RuntimeError("Docker build went wrong.") + return image.tags[0] + + +@pytest.mark.local +@pytest.mark.docker +def test_docker_version(my_docker_image_tag): + runner = CadetDockerRunner(my_docker_image_tag) + assert len(runner.cadet_version) > 0 + assert len(runner.cadet_branch) > 0 + assert len(runner.cadet_build_type) > 0 + assert len(runner.cadet_commit_hash) > 0 + +@pytest.mark.local +@pytest.mark.docker +def test_docker_runner(my_docker_image_tag): + model = setup_model(Cadet.autodetect_cadet(), file_name=f"LWE_docker.h5") + runner = CadetDockerRunner(my_docker_image_tag) + runner.run(model) + model.load() + assert hasattr(model.root, "output") + assert hasattr(model.root.output, "solution") + assert hasattr(model.root.output.solution, "unit_000") + + +@pytest.mark.local +@pytest.mark.docker +def test_docker_run(my_docker_image_tag): + simulator = Cadet(docker_container=my_docker_image_tag) + simulator.filename = "LWE_docker.h5" + simulator.save() + model = setup_model(Cadet.autodetect_cadet(), file_name=f"LWE_docker.h5") + simulator.root.input.update(model.root.input) + simulator.run_load() + assert hasattr(simulator.root, "output") + assert hasattr(simulator.root.output, "solution") + assert hasattr(simulator.root.output.solution, "unit_000")