Add a CadetDockerRunner and tests

cadet/cadet.py

@@ -9,7 +9,7 @@
 from addict import Dict
 
 from cadet.h5 import H5
-from cadet.runner import CadetRunnerBase, CadetCLIRunner, ReturnInformation
+from cadet.runner import CadetRunnerBase, CadetCLIRunner, ReturnInformation, CadetDockerRunner
 from cadet.cadet_dll import CadetDLLRunner
 
 
@@ -184,7 +184,10 @@ class Cadet(H5, metaclass=CadetMeta):
         Stores the information returned after a simulation run.
     """
 
-    def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, *data):
+    def __init__(
+            self, install_path: Optional[Path] = None, use_dll: bool = False,
+            docker_container: Optional[str] = None, *data
+    ):
         """
         Initialize a new instance of the Cadet class.
         Priority order of install_paths is:
@@ -194,6 +197,12 @@ def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, *
 
         Parameters
         ----------
+        install_path : Optional[Path]
+            The root directory of the CADET installation.
+        use_dll : Optional[bool]
+            Indicates whether to use a DLL or a CLI executable.
+        docker_container : Optional[str]
+            Label of the docker container to use.
         *data : tuple
             Additional data to be passed to the H5 base class initialization.
         """
@@ -208,6 +217,12 @@ def __init__(self, install_path: Optional[Path] = None, use_dll: bool = False, *
             self.install_path = install_path  # This will set _cadet_dll_runner and _cadet_cli_runner
             return
 
+        # If we get a docker_container, we use the docker container
+        if docker_container is not None:
+            self._cadet_docker_runner: Optional[CadetDockerRunner] = CadetDockerRunner(
+                docker_container
+            )
+
         # If _cadet_cli_runner_class has been set in the Meta Class, use them, else instantiate Nones
         if hasattr(self, "cadet_cli_path") and self.cadet_cli_path is not None:
             self._cadet_cli_runner: Optional[CadetCLIRunner] = CadetCLIRunner(

cadet/runner.py

@@ -1,12 +1,18 @@
 import os
-import pathlib
 import re
 import subprocess
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Optional
 
+try:
+    import docker
+    from docker.types import Mount
+    from docker.errors import ContainerError
+except ImportError:
+    pass
+
 
 @dataclass
 class ReturnInformation:
@@ -255,3 +261,178 @@ def cadet_commit_hash(self) -> str:
     @property
     def cadet_path(self) -> os.PathLike:
         return self._cadet_path
+
+
+class CadetDockerRunner(CadetRunnerBase):
+    """
+    Docker-based CADET runner.
+
+    This class runs CADET simulations using a command-line interface (CLI) executable that is
+    contained inside a Docker container. This expects the Docker container to have the
+    root of the installation as the working folder. This gives us access to the cadet-cli and
+    tools such as createLWE.
+    """
+
+    def __init__(self, image_name: str) -> None:
+        """
+        Initialize the CadetFileRunner.
+
+        Parameters
+        ----------
+        image_name : os.PathLike
+            Name of the docker image to use for CADET
+        """
+
+        self.docker_container = image_name
+        try:
+            self.client = docker.from_env()
+        except NameError as e:
+            raise NameError("Could not import Docker.") from e
+
+        self.image = self.client.images.get(name=image_name)
+        self._get_cadet_version()
+
+    def run(
+            self,
+            simulation: "Cadet",
+            timeout: Optional[int] = None,
+    ) -> ReturnInformation:
+        """
+        Run a CADET simulation using the CLI executable.
+
+        Parameters
+        ----------
+        simulation : Cadet
+            Not used in this runner.
+        timeout : Optional[int]
+            Maximum time allowed for the simulation to run, in seconds.
+
+        Raises
+        ------
+        RuntimeError
+            If the simulation process returns a non-zero exit code.
+
+        Returns
+        -------
+        ReturnInformation
+            Information about the simulation run.
+        """
+        if simulation.filename is None:
+            raise ValueError("Filename must be set before run can be used")
+
+        filename = Path(simulation.filename)
+
+        mount = Mount(
+            source=filename.parent.absolute().as_posix(),
+            type="bind",
+            target="/data"
+        )
+
+        try:
+            log = self.client.containers.run(
+                image=self.image,
+                mounts=[mount],
+                command=f"cadet-cli /data/{filename.name}",
+                stdout=True,
+                stderr=False
+            )
+
+            return_info = ReturnInformation(
+                return_code=0,
+                error_message="",
+                log=log
+            )
+        except ContainerError as e:
+            return_info = ReturnInformation(
+                return_code=-1,
+                error_message=e.stderr,
+                log=""
+            )
+
+        return return_info
+
+    def clear(self) -> None:
+        """
+        Clear the simulation data.
+
+        This method can be extended if any cleanup is required.
+        """
+        pass
+
+    def load_results(self, sim: "Cadet") -> None:
+        """
+        Load the results of the simulation into the provided object.
+
+        Parameters
+        ----------
+        sim : Cadet
+            The simulation object where results will be loaded.
+        """
+        sim.load(paths=["/meta", "/output"], update=True)
+
+    def _get_cadet_version(self) -> dict:
+        """
+        Get version and branch name of the currently instanced CADET build.
+        Returns
+        -------
+        dict
+            Dictionary containing: cadet_version as x.x.x, cadet_branch, cadet_build_type, cadet_commit_hash
+        Raises
+        ------
+        ValueError
+            If version and branch name cannot be found in the output string.
+        RuntimeError
+            If any unhandled event during running the subprocess occurs.
+        """
+        return_info = self.client.containers.run(
+            image=self.image,
+            command="cadet-cli --version"
+        )
+
+        version_output = return_info.decode()[:10000]
+
+        version_match = re.search(
+            r'cadet-cli version ([\d.]+) \((.*) branch\)\n',
+            version_output
+        )
+
+        commit_hash_match = re.search(
+            "Built from commit (.*)\n",
+            version_output
+        )
+
+        build_variant_match = re.search(
+            "Build variant (.*)\n",
+            version_output
+        )
+
+        if version_match:
+            self._cadet_version = version_match.group(1)
+            self._cadet_branch = version_match.group(2)
+            self._cadet_commit_hash = commit_hash_match.group(1)
+            if build_variant_match:
+                self._cadet_build_type = build_variant_match.group(1)
+            else:
+                self._cadet_build_type = None
+        else:
+            raise ValueError("CADET version or branch name missing from output.")
+
+    @property
+    def cadet_version(self) -> str:
+        return self._cadet_version
+
+    @property
+    def cadet_branch(self) -> str:
+        return self._cadet_branch
+
+    @property
+    def cadet_build_type(self) -> str:
+        return self._cadet_build_type
+
+    @property
+    def cadet_commit_hash(self) -> str:
+        return self._cadet_commit_hash
+
+    @property
+    def cadet_path(self) -> os.PathLike:
+        return self.image.tags[0]

pyproject.toml

@@ -37,6 +37,9 @@ testing = [
     "pytest",
     "joblib"
 ]
+docker = [
+    "docker"
+]
 
 [project.urls]
 "homepage" = "https://github.com/cadet/CADET-Python"
@@ -57,4 +60,5 @@ testpaths = ["tests"]
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "local: marks tests as only useful on local installs (deselect with '-m \"not local\"')",
+    "docker: marks tests as requiring a docker installation (deselect with '-m \"not docker\"')",
 ]

tests/docker_resources/Dockerfile

@@ -0,0 +1,26 @@
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/go/dockerfile-reference/
+
+# Want to help us make this template better? Share your feedback here: https://forms.gle/ybq9Krt8jtBL3iCk7
+
+ARG MAMBA_VERSION=1.5.8
+FROM mambaorg/micromamba:${MAMBA_VERSION}-focal AS base
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY --chown=$MAMBA_USER:$MAMBA_USER environment.yml /tmp/environment.yml
+
+RUN micromamba install -y -n base -f /tmp/environment.yml && \
+    micromamba clean --all --yes
+
+WORKDIR /opt/conda/bin

tests/docker_resources/environment.yml

@@ -0,0 +1,6 @@
+name: cadet
+channels:
+  - conda-forge
+dependencies:
+  - python
+  - cadet

tests/test_docker_integration.py

@@ -0,0 +1,67 @@
+import os
+import docker
+import pytest
+
+from cadet import Cadet
+from cadet.runner import CadetDockerRunner
+from tests.test_dll import setup_model
+
+os.chdir("docker_resources")
+
+
+@pytest.fixture
+def my_docker_image_tag():
+    client = docker.from_env()
+
+    image, logs = client.images.build(
+        path=".",
+        tag="cadet-docker-test",
+        quiet=False,
+    )
+
+    # for log in logs:
+    #     print(log)
+
+    return_info = client.containers.run(
+        image=image,
+        command="cadet-cli --version"
+    )
+
+    if len(return_info.decode()[:10000]) < 1:
+        raise RuntimeError("Docker build went wrong.")
+    return image.tags[0]
+
+
+@pytest.mark.local
+@pytest.mark.docker
+def test_docker_version(my_docker_image_tag):
+    runner = CadetDockerRunner(my_docker_image_tag)
+    assert len(runner.cadet_version) > 0
+    assert len(runner.cadet_branch) > 0
+    assert len(runner.cadet_build_type) > 0
+    assert len(runner.cadet_commit_hash) > 0
+
+@pytest.mark.local
+@pytest.mark.docker
+def test_docker_runner(my_docker_image_tag):
+    model = setup_model(Cadet.autodetect_cadet(), file_name=f"LWE_docker.h5")
+    runner = CadetDockerRunner(my_docker_image_tag)
+    runner.run(model)
+    model.load()
+    assert hasattr(model.root, "output")
+    assert hasattr(model.root.output, "solution")
+    assert hasattr(model.root.output.solution, "unit_000")
+
+
+@pytest.mark.local
+@pytest.mark.docker
+def test_docker_run(my_docker_image_tag):
+    simulator = Cadet(docker_container=my_docker_image_tag)
+    simulator.filename = "LWE_docker.h5"
+    simulator.save()
+    model = setup_model(Cadet.autodetect_cadet(), file_name=f"LWE_docker.h5")
+    simulator.root.input.update(model.root.input)
+    simulator.run_load()
+    assert hasattr(simulator.root, "output")
+    assert hasattr(simulator.root.output, "solution")
+    assert hasattr(simulator.root.output.solution, "unit_000")