feat: handle conda env ymls (#63)

* feat: handle conda env ymls

* Add conda with yml tests

* Add pyyaml dependency

* PR comments

* Remove env_yml_str from CondaEnvironment properties

.github/workflows/test.yml

@@ -50,4 +50,5 @@ jobs:
     - name: Test
       run: |
         export ISOLATE_PYENV_EXECUTABLE=pyenv/bin/pyenv
+        export AGENT_REQUIREMENTS_TXT=tools/agent_requirements.txt
         python -m pytest

pyproject.toml

@@ -13,6 +13,7 @@ rich = ">=12.0"
 grpcio = ">=1.49"
 protobuf = "*"
 tblib = "^1.7.0"
+PyYAML = "^6.0"
 
 [tool.poetry.extras]
 grpc = []

src/isolate/backends/conda.py

@@ -4,6 +4,8 @@
 import os
 import shutil
 import subprocess
+import tempfile
+import yaml
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, ClassVar, Dict, List, Optional
@@ -33,24 +35,38 @@ class CondaEnvironment(BaseEnvironment[Path]):
 
     packages: List[str] = field(default_factory=list)
     python_version: Optional[str] = None
+    env_dict: Optional[Dict[str, Any]] = None
 
     @classmethod
     def from_config(
         cls,
         config: Dict[str, Any],
         settings: IsolateSettings = DEFAULT_SETTINGS,
     ) -> BaseEnvironment:
+        if config.get('env_dict') and config.get('env_yml_str'):
+            raise EnvironmentCreationError("Either env_dict or env_yml_str can be provided, not both!")
+        if config.get('env_yml_str'):
+            config['env_dict'] = yaml.safe_load(config['env_yml_str'])
+            del config['env_yml_str']
         environment = cls(**config)
         environment.apply_settings(settings)
         return environment
 
     @property
     def key(self) -> str:
+        if self.env_dict:
+            return sha256_digest_of(str(self._compute_dependencies()))
         return sha256_digest_of(*self._compute_dependencies())
 
-    def _compute_dependencies(self) -> List[str]:
-        user_dependencies = self.packages.copy()
+    def _compute_dependencies(self) -> List[Any]:
+        if self.env_dict:
+            user_dependencies = self.env_dict.get('dependencies', []).copy()
+        else:
+            user_dependencies = self.packages.copy()
         for raw_requirement in user_dependencies:
+            # It could be 'pip': [...]
+            if type(raw_requirement) is dict:
+                continue
             # Get rid of all whitespace characters (python = 3.8 becomes python=3.8)
             raw_requirement = raw_requirement.replace(" ", "")
             if not raw_requirement.startswith("python"):
@@ -87,28 +103,46 @@ def create(self) -> Path:
             if env_path.exists():
                 return env_path
 
-            # Since our agent needs Python to be installed (at very least)
-            # we need to make sure that the base environment is created with
-            # the same Python version as the one that is used to run the
-            # isolate agent.
-            dependencies = self._compute_dependencies()
-
-            self.log(f"Creating the environment at '{env_path}'")
-            self.log(f"Installing packages: {', '.join(dependencies)}")
-
-            try:
-                self._run_conda(
-                    "create",
-                    "--yes",
-                    "--prefix",
-                    env_path,
-                    *dependencies,
-                )
-            except subprocess.SubprocessError as exc:
-                raise EnvironmentCreationError("Failure during 'conda create'") from exc
-
-        self.log(f"New environment cached at '{env_path}'")
-        return env_path
+            if self.env_dict:
+                self.env_dict['dependencies'] = self._compute_dependencies()
+                with tempfile.NamedTemporaryFile(mode='w', suffix='.yml') as tf:
+                    yaml.dump(self.env_dict, tf)
+                    tf.flush()
+                    try:
+                        self._run_conda(
+                            "env",
+                            "create",
+                            "-f",
+                            tf.name,
+                            "--prefix",
+                            env_path
+                        )
+                    except subprocess.SubprocessError as exc:
+                        raise EnvironmentCreationError("Failure during 'conda create'") from exc
+
+            else:
+                # Since our agent needs Python to be installed (at very least)
+                # we need to make sure that the base environment is created with
+                # the same Python version as the one that is used to run the
+                # isolate agent.
+                dependencies = self._compute_dependencies()
+
+                self.log(f"Creating the environment at '{env_path}'")
+                self.log(f"Installing packages: {', '.join(dependencies)}")
+
+                try:
+                    self._run_conda(
+                        "create",
+                        "--yes",
+                        "--prefix",
+                        env_path,
+                        *dependencies,
+                    )
+                except subprocess.SubprocessError as exc:
+                    raise EnvironmentCreationError("Failure during 'conda create'") from exc
+
+            self.log(f"New environment cached at '{env_path}'")
+            return env_path
 
     def destroy(self, connection_key: Path) -> None:
         with self.settings.cache_lock_for(connection_key):

tests/test_backends.py

@@ -380,6 +380,11 @@ class TestConda(GenericEnvironmentTests):
         "new-python": {
             "python_version": "3.10",
         },
+        "yml-with-isolate": {
+            "env_yml_str": 'name: test\n' + \
+            'channels:\n  - defaults\n' + \
+            'dependencies:\n  - pip:\n    - isolate==0.7.1\n    - pyjokes==0.5.0\n'
+        }
     }
     creation_entry_point = ("subprocess.check_call", subprocess.SubprocessError)
 
@@ -431,6 +436,10 @@ def test_fail_when_user_overwrites_python(
         ):
             environment.create()
 
+    def test_environment_with_yml(self, tmp_path):
+        environment = self.get_project_environment(tmp_path, "yml-with-isolate")
+        connection_key = environment.create()
+        assert self.get_example_version(environment, connection_key) == "0.5.0"
 
 def test_local_python_environment():
     """Since 'local' environment does not support installation of extra dependencies

tools/Dockerfile

@@ -9,6 +9,27 @@
 FROM python:3.9
 
 RUN apt-get update && apt-get install -y git
+
+RUN mkdir -p /opt
+RUN git clone https://github.com/pyenv/pyenv --branch v2.3.6 --depth=1 /opt/pyenv
+# TODO: Investigate whether we can leverage the compiled pyenv extension.
+ENV ISOLATE_PYENV_EXECUTABLE=/opt/pyenv/bin/pyenv
+
+#### CONDA ####
+# Will copy from existing Docker image
+COPY --from=continuumio/miniconda3:4.12.0 /opt/conda /opt/conda
+
+ENV PATH=/opt/conda/bin:$PATH
+ENV ISOLATE_CONDA_HOME=/opt/conda/bin
+
+# Usage examples
+RUN set -ex && \
+    conda config --set always_yes yes --set changeps1 no && \
+    conda info -a && \
+    conda config --add channels conda-forge && \
+    conda install --quiet --freeze-installed -c main conda-pack
+
+#### END CONDA ####
 RUN pip install --upgrade pip virtualenv wheel poetry-core
 
 # Since system-level debian does not comply with
@@ -22,9 +43,10 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 COPY tools/requirements.txt /tmp/requirements.txt
 RUN pip install -r /tmp/requirements.txt
 
-COPY . .
-RUN pip install .
+COPY . /isolate
+RUN pip install /isolate[server]
 
 ENV ISOLATE_INHERIT_FROM_LOCAL=1
+ENV AGENT_REQUIREMENTS_TXT=/isolate/tools/agent_requirements.txt
 
 CMD ["python", "-m", "isolate.server.server"]

tools/agent_requirements.txt

@@ -0,0 +1,3 @@
+/isolate[server]
+dill==0.3.5.1
+google-cloud-storage==2.6.0