From 3b121dc55280f73d77217c9c64373cf0bc200646 Mon Sep 17 00:00:00 2001
From: Romain Hugonnet <romain.hugonnet@gmail.com>
Date: Fri, 13 Oct 2023 16:35:23 -0700
Subject: [PATCH] Test

---
 .github/scripts/generate_yml_env_fixed_py.py | 58 ++++++++++----------
 .github/workflows/python-package.yml         | 16 +++++-
 2 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/.github/scripts/generate_yml_env_fixed_py.py b/.github/scripts/generate_yml_env_fixed_py.py
index 6951c301..f30b59b2 100644
--- a/.github/scripts/generate_yml_env_fixed_py.py
+++ b/.github/scripts/generate_yml_env_fixed_py.py
@@ -1,54 +1,52 @@
 import argparse
-import os
 
 import yaml  # type: ignore
 
 
-def environment_yml_nopy(fn_env: str, print_dep: str = "both") -> None:
+def environment_yml_nopy(fn_env: str, py_version: float, add_deps: list[str] = None) -> None:
     """
-    List dependencies in environment.yml without python version for setup of continuous integration.
+    Generate temporary environment-py3.XX.yml files forcing python versions for setup of continuous integration.
 
     :param fn_env: Filename path to environment.yml
-    :param print_dep: Whether to print conda differences "conda", pip differences "pip" or both.
+    :param py_version: Python version to force.
+    :param add_deps: Additional dependencies to solve for directly (for instance graphviz fails with mamba update).
     """
 
     # Load the yml as dictionary
     yaml_env = yaml.safe_load(open(fn_env))
     conda_dep_env = list(yaml_env["dependencies"])
 
-    if isinstance(conda_dep_env[-1], dict):
-        pip_dep_env = list(conda_dep_env.pop()["pip"])
-    else:
-        pip_dep_env = ["None"]
+    # Force python version
+    conda_dep_env_forced_py = ["python="+str(py_version) if "python" in dep else dep for dep in conda_dep_env]
 
-    conda_dep_env_without_python = [dep for dep in conda_dep_env if "python" not in dep]
+    # Optionally, add other dependencies
+    if add_deps is not None:
+        conda_dep_env_forced_py.extend(add_deps)
 
-    # Join the lists
-    joined_list_conda_dep = " ".join(conda_dep_env_without_python)
-    joined_list_pip_dep = " ".join(pip_dep_env)
+    # Copy back to new yaml dict
+    yaml_out = yaml_env.copy()
+    yaml_out["dependencies"] = conda_dep_env_forced_py
 
-    # Print to be captured in bash
-    if print_dep == "both":
-        print(joined_list_conda_dep)
-        print(joined_list_pip_dep)
-    elif print_dep == "conda":
-        print(joined_list_conda_dep)
-    elif print_dep == "pip":
-        print(joined_list_pip_dep)
-    else:
-        raise ValueError('The argument "print_dep" can only be "conda", "pip" or "both".')
+    with open("environment-ci-py"+str(py_version)+".yml", 'w') as outfile:
+        yaml.dump(yaml_out, outfile, default_flow_style=False)
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Get environment list without python version.")
-    parser.add_argument("fn_env", metavar="fn_env", type=str, help="Path to the environment file.")
+    parser = argparse.ArgumentParser(description="Generate environment files for CI with fixed python versions.")
+    parser.add_argument("fn_env", metavar="fn_env", type=str, help="Path to the generic environment file.")
     parser.add_argument(
-        "--p",
-        dest="print_dep",
-        default="both",
+        "--pyv",
+        dest="py_versions",
+        default=3.9,
+        type=list,
+        help="List of Python versions to force.",
+    )
+    parser.add_argument(
+        "--add",
+        dest="add_deps",
+        default=None,
         type=str,
-        help="Whether to print conda dependencies, pip ones, or both.",
+        help="List of dependencies to add.",
     )
-
     args = parser.parse_args()
-    environment_yml_nopy(fn_env=args.fn_env, print_dep=args.print_dep)
+    environment_yml_nopy(fn_env=args.fn_env, py_versions=args.py_versions, add_deps=args.add_deps.split(","))
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 583bd91f..5a9a5d3c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -37,8 +37,7 @@ jobs:
         use-mamba: true
         channel-priority: strict
         activate-environment: xdem-dev
-        environment-file: environment.yml
-        python-version: ${{ matrix.python-version }}
+        python-version:
 
     - name: Get month for resetting cache
       id: get-date
@@ -54,6 +53,19 @@ jobs:
         CACHE_NUMBER: 0 # Increase this value to reset cache if environment.yml has not changed
       id: cache
 
+    # The trick below is necessary because the generic environment file does not specify a Python version, and ONLY
+    # "conda env update" CAN BE USED WITH CACHING, which upgrades the Python version when using the base environment
+    # (we add "graphviz" from dev-environment to solve all dependencies at once, at graphviz relies on image
+    # processing packages very much like geo-packages; not a problem for docs, dev installs where all is done at once)
+    - name: Install base environment with a fixed Python version
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+        python .github/scripts/generate_yml_env_fixed_py.py --pyv ${{ matrix.python-version }} --add "graphviz,opencv"
+        mamba env update -n xdem-dev -f environment-ci-py${{ matrix.python-version }}.yml
+
+    - name: Install project
+      run: pip install -e . --no-dependencies
+
     # This steps allows us to check the "import xdem" with the base environment provided to users, before adding
     # development-specific dependencies by differencing the env and dev-env yml files
     - name: Check import works with base environment