From cade7ae6daa9ad3da5b8336e321c682ac7e02b43 Mon Sep 17 00:00:00 2001
From: Dennis Bader <dennis.bader@gmx.ch>
Date: Sun, 3 Nov 2024 12:18:16 +0100
Subject: [PATCH] Update CI/CD pipelines Part 1 (#2577)

* reduce tests to failing macos

* retry

* retry

* retry

* add back cache file

* add comment

* try different caching

* fix bug

* remove venv

* add back codecov upload and ubuntu os

* add linting, docs, and example checks

* fix papermill missing

* change to exmaples workdir for example checks

* improve caching, update merge workflow

* update merge workflow to use cache

* add update cache step whenever a branch is merged to master
---
 .github/workflows/develop.yml      | 173 ++++++++++++++--------------
 .github/workflows/merge.yml        | 174 +++++++++++++++--------------
 .github/workflows/update-cache.yml |  41 +++++++
 3 files changed, 216 insertions(+), 172 deletions(-)
 create mode 100644 .github/workflows/update-cache.yml

diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index d0320e21d8..834c1a1026 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -9,30 +9,23 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
-
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install Dev Dependencies"
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements/dev.txt
 
-      - name: "4. Lint"
+      - name: "Lint"
         run: |
-          ./gradlew lint
+          pre-commit run --all-files
+
 
   tests:
     runs-on: ${{ matrix.os }}
@@ -43,47 +36,39 @@ jobs:
         flavour: ['all']
 
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python ${{ matrix.python-version }}"
-        uses: actions/setup-python@v1
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
+      - name: "Cache python environment"
+        uses: actions/cache@v4
+        id: pythonenv-cache
         with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
 
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
 
-      - name: "4. Setup pip"
+      - name: "Install Dependencies"
         run: |
-          ./gradlew installPipLatest
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
 
-      - name: "5. Install libomp (for LightGBM)"
+      - name: "Install libomp (for LightGBM)"
         run: |
           ./.github/scripts/libomp-${{ runner.os }}.sh
 
-      - name: "6. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: tests-${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements-latest.txt') }}
-
-      - name: "7. Tests"
+      - name: "Run tests"
         run: |
-          ./gradlew "test_${{matrix.flavour}}"
+          pytest --durations=50 --cov=darts --cov-config=.coveragerc --cov-report=xml darts/tests
 
-      - name: "8. Codecov upload"
+      - name: "Codecov upload"
         if: ${{ matrix.flavour == 'all' }}
         uses: codecov/codecov-action@v4
         with:
@@ -93,45 +78,46 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: "3. Install pandoc"
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
+
+      - name: "Install pandoc"
         run: |
           sudo apt-get install -y pandoc
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "4. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
 
-      - name: "4.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
 
-      - name: "5. Setup pip"
+      - name: "Install libomp (for LightGBM)"
         run: |
-          ./gradlew setupPip
+          ./.github/scripts/libomp-${{ runner.os }}.sh
 
-      - name: "6. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: tests-${{ runner.os }}-3.9-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      - name: "7. Build docs"
+      - name: "Build docs"
         run: |
-          ./gradlew buildDocs
+          make --directory ./docs build-all-docs
 
   check-examples:
     runs-on: ubuntu-latest
@@ -139,27 +125,40 @@ jobs:
       matrix:
         example-name: [03-FFT-examples.ipynb, 04-RNN-examples.ipynb, 00-quickstart.ipynb, 02-data-processing.ipynb, 01-multi-time-series-and-covariates.ipynb]
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
         with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
 
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
+
+      - name: "Install libomp (for LightGBM)"
+        run: |
+          ./.github/scripts/libomp-${{ runner.os }}.sh
+
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      - name: "4. Run examples ${{matrix.example-name}}"
+      - name: "Run example ${{matrix.example-name}}"
+        working-directory: ./examples
         run: |
-          ./gradlew checkExample -PexampleName=${{matrix.example-name}}
+          papermill ${{matrix.example-name}} ${{matrix.example-name}}
diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml
index 16e69f5798..41cf87f7a7 100644
--- a/.github/workflows/merge.yml
+++ b/.github/workflows/merge.yml
@@ -9,30 +9,22 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.10"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.10"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
-
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install Dev Dependencies"
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements/dev.txt
 
-      - name: "4. Lint"
+      - name: "Lint"
         run: |
-          ./gradlew lint
+          pre-commit run --all-files
 
   tests:
     runs-on: ${{ matrix.os }}
@@ -43,40 +35,40 @@ jobs:
         flavour: ['core', 'torch', 'all']
 
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python ${{ matrix.python-version }}"
-        uses: actions/setup-python@v1
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
-
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
-
-      - name: "4. Setup pip"
+      - name: "Setup pip"
         run: |
-          ./gradlew setupPip
+          python -m pip install --upgrade pip
 
-      - name: "5. Install libomp (for LightGBM)"
+      - name: "Install `flavor=${{ matrix.flavor }}` Dependencies"
+        run: |
+          if [ "${{ matrix.flavor }}" == "core" ]; then
+            pip install -r requirements/core.txt -r requirements/dev.txt
+          elif [ "${{ matrix.flavor }}" == "torch" ]; then
+            pip install -r requirements/core.txt -r requirements/torch.txt -r requirements/dev.txt
+          elif [ "${{ matrix.flavor }}" == "all" ]; then
+            pip install -r requirements/dev-all.txt
+          fi
+
+      - name: "Install libomp (for LightGBM)"
         run: |
           ./.github/scripts/libomp-${{ runner.os }}.sh
 
-      - name: "6. Tests"
+      - name: "Run tests"
         run: |
-          ./gradlew "test_${{matrix.flavour}}"
+          if [ "${{ matrix.flavor }}" == "all" ]; then
+            pytest --durations=50 --cov=darts --cov-config=.coveragerc --cov-report=xml darts/tests
+          else
+            pytest --durations=50
 
-      - name: "7. Codecov upload"
+      - name: "Codecov upload"
         if: ${{ matrix.flavour == 'all' }}
         uses: codecov/codecov-action@v4
         with:
@@ -89,72 +81,84 @@ jobs:
       matrix:
         example-name: [00-quickstart.ipynb, 01-multi-time-series-and-covariates.ipynb, 02-data-processing.ipynb, 03-FFT-examples.ipynb, 04-RNN-examples.ipynb, 05-TCN-examples.ipynb, 06-Transformer-examples.ipynb, 07-NBEATS-examples.ipynb, 08-DeepAR-examples.ipynb, 09-DeepTCN-examples.ipynb, 10-Kalman-filter-examples.ipynb, 11-GP-filter-examples.ipynb, 12-Dynamic-Time-Warping-example.ipynb, 13-TFT-examples.ipynb, 15-static-covariates.ipynb, 16-hierarchical-reconciliation.ipynb, 18-TiDE-examples.ipynb, 19-EnsembleModel-examples.ipynb, 20-RegressionModel-examples.ipynb, 21-TSMixer-examples.ipynb, 22-anomaly-detection-examples.ipynb]
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "3. Cache gradle distribution"
-        uses: actions/cache@v2
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
         with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
 
-      - name: "3.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
+
+      - name: "Install libomp (for LightGBM)"
+        run: |
+          ./.github/scripts/libomp-${{ runner.os }}.sh
+
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      # TODO: why is this a matrix? there is no pip caching, and we restart this for each item in the matrix
-      - name: "4. Run examples ${{matrix.example-name}}"
+      - name: "Run example ${{matrix.example-name}}"
+        working-directory: ./examples
         run: |
-          ./gradlew checkExample -PexampleName=${{matrix.example-name}}
+          papermill ${{matrix.example-name}} ${{matrix.example-name}}
 
   docs:
     runs-on: ubuntu-latest
-
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: "3. Install pandoc"
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
+
+      - name: "Install pandoc"
         run: |
           sudo apt-get install -y pandoc
 
-      # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "4. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
 
-      - name: "4.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
 
-      - name: "5. Setup pip"
+      - name: "Install libomp (for LightGBM)"
         run: |
-          ./gradlew setupPip
+          ./.github/scripts/libomp-${{ runner.os }}.sh
 
-      - name: "6. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: release-${{ runner.os }}-3.9-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      - name: "7. Build docs"
+      - name: "Build docs"
         run: |
-          ./gradlew buildDocs
+          make --directory ./docs build-all-docs
diff --git a/.github/workflows/update-cache.yml b/.github/workflows/update-cache.yml
new file mode 100644
index 0000000000..e6cfa2cace
--- /dev/null
+++ b/.github/workflows/update-cache.yml
@@ -0,0 +1,41 @@
+name: update-cache
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  # This workflow updates the python environment cache so that other workflows in different branches have access to it
+  build-cache:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-13, ubuntu-latest]
+        python-version: ['3.9']
+        flavour: ['all']
+
+    steps:
+      - name: "Clone repository"
+        uses: actions/checkout@v4
+
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: "Cache python environment"
+        uses: actions/cache@v4
+        id: pythonenv-cache
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}
+
+      - name: "Setup Pip"
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: "Install Latest Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt