Skip to content

Commit

Permalink
Update CI/CD pipelines Part 1 (#2577)
Browse files Browse the repository at this point in the history 
* reduce tests to failing macos

* retry

* retry

* retry

* add back cache file

* add comment

* try different caching

* fix bug

* remove venv

* add back codecov upload and ubuntu os

* add linting, docs, and example checks

* fix papermill missing

* change to exmaples workdir for example checks

* improve caching, update merge workflow

* update merge workflow to use cache

* add update cache step whenever a branch is merged to master
  • Loading branch information
@dennisbader
dennisbader authored Nov 3, 2024
1 parent ad93612 commit cade7ae
Show file tree
Hide file tree
Showing 3 changed files with 216 additions and 172 deletions.
173 changes: 86 additions & 87 deletions .github/workflows/develop.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,23 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: "1. Clone repository"
uses: actions/checkout@v2
- name: "Clone repository"
uses: actions/checkout@v4

- name: "2. Set up Python 3.9"
uses: actions/setup-python@v1
- name: "Set up Python 3.9"
uses: actions/setup-python@v5
with:
python-version: '3.9'

# downloading gradle multiple times in parallel can yield to connection errors
- name: "3. Cache gradle distribution"
uses: actions/cache@v2
with:
path: ~/.gradle/wrapper/dists
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}

- name: "3.1 Cache gradle packages"
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
- name: "Install Dev Dependencies"
run: |
python -m pip install --upgrade pip
pip install -r requirements/dev.txt
- name: "4. Lint"
- name: "Lint"
run: |
./gradlew lint
pre-commit run --all-files
tests:
runs-on: ${{ matrix.os }}
Expand All @@ -43,47 +36,39 @@ jobs:
flavour: ['all']

steps:
- name: "1. Clone repository"
uses: actions/checkout@v2
- name: "Clone repository"
uses: actions/checkout@v4

- name: "2. Set up Python ${{ matrix.python-version }}"
uses: actions/setup-python@v1
- name: "Set up Python ${{ matrix.python-version }}"
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

# downloading gradle multiple times in parallel can yield to connection errors
- name: "3. Cache gradle distribution"
uses: actions/cache@v2
- name: "Cache python environment"
uses: actions/cache@v4
id: pythonenv-cache
with:
path: ~/.gradle/wrapper/dists
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}

- name: "3.1 Cache gradle packages"
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
- name: "Setup Pip"
run: |
python -m pip install --upgrade pip
- name: "4. Setup pip"
- name: "Install Dependencies"
run: |
./gradlew installPipLatest
# install latest dependencies (potentially updating cached dependencies)
pip install -U -r requirements/dev-all.txt
- name: "5. Install libomp (for LightGBM)"
- name: "Install libomp (for LightGBM)"
run: |
./.github/scripts/libomp-${{ runner.os }}.sh
- name: "6. Attach cache for pip"
uses: actions/cache@v1
id: cache
with:
path: ~/.cache/pip
key: tests-${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('requirements-latest.txt') }}

- name: "7. Tests"
- name: "Run tests"
run: |
./gradlew "test_${{matrix.flavour}}"
pytest --durations=50 --cov=darts --cov-config=.coveragerc --cov-report=xml darts/tests
- name: "8. Codecov upload"
- name: "Codecov upload"
if: ${{ matrix.flavour == 'all' }}
uses: codecov/codecov-action@v4
with:
Expand All @@ -93,73 +78,87 @@ jobs:
docs:
runs-on: ubuntu-latest
steps:
- name: "1. Clone repository"
uses: actions/checkout@v2
- name: "Clone repository"
uses: actions/checkout@v4

- name: "2. Set up Python 3.9"
uses: actions/setup-python@v1
- name: "Set up Python 3.9"
uses: actions/setup-python@v5
with:
python-version: '3.9'

- name: "3. Install pandoc"
# only restore cache but do not upload
- name: "Restore cached python environment"
uses: actions/cache/restore@v4
id: pythonenv-cache
with:
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}

- name: "Install pandoc"
run: |
sudo apt-get install -y pandoc
# downloading gradle multiple times in parallel can yield to connection errors
- name: "4. Cache gradle distribution"
uses: actions/cache@v2
with:
path: ~/.gradle/wrapper/dists
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
- name: "Setup Pip"
run: |
python -m pip install --upgrade pip
- name: "4.1 Cache gradle packages"
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
- name: "Install Dependencies"
run: |
# install latest dependencies (potentially updating cached dependencies)
pip install -U -r requirements/dev-all.txt
- name: "5. Setup pip"
- name: "Install libomp (for LightGBM)"
run: |
./gradlew setupPip
./.github/scripts/libomp-${{ runner.os }}.sh
- name: "6. Attach cache for pip"
uses: actions/cache@v1
id: cache
with:
path: ~/.cache/pip
key: tests-${{ runner.os }}-3.9-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
- name: "Install Locally"
run: |
pip install .
- name: "7. Build docs"
- name: "Build docs"
run: |
./gradlew buildDocs
make --directory ./docs build-all-docs
check-examples:
runs-on: ubuntu-latest
strategy:
matrix:
example-name: [03-FFT-examples.ipynb, 04-RNN-examples.ipynb, 00-quickstart.ipynb, 02-data-processing.ipynb, 01-multi-time-series-and-covariates.ipynb]
steps:
- name: "1. Clone repository"
uses: actions/checkout@v2
- name: "Clone repository"
uses: actions/checkout@v4

- name: "2. Set up Python 3.9"
uses: actions/setup-python@v1
- name: "Set up Python 3.9"
uses: actions/setup-python@v5
with:
python-version: '3.9'

# downloading gradle multiple times in parallel can yield to connection errors
- name: "3. Cache gradle distribution"
uses: actions/cache@v2
# only restore cache but do not upload
- name: "Restore cached python environment"
uses: actions/cache/restore@v4
id: pythonenv-cache
with:
path: ~/.gradle/wrapper/dists
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
path: ${{ env.pythonLocation }}
key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt') }}

- name: "3.1 Cache gradle packages"
uses: actions/cache@v2
with:
path: ~/.gradle/caches
key: tests-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
- name: "Setup Pip"
run: |
python -m pip install --upgrade pip
- name: "Install Dependencies"
run: |
# install latest dependencies (potentially updating cached dependencies)
pip install -U -r requirements/dev-all.txt
- name: "Install libomp (for LightGBM)"
run: |
./.github/scripts/libomp-${{ runner.os }}.sh
- name: "Install Locally"
run: |
pip install .
- name: "4. Run examples ${{matrix.example-name}}"
- name: "Run example ${{matrix.example-name}}"
working-directory: ./examples
run: |
./gradlew checkExample -PexampleName=${{matrix.example-name}}
papermill ${{matrix.example-name}} ${{matrix.example-name}}
Loading

0 comments on commit cade7ae

Please sign in to comment.