CI/CD #3958
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| tags: | |
| - v*.*.* | |
| pull_request: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| # Run daily at 0:01 UTC | |
| schedule: | |
| - cron: '1 0 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| name: pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v4 | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --hook-stage manual --all-files | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macOS-latest, windows-latest] | |
| java-version: [17] | |
| java-distribution: ["corretto"] | |
| python-version: ["3.8", "3.11"] | |
| name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up JDK ${{ matrix.java-distribution }}/${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: ${{ matrix.java-distribution }} | |
| - name: Set python test settings | |
| run: | | |
| echo "INSTALL_EXTRAS='[dev,parsl,dask,servicex]'" >> $GITHUB_ENV | |
| - name: Install dependencies (Linux) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| # mltool installs | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| python -m pip install xgboost | |
| python -m pip install tritonclient[grpc,http] | |
| # install checked out coffea | |
| python -m pip install -q -e '.[dev,parsl,dask,spark]' | |
| python -m pip list | |
| java -version | |
| - name: Install dependencies (MacOS) | |
| if: matrix.os == 'macOS-latest' | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| # mltool installs | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| python -m pip install xgboost | |
| # install checked out coffea | |
| python -m pip install -q -e '.[dev,dask,spark]' | |
| python -m pip list | |
| java -version | |
| - name: Install dependencies (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| # mltool installs | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| python -m pip install xgboost | |
| # install checked out coffea | |
| python -m pip install -q -e '.[dev,dask]' | |
| python -m pip list | |
| java -version | |
| - name: Start triton server with example model | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-py3 tritonserver --model-repository=/models | |
| - name: Test with pytest | |
| run: | | |
| pytest --cov-report=xml --cov=coffea tests | |
| - name: Upload codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: codecov/codecov-action@v3 | |
| - name: Install graphviz | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: ts-graphviz/setup-graphviz@v1 | |
| - name: Install pandoc | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: r-lib/actions/setup-pandoc@v2 | |
| - name: Build documentation | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| run: | | |
| cd docs && make html | |
| touch build/html/.nojekyll | |
| - name: Deploy documentation | |
| if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: crazy-max/ghaction-github-pages@v4 | |
| with: | |
| target_branch: gh-pages | |
| build_dir: docs/build/html | |
| env: | |
| GH_PAT: ${{ secrets.GITHUB_OAUTH }} | |
| testwq: | |
| runs-on: ubuntu-latest | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: test coffea-workqueue | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Conda | |
| uses: conda-incubator/setup-miniconda@v2 | |
| env: | |
| ACTIONS_ALLOW_UNSECURE_COMMANDS: true | |
| with: | |
| auto-update-conda: true | |
| python-version: ${{ matrix.python-version }} | |
| channels: conda-forge | |
| - name: Test work_queue | |
| shell: bash -l {0} | |
| run: | | |
| conda create --yes --name coffea-env -c conda-forge python=${{ matrix.python-version }} ndcctools dill conda-pack conda | |
| conda activate coffea-env | |
| python -m pip install --ignore-installed . | |
| cd tests | |
| conda-pack --output coffea-env.tar.gz | |
| python wq.py coffea-env.tar.gz | |
| # testskyhookjob: | |
| # runs-on: ubuntu-latest | |
| # needs: pre-commit | |
| # name: test coffea-skyhook-job | |
| # | |
| # steps: | |
| # - uses: actions/[email protected] | |
| # - name: Test Coffea Skyhook Bindings | |
| # shell: bash -l {0} | |
| # run: | | |
| # docker build -t coffea-skyhook-test \ | |
| # --file docker/skyhook/Dockerfile \ | |
| # . | |
| # docker run \ | |
| # -v $(pwd):/w \ | |
| # -w /w \ | |
| # -e IS_CI=true \ | |
| # --privileged \ | |
| # coffea-skyhook-test \ | |
| # ./docker/skyhook/script.sh | |
| release: | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| needs: [test, testwq] | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: deploy release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build package for PyPI | |
| run: | | |
| python -m pip install pip hatch --upgrade | |
| python -m hatch build -t sdist -t wheel | |
| - name: Publish package to PyPI | |
| uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| pass: | |
| needs: [test, testwq] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All jobs passed" |