Merge pull request #1207 from ikrommyd/switch-to-pyproject #5527
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| tags: | |
| - v*.*.* | |
| pull_request: | |
| branches: | |
| - master | |
| - backports-v0.7.x | |
| - backports-v0.6.x | |
| # Run daily at 0:01 UTC | |
| schedule: | |
| - cron: '1 0 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-commit: | |
| name: pre-commit | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - uses: pre-commit/[email protected] | |
| with: | |
| extra_args: --hook-stage manual --all-files | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macOS-latest, windows-latest] | |
| java-version: [17] | |
| java-distribution: ["corretto"] | |
| python-version: ["3.9", "3.11"] | |
| name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up JDK ${{ matrix.java-distribution }}/${{ matrix.java-version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: ${{ matrix.java-distribution }} | |
| - name: Set python test settings | |
| run: | | |
| echo "INSTALL_EXTRAS='[dev,parsl,dask,servicex]'" >> $GITHUB_ENV | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Install dependencies (Linux) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| uv pip install --system 'tritonclient[grpc,http]!=2.41.0' | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,parsl,dask]' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Install dependencies (MacOS) | |
| if: matrix.os == 'macOS-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,dask]' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Install dependencies (Windows) | |
| if: matrix.os == 'windows-latest' | |
| run: | | |
| uv pip install --system --upgrade pip setuptools wheel | |
| # mltool installs | |
| # c.f. https://github.com/astral-sh/uv/issues/3437 | |
| python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu | |
| uv pip install --system xgboost | |
| # install checked out coffea | |
| uv pip install --system -q '.[dev,dask]' 'numpy<2' --upgrade | |
| uv pip list --system | |
| java -version | |
| - name: Start triton server with example model | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-pyt-python-py3 tritonserver --model-repository=/models | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest --cov-report=xml --cov=coffea --deselect=test_taskvine | |
| - name: Upload codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: codecov/codecov-action@v5 | |
| - name: Install graphviz | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: ts-graphviz/setup-graphviz@v2 | |
| - name: Install pandoc | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: r-lib/actions/setup-pandoc@v2 | |
| - name: Build documentation | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| run: | | |
| cd docs && make html | |
| touch build/html/.nojekyll | |
| - name: Deploy documentation | |
| if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11 | |
| uses: crazy-max/ghaction-github-pages@v4 | |
| with: | |
| target_branch: gh-pages | |
| build_dir: docs/build/html | |
| env: | |
| GH_PAT: ${{ secrets.GITHUB_OAUTH }} | |
| testwq: | |
| runs-on: ubuntu-latest | |
| needs: pre-commit | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: test coffea-workqueue | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Conda | |
| uses: conda-incubator/[email protected] | |
| with: | |
| auto-update-conda: true | |
| python-version: ${{ matrix.python-version }} | |
| - name: Test work_queue | |
| shell: bash -l {0} | |
| run: | | |
| conda create --yes --name coffea-env -c conda-forge python=${{ matrix.python-version }} ndcctools dill conda-pack conda 'setuptools<71' | |
| conda activate coffea-env | |
| python -m pip install --ignore-installed . | |
| cd tests | |
| conda-pack --output coffea-env.tar.gz | |
| python wq.py coffea-env.tar.gz | |
| # testskyhookjob: | |
| # runs-on: ubuntu-latest | |
| # needs: pre-commit | |
| # name: test coffea-skyhook-job | |
| # | |
| # steps: | |
| # - uses: actions/checkout@3 | |
| # - name: Test Coffea Skyhook Bindings | |
| # shell: bash -l {0} | |
| # run: | | |
| # docker build -t coffea-skyhook-test \ | |
| # --file docker/skyhook/Dockerfile \ | |
| # . | |
| # docker run \ | |
| # -v $(pwd):/w \ | |
| # -w /w \ | |
| # -e IS_CI=true \ | |
| # --privileged \ | |
| # coffea-skyhook-test \ | |
| # ./docker/skyhook/script.sh | |
| release: | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| needs: [test, testwq] | |
| permissions: | |
| id-token: write | |
| attestations: write | |
| contents: read | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| name: deploy release | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build package for PyPI | |
| run: | | |
| pipx run hatch build -t sdist -t wheel | |
| - name: Verify the distribution | |
| run: pipx run twine check --strict dist/* | |
| - name: Generate artifact attestation for sdist and wheel | |
| uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4 | |
| with: | |
| subject-path: "dist/coffea-*" | |
| - name: Publish package to PyPI | |
| uses: pypa/[email protected] | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI_TOKEN }} | |
| pass: | |
| needs: [test, testwq] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "All jobs passed" |