CI (Python) #276
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI (Python) | |
| on: | |
| pull_request: | |
| types: [labeled, synchronize, opened] | |
| push: | |
| branches: | |
| - "main" | |
| tags: | |
| - "v*.*.*" # on release tag | |
| workflow_dispatch: | |
| env: | |
| PYTHON_VERSION: "3.8" | |
| PRE_RELEASE_INSTRUCTIONS: | | |
| ## Installing the pre-release Python SDK | |
| 1. Download the correct `.whl`. For Mac M1/M2, grab the "universal2" `.whl` | |
| 2. Run `pip install rerun_sdk<...>.whl` (replace `<...>` with the actual filename) | |
| 3. Test it: `rerun --version` | |
| UBUNTU_REQUIRED_PKGS: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} # Cancel previous CI jobs only on pull-requests | |
| cancel-in-progress: true | |
| jobs: | |
| lint: | |
| name: Python lints (black, mypy, flake8) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: extractions/setup-just@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| just-version: 1.5 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: "pip" | |
| cache-dependency-path: "rerun_py/requirements-lint.txt" | |
| - name: Install Python dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r rerun_py/requirements-lint.txt | |
| - name: Lint Python | |
| run: | | |
| just py-lint | |
| - name: Check requirements | |
| run: | | |
| just py-requirements | |
| # --------------------------------------------------------------------------- | |
| matrix-setup: | |
| # Building all the wheels is expensive, so we only run this job when we push (to main or release tags), | |
| # or if the PR has the 'build wheels' label for explicit testing of wheels. | |
| if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels') || github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - name: Dump GitHub context | |
| env: | |
| GITHUB_CONTEXT: ${{ toJson(github) }} | |
| run: echo "$GITHUB_CONTEXT" | |
| - name: Dump job context | |
| env: | |
| JOB_CONTEXT: ${{ toJson(job) }} | |
| run: echo "$JOB_CONTEXT" | |
| # Sets TAGGED_OR_MAIN if this workflow is running on a tag or the main branch. | |
| - name: Set TAGGED_OR_MAIN | |
| if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' | |
| run: echo "TAGGED_OR_MAIN=1" >> $GITHUB_ENV | |
| - id: set-matrix | |
| shell: bash | |
| run: | | |
| matrix=() | |
| matrix+=('{"platform": "macos", "runs_on": "macos-latest", "pip": "pip", "python": "python3"},') | |
| matrix+=('{"platform": "windows", "runs_on": "windows-latest", "pip": "pip", "python": "python"},') | |
| matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}, "pip": "pip", "python": "python3"},') | |
| matrix+=('{"platform": "aarch64", "runs_on": ["self-hosted", "linux", "ARM64"], container: {"image": "quay.io/pypa/manylinux_2_28_aarch64"}, "pip": "python3.8 -m pip", "python": "python3.8"}') | |
| echo "Matrix values: ${matrix[@]}" | |
| echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT | |
| wheels: | |
| name: Build Python Wheels | |
| needs: [lint, matrix-setup] | |
| strategy: | |
| matrix: ${{fromJson(needs.matrix-setup.outputs.matrix)}} | |
| runs-on: ${{ matrix.runs_on }} | |
| container: ${{ matrix.container }} | |
| steps: | |
| - name: Install rust | |
| if: matrix.platform == 'aarch64' | |
| run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - uses: actions/checkout@v3 | |
| # These should already be in the docker container, but run for good measure. A no-op install | |
| # should be fast, and this way things don't break if we add new packages without rebuilding | |
| # docker | |
| - name: Cache APT Packages | |
| if: matrix.platform == 'linux' | |
| uses: awalsh128/[email protected] | |
| with: | |
| packages: ${{ env.UBUNTU_REQUIRED_PKGS }} | |
| version: 2.0 # Increment this to pull newer packages | |
| execute_install_scripts: true | |
| - name: Cache DNF packages | |
| if: matrix.platform == 'aarch64' | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| /var/cache/dnf | |
| key: ${{ runner.os }}-dnf-aarch64 | |
| - name: AARCH64 install packages | |
| if: matrix.platform == 'aarch64' | |
| run: | | |
| dnf makecache --refresh | |
| dnf install -y gtk3-devel openssl-devel | |
| - name: Set up cargo cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY | |
| # Don't update the cache -- it will be updated by the lint job | |
| # TODO(jleibs): this job will likely run before rust.yml updates | |
| # the cache. Better cross-job sequencing would be nice here | |
| save-if: False | |
| # The pip-cache setup logic doesn't work in the ubuntu docker container | |
| # That's probably fine since we bake these deps into the container already | |
| - name: Setup python | |
| if: matrix.platform != 'linux' && matrix.platform != 'aarch64' | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| cache: "pip" | |
| cache-dependency-path: "rerun_py/requirements-build.txt" | |
| # These should already be in the docker container, but run for good measure. A no-op install | |
| # should be fast, and this way things don't break if we add new packages without rebuilding | |
| # docker | |
| - name: "Install build requirements" | |
| run: | | |
| if [ "${{ matrix.platform }}" = "aarch64" ]; then | |
| ${{ matrix.pip }} install -r rerun_py/requirements-build.txt && echo "/opt/_internal/cpython-3.8.16/bin" >> $GITHUB_PATH | |
| else | |
| ${{ matrix.pip }} install -r rerun_py/requirements-build.txt | |
| fi | |
| shell: bash | |
| # ---------------------------------------------------------------------------------- | |
| # Install prerequisites for building the web-viewer Wasm | |
| # We have a nice script for that: ./scripts/setup_web.sh | |
| # Unfortunately, we can't run that on Windows, because Windows doesn't come with | |
| # a package manager like grown-up OSes do (at least not the CI version of Windows). | |
| # Also we can't run it on linux because the 20.04 Docker container will install | |
| # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag | |
| # So we only run the script on macos, and then on Windows we do the parts of the script manually. | |
| # On ubuntu, the correct packages are pre-installed in our docker container. | |
| - name: Install prerequisites for building the web-viewer Wasm (non-Windows) | |
| if: matrix.platform == 'macos' | |
| shell: bash | |
| run: ./scripts/setup_web.sh | |
| # The first steps of setup_web.sh, for Windows: | |
| - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows | |
| if: matrix.platform == 'windows' | |
| shell: bash | |
| run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 | |
| # The last step of setup_web.sh, for Windows. | |
| # Since 'winget' is not available within the GitHub runner, we download the package directly: | |
| # See: https://github.com/marketplace/actions/engineerd-configurator | |
| - name: Install binaryen for building the web-viewer Wasm on windows | |
| if: matrix.platform == 'windows' | |
| uses: engineerd/[email protected] | |
| with: | |
| name: "wasm-opt.exe" | |
| url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" | |
| pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" | |
| # ---------------------------------------------------------------------------------- | |
| - name: Patch Cargo.toml for pre-release | |
| if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' | |
| # After patching the pre-release version, run cargo check. | |
| # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing | |
| run: | | |
| ${{ matrix.python }} scripts/version_util.py --patch_prerelease | |
| cargo check | |
| - name: Version check for tagged-release | |
| if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch' | |
| # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z | |
| run: | | |
| ${{ matrix.python }} scripts/version_util.py --check_version | |
| - name: Build Wheel | |
| if: matrix.platform != 'aarch64' | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| maturin-version: "0.14.10" | |
| manylinux: manylinux_2_31 | |
| container: off | |
| command: build | |
| args: | | |
| --manifest-path rerun_py/Cargo.toml | |
| --release | |
| --no-default-features | |
| --features pypi | |
| --universal2 | |
| --out pre-dist | |
| -i ${{ matrix.python }} | |
| - name: Build Wheel (aarch64) | |
| if: matrix.platform == 'aarch64' | |
| run: python3.8 -m maturin build --manifest-path rerun_py/Cargo.toml --release --no-default-features --features pypi --universal2 --out pre-dist | |
| - name: Install built wheel | |
| run: | | |
| ${{ matrix.pip }} install depthai_viewer --find-links pre-dist --force-reinstall | |
| - name: Run tests | |
| shell: bash | |
| run: | | |
| if [ "${{ matrix.platform }}" = "aarch64" ]; then | |
| cd rerun_py/tests && python3.8 -m pytest | |
| else | |
| cd rerun_py/tests && pytest | |
| fi | |
| - name: Unpack the wheel | |
| shell: bash | |
| run: | | |
| mkdir unpack-dist | |
| if [ "${{ matrix.platform }}" == "aarch64" ]; then | |
| python3.8 -m wheel unpack pre-dist/*.whl --dest unpack-dist | |
| else | |
| wheel unpack pre-dist/*.whl --dest unpack-dist | |
| fi | |
| - name: Get the folder name | |
| shell: bash | |
| run: | | |
| echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV | |
| - name: Repack the wheel | |
| shell: bash | |
| run: | | |
| mkdir dist | |
| if [ "${{ matrix.platform }}" == "aarch64" ]; then | |
| python3.8 -m wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ | |
| else | |
| wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ | |
| fi | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| # --------------------------------------------------------------------------- | |
| # See https://github.com/ncipollo/release-action | |
| pre-release: | |
| name: Pre Release | |
| needs: [wheels] | |
| if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| # First delete the old prerelease. If we don't do this, we don't get things like | |
| # proper source-archives and changelog info. | |
| # https://github.com/dev-drprasad/delete-tag-and-release | |
| - uses: dev-drprasad/[email protected] | |
| with: | |
| tag_name: prerelease | |
| delete_release: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Re-tag the prerelease with the commit from this build | |
| # https://github.com/richardsimko/update-tag | |
| - name: Update prerelease tag | |
| uses: richardsimko/update-tag@v1 | |
| with: | |
| tag_name: prerelease | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Create the actual prerelease | |
| # https://github.com/softprops/action-gh-release | |
| - name: GitHub Release | |
| uses: softprops/[email protected] | |
| with: | |
| name: "Development Build" | |
| body: ${{ env.PRE_RELEASE_INSTRUCTIONS }} | |
| prerelease: true | |
| tag_name: prerelease | |
| files: dist/* | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| generate_release_notes: true | |
| # --------------------------------------------------------------------------- | |
| # This job is run on tags starting with "v", e.g., "v0.1.0" | |
| tagged-release: | |
| name: Release | |
| runs-on: ubuntu-latest | |
| if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch' | |
| needs: [wheels] | |
| steps: | |
| - name: Download Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| - name: GitHub Release | |
| uses: softprops/[email protected] | |
| with: | |
| prerelease: false | |
| files: dist/* | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| generate_release_notes: true | |
| - name: Publish to PyPI | |
| uses: PyO3/maturin-action@v1 | |
| env: | |
| # These are both set in the GitHub project configuration | |
| MATURIN_REPOSITORY: ${{ vars.PYPI_REPOSITORY }} | |
| MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }} | |
| with: | |
| command: upload | |
| args: --skip-existing dist/* |