python.yml

name: CI (Python)

on:
  pull_request:
    types: [labeled, synchronize, opened]
  push:
    branches:
      - "main"
    tags:
      - "v*.*.*" # on release tag
  workflow_dispatch:

env:
  PYTHON_VERSION: "3.8"
  PRE_RELEASE_INSTRUCTIONS: |
    ## Installing the pre-release Python SDK
    1. Download the correct `.whl`. For Mac M1/M2, grab the "universal2" `.whl`
    2. Run `pip install rerun_sdk<...>.whl` (replace `<...>` with the actual filename)
    3. Test it: `rerun --version`
  UBUNTU_REQUIRED_PKGS: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} # Cancel previous CI jobs only on pull-requests
  cancel-in-progress: true

jobs:
  # lint:
  #   name: Python lints (black, mypy, flake8)
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v3

  #     - uses: extractions/setup-just@v1
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #       with:
  #         just-version: 1.5

  #     - name: Set up Python
  #       uses: actions/setup-python@v4
  #       with:
  #         python-version: ${{ env.PYTHON_VERSION }}
  #         cache: "pip"
  #         cache-dependency-path: "rerun_py/requirements-lint.txt"

  #     - name: Install Python dependencies
  #       run: |
  #         pip install --upgrade pip
  #         pip install -r rerun_py/requirements-lint.txt

  #     - name: Lint Python
  #       run: |
  #         just py-lint

  #     - name: Check requirements
  #       run: |
  #         just py-requirements

  # ---------------------------------------------------------------------------

  matrix-setup:
    # Building all the wheels is expensive, so we only run this job when we push (to main or release tags),
    # or if the PR has the 'build wheels' label for explicit testing of wheels.
    if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels') || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJson(job) }}
        run: echo "$JOB_CONTEXT"
      # Sets TAGGED_OR_MAIN if this workflow is running on a tag or the main branch.
      - name: Set TAGGED_OR_MAIN
        if: startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main'
        run: echo "TAGGED_OR_MAIN=1" >> $GITHUB_ENV

      - id: set-matrix
        shell: bash
        run: |
          matrix=()

          if [[ $TAGGED_OR_MAIN ]]; then
            # MacOS build is really slow (>30 mins); uses up a lot of CI minutes
            matrix+=('{"platform": "macos", "runs_on": "macos-latest", "pip": "pip", "python": "python3"},')
          fi
          # matrix+=('{"platform": "windows", "runs_on": "windows-latest", "pip": "pip", "python": "python"},')
          # matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}, "pip": "pip", "python": "python3"},')
          matrix+=('{"platform": "aarch64", "runs_on": ["self-hosted", "linux", "ARM64"], container: {"image": "quay.io/pypa/manylinux_2_28_aarch64"}, "pip": "python3.8 -m pip", "python": "python3.8"}')

          echo "Matrix values: ${matrix[@]}"

          echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT

  wheels:
    name: Build Python Wheels
    needs: [matrix-setup]

    strategy:
      matrix: ${{fromJson(needs.matrix-setup.outputs.matrix)}}

    runs-on: ${{ matrix.runs_on }}

    container: ${{ matrix.container }}

    steps:
      - name: Install rust
        if: matrix.platform == 'aarch64'
        run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - uses: actions/checkout@v3

      # These should already be in the docker container, but run for good measure. A no-op install
      # should be fast, and this way things don't break if we add new packages without rebuilding
      # docker
      - name: Cache APT Packages
        if: matrix.platform == 'linux'
        uses: awalsh128/cache-apt-pkgs-action@v1.2.4
        with:
          packages: ${{ env.UBUNTU_REQUIRED_PKGS }}
          version: 2.0 # Increment this to pull newer packages
          execute_install_scripts: true

      - name: Cache DNF packages
        if: matrix.platform == 'aarch64'
        uses: actions/cache@v2
        with:
          path: |
            /var/cache/dnf
          key: ${{ runner.os }}-dnf-aarch64

      - name: AARCH64 install packages
        if: matrix.platform == 'aarch64'
        run: |
          dnf makecache --refresh
          dnf install -y gtk3-devel openssl-devel

      - name: Set up cargo cache
        uses: Swatinem/rust-cache@v2
        with:
          env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
          # Don't update the cache -- it will be updated by the lint job
          # TODO(jleibs): this job will likely run before rust.yml updates
          # the cache. Better cross-job sequencing would be nice here
          save-if: False

      # - name: (aarch64) Set up default python
      #   if: matrix.platform == 'aarch64'
      #   run: rm /usr/bin/python3 && mv /usr/local/bin/python3.8 /usr/bin/python3

      # The pip-cache setup logic doesn't work in the ubuntu docker container
      # That's probably fine since we bake these deps into the container already
      - name: Setup python
        if: matrix.platform != 'linux' && matrix.platform != 'aarch64'
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pip"
          cache-dependency-path: "rerun_py/requirements-build.txt"

      # These should already be in the docker container, but run for good measure. A no-op install
      # should be fast, and this way things don't break if we add new packages without rebuilding
      # docker
      - name: "Install build requirements"
        run: |
          if [ "${{ matrix.platform }}" = "aarch64" ]; then
            ${{ matrix.pip }} install -r rerun_py/requirements-build.txt && echo "/opt/_internal/cpython-3.8.16/bin" >> $GITHUB_PATH
          else
            ${{ matrix.pip }} install -r rerun_py/requirements-build.txt
          fi
        shell: bash

      # ----------------------------------------------------------------------------------
      # Install prerequisites for building the web-viewer Wasm

      # We have a nice script for that: ./scripts/setup_web.sh
      # Unfortunately, we can't run that on Windows, because Windows doesn't come with
      # a package manager like grown-up OSes do (at least not the CI version of Windows).
      # Also we can't run it on linux because the 20.04 Docker container will install
      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
      # On ubuntu, the correct packages are pre-installed in our docker container.

      - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
        if: matrix.platform == 'macos'
        shell: bash
        run: ./scripts/setup_web.sh

      # The first steps of setup_web.sh, for Windows:
      - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows
        if: matrix.platform == 'windows'
        shell: bash
        run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84

      # The last step of setup_web.sh, for Windows.
      # Since 'winget' is not available within the GitHub runner, we download the package directly:
      # See: https://github.com/marketplace/actions/engineerd-configurator
      - name: Install binaryen for building the web-viewer Wasm on windows
        if: matrix.platform == 'windows'
        uses: engineerd/configurator@v0.0.9
        with:
          name: "wasm-opt.exe"
          url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
          pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"

      # ----------------------------------------------------------------------------------

      - name: Patch Cargo.toml for pre-release
        if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch'
        # After patching the pre-release version, run cargo check.
        # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing
        run: |
          python3 scripts/version_util.py --patch_prerelease
          cargo check

      - name: Version check for tagged-release
        if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch'
        # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z
        run: |
          python3 scripts/version_util.py --check_version

      - name: Build Wheel
        if: matrix.platform != 'aarch64'
        uses: PyO3/maturin-action@v1
        with:
          maturin-version: "0.14.10"
          manylinux: manylinux_2_31
          container: off
          command: build
          args: |
            --manifest-path rerun_py/Cargo.toml
            --release
            --no-default-features
            --features pypi
            --universal2
            --out pre-dist
            -i ${{ matrix.python }}
      - name: Build Wheel (aarch64)
        if: matrix.platform == 'aarch64'
        run: python3.8 -m maturin build --manifest-path rerun_py/Cargo.toml --release --no-default-features --features pypi --universal2 --out pre-dist

      - name: Install built wheel
        run: |
          ${{ matrix.pip }} install depthai_viewer --find-links pre-dist --force-reinstall

      - name: Run tests
        run: |
          if [ "${{ matrix.platform }}" = "aarch64" ]; then
            cd rerun_py/tests && python3.8 -m pytest
          else
            cd rerun_py/tests && pytest
          fi

      - name: Unpack the wheel
        shell: bash
        run: |
          mkdir unpack-dist
          wheel unpack pre-dist/*.whl --dest unpack-dist

      - name: Get the folder name
        shell: bash
        run: |
          echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV

      - name: Repack the wheel
        shell: bash
        run: |
          mkdir dist
          wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/

      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist

  # ---------------------------------------------------------------------------

  # See https://github.com/ncipollo/release-action
  pre-release:
    name: Pre Release
    needs: [wheels]
    if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch'
    runs-on: "ubuntu-latest"
    steps:
      - name: Download Artifact
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      # First delete the old prerelease. If we don't do this, we don't get things like
      # proper source-archives and changelog info.
      # https://github.com/dev-drprasad/delete-tag-and-release
      - uses: dev-drprasad/delete-tag-and-release@v0.2.0
        with:
          tag_name: prerelease
          delete_release: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Re-tag the prerelease with the commit from this build
      # https://github.com/richardsimko/update-tag
      - name: Update prerelease tag
        uses: richardsimko/update-tag@v1
        with:
          tag_name: prerelease
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Create the actual prerelease
      # https://github.com/softprops/action-gh-release
      - name: GitHub Release
        uses: softprops/action-gh-release@v0.1.15
        with:
          name: "Development Build"
          body: ${{ env.PRE_RELEASE_INSTRUCTIONS }}
          prerelease: true
          tag_name: prerelease
          files: dist/*
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: true

  # ---------------------------------------------------------------------------

  # This job is run on tags starting with "v", e.g., "v0.1.0"
  tagged-release:
    name: Release
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch'
    needs: [wheels]
    steps:
      - name: Download Artifact
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      - name: GitHub Release
        uses: softprops/action-gh-release@v0.1.15
        with:
          prerelease: false
          files: dist/*
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: true

      - name: Publish to PyPI
        uses: PyO3/maturin-action@v1
        env:
          # These are both set in the GitHub project configuration
          MATURIN_REPOSITORY: ${{ vars.PYPI_REPOSITORY }}
          MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }}
        with:
          command: upload
          args: --skip-existing dist/*