python.yml

name: CI (Python)

on:
  pull_request:
    types: [labeled, synchronize, opened]
  push:
    branches:
      - "main"
    tags:
      - "v*.*.*" # on release tag
  workflow_dispatch:
    inputs:
      force_build_wheel:
        description: "Build python wheels"
        required: true
        default: false
        type: boolean

env:
  PYTHON_VERSION: "3.8"
  PRE_RELEASE_INSTRUCTIONS: |
    ## Installing the pre-release Python SDK
    1. Download the correct `.whl`.
    2. Run `pip install rerun_sdk<...>.whl` (replace `<...>` with the actual filename)
    3. Test it: `rerun --version`
  UBUNTU_REQUIRED_PKGS: libgtk-3-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxkbcommon-dev libssl-dev libfontconfig1-dev libatk-bridge2.0 libfreetype6-dev libglib2.0-dev

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} # Cancel previous CI jobs only on pull-requests
  cancel-in-progress: true

jobs:
  lint:
    name: Python lints (black, mypy, flake8)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: extractions/setup-just@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          just-version: 1.5

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pip"
          cache-dependency-path: "rerun_py/requirements-lint.txt"

      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -r rerun_py/requirements-lint.txt

      - name: Lint Python
        run: |
          just py-lint

      - name: Check requirements
        run: |
          just py-requirements

  # ---------------------------------------------------------------------------

  matrix-setup:
    # Building all the wheels is expensive, so we only run this job when we push (to main or release tags),
    # or if the job was manually triggered with `force_build_wheel` set to true.
    if: github.event_name == 'push' || github.event.inputs.force_build_wheel
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJson(job) }}
        run: echo "$JOB_CONTEXT"

      - id: set-matrix
        shell: bash
        run: |
          matrix=()
          matrix+=('{"platform": "macos",   "target": "x86_64-apple-darwin",      "wheel_suffix": "x86_64", "runs_on": "macos-latest"          },')
          matrix+=('{"platform": "macos",   "target": "aarch64-apple-darwin",     "wheel_suffix": "arm64",  "runs_on": "macos-latest"          },')
          matrix+=('{"platform": "windows", "target": "x86_64-pc-windows-msvc",   "wheel_suffix": "",       "runs_on": "windows-latest-8-cores"},')
          matrix+=('{"platform": "linux",   "target": "x86_64-unknown-linux-gnu", "wheel_suffix": "",       "runs_on": "ubuntu-latest-16-cores", container: {"image": "rerunio/ci_docker:0.6"}}')

          echo "Matrix values: ${matrix[@]}"

          echo "matrix={\"include\":[${matrix[@]}]}" >> $GITHUB_OUTPUT

  wheels:
    name: Build Python Wheels
    needs: [lint, matrix-setup]

    strategy:
      matrix: ${{fromJson(needs.matrix-setup.outputs.matrix)}}

    runs-on: ${{ matrix.runs_on }}

    container: ${{ matrix.container }}

    steps:
      - uses: actions/checkout@v3

      # These should already be in the docker container, but run for good measure. A no-op install
      # should be fast, and this way things don't break if we add new packages without rebuilding
      # docker
      - name: Cache APT Packages
        if: matrix.platform == 'linux'
        uses: awalsh128/cache-apt-pkgs-action@v1.2.2
        with:
          packages: ${{ env.UBUNTU_REQUIRED_PKGS }}
          version: 2.0 # Increment this to pull newer packages
          execute_install_scripts: true

      - name: Set up cargo cache
        uses: Swatinem/rust-cache@v2
        with:
          env-vars: CARGO CC CFLAGS CXX CMAKE RUST CACHE_KEY
          # Don't update the cache -- it will be updated by the lint job
          # TODO(jleibs): this job will likely run before rust.yml updates
          # the cache. Better cross-job sequencing would be nice here
          save-if: False

      # The pip-cache setup logic doesn't work in the ubuntu docker container
      # That's probably fine since we bake these deps into the container already
      - name: Setup python
        if: matrix.platform != 'linux'
        uses: actions/setup-python@v4
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: "pip"
          cache-dependency-path: "rerun_py/requirements-build.txt"

      # These should already be in the docker container, but run for good measure. A no-op install
      # should be fast, and this way things don't break if we add new packages without rebuilding
      # docker
      - run: pip install -r rerun_py/requirements-build.txt

      # ----------------------------------------------------------------------------------
      # Install prerequisites for building the web-viewer Wasm

      # We have a nice script for that: ./scripts/setup_web.sh
      # Unfortunately, we can't run that on Windows, because Windows doesn't come with
      # a package manager like grown-up OSes do (at least not the CI version of Windows).
      # Also we can't run it on linux because the 20.04 Docker container will install
      # an old version of binaryen/wasm-opt that barfs on the `--fast-math` flag
      # So we only run the script on macos, and then on Windows we do the parts of the script manually.
      # On ubuntu, the correct packages are pre-installed in our docker container.

      - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
        if: matrix.platform == 'macos'
        shell: bash
        run: ./scripts/setup_web.sh

      # The first steps of setup_web.sh, for Windows:
      - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows
        if: matrix.platform == 'windows'
        shell: bash
        run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84

      # The last step of setup_web.sh, for Windows.
      # Since 'winget' is not available within the GitHub runner, we download the package directly:
      # See: https://github.com/marketplace/actions/engineerd-configurator
      - name: Install binaryen for building the web-viewer Wasm on windows
        if: matrix.platform == 'windows'
        uses: engineerd/configurator@v0.0.9
        with:
          name: "wasm-opt.exe"
          url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
          pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"

      # ----------------------------------------------------------------------------------

      - name: Patch Cargo.toml for pre-release
        if: github.ref == 'refs/heads/main'
        # After patching the pre-release version, run cargo update.
        # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing
        run: |
          python3 scripts/version_util.py --patch_prerelease
          cargo update -w

      - name: Version check for tagged-release
        if: startsWith(github.ref, 'refs/tags/v')
        # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z
        run: |
          python3 scripts/version_util.py --check_version

      - name: Build Wheel
        uses: PyO3/maturin-action@v1
        with:
          maturin-version: "0.14.10"
          manylinux: manylinux_2_31
          container: off
          command: build
          args: |
            --manifest-path rerun_py/Cargo.toml
            --release
            --target ${{ matrix.target }}
            --no-default-features
            --features pypi
            --out pre-dist

      - name: Install built wheel
        shell: bash
        run: |
          pip install pre-dist/*${{ matrix.wheel_suffix }}.whl --force-reinstall

      - name: Verify built wheel version
        shell: bash
        run: |
          python3 -m rerun --version
          which rerun
          rerun --version

      - name: Run unit tests
        shell: bash
        run: cd rerun_py/tests && pytest

      - name: Install requriements for e2e test
        run: |
          pip install -r examples/python/api_demo/requirements.txt
          pip install -r examples/python/car/requirements.txt
          pip install -r examples/python/multithreading/requirements.txt
          pip install -r examples/python/plots/requirements.txt
          pip install -r examples/python/text_logging/requirements.txt

      - name: Run e2e test
        shell: bash
        run: scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed

      - name: Unpack the wheel
        shell: bash
        run: |
          mkdir unpack-dist
          wheel unpack pre-dist/*.whl --dest unpack-dist

      - name: Get the folder name
        shell: bash
        run: |
          echo "pkg_folder=$(ls unpack-dist)" >> $GITHUB_ENV

      - name: Cache RRD dataset
        id: dataset
        uses: actions/cache@v3
        with:
          path: examples/python/colmap/dataset/
          # TODO(jleibs): Derive this key from the invocation below
          key: colmap-dataset-colmap-fiat-v0

      - name: Generate Embedded RRD file
        shell: bash
        # If you change the line below you should almost definitely change the `key:` line above by giving it a new, unique name
        run: |
          mkdir rrd
          pip install -r examples/python/colmap/requirements.txt
          python3 examples/python/colmap/main.py --dataset colmap_fiat --resize 800x600 --save rrd/colmap_fiat.rrd
          cp rrd/colmap_fiat.rrd unpack-dist/${{ env.pkg_folder }}/rerun_sdk/rerun_demo/colmap_fiat.rrd

      - name: Repack the wheel
        shell: bash
        run: |
          mkdir dist
          wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/

      - name: Upload wheels
        uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: dist

      # All platforms are currently creating the same rrd file, upload one of them
      - name: Save RRD artifact
        if: matrix.platform == 'linux'
        uses: actions/upload-artifact@v3
        with:
          name: rrd
          path: rrd

  # ---------------------------------------------------------------------------

  upload_rrd:
    name: Upload RRD to GCloud
    permissions:
      contents: "read"
      id-token: "write"
    needs: [wheels]
    runs-on: "ubuntu-latest"
    steps:
      - name: Download Artifact
        uses: actions/download-artifact@v3
        with:
          name: rrd
          path: rrd

      - id: "auth"
        uses: google-github-actions/auth@v1
        with:
          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}

      - name: Add SHORT_SHA env property with commit short sha
        run: echo "SHORT_SHA=`echo ${{github.sha}} | cut -c1-7`" >> $GITHUB_ENV

      - name: "Upload RRD (commit)"
        uses: google-github-actions/upload-cloud-storage@v1
        with:
          path: "rrd"
          destination: "rerun-example-rrd/commit/${{env.SHORT_SHA}}"
          parent: false

      - name: "Upload RRD (prerelease)"
        if: "!startsWith(github.ref , 'refs/tags/v')"
        uses: google-github-actions/upload-cloud-storage@v1
        with:
          path: "rrd"
          destination: "rerun-example-rrd/prerelease"
          parent: false

      - name: "Upload RRD (tagged)"
        if: startsWith(github.ref, 'refs/tags/v')
        uses: google-github-actions/upload-cloud-storage@v1
        with:
          path: "rrd"
          destination: "rerun-example-rrd/version/${{github.ref_name}}"
          parent: false

      - name: "Upload RRD (latest release)"
        if: github.ref == 'latest'
        uses: google-github-actions/upload-cloud-storage@v1
        with:
          path: "rrd"
          destination: "rerun-example-rrd/latest"
          parent: false

  # See https://github.com/ncipollo/release-action
  pre-release:
    name: Pre Release
    needs: [wheels]
    if: github.ref == 'refs/heads/main'
    runs-on: "ubuntu-latest"
    steps:
      - name: Download Artifact
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      # First delete the old prerelease. If we don't do this, we don't get things like
      # proper source-archives and changelog info.
      # https://github.com/dev-drprasad/delete-tag-and-release
      - uses: dev-drprasad/delete-tag-and-release@v0.2.1
        with:
          tag_name: prerelease
          delete_release: true
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Create the actual prerelease
      # https://github.com/ncipollo/release-action
      - name: GitHub Release
        uses: ncipollo/release-action@v1.12.0
        with:
          body: ${{ env.PRE_RELEASE_INSTRUCTIONS }}
          prerelease: true
          artifacts: dist/*
          name: "Development Build"
          tag: "prerelease"
          token: ${{ secrets.GITHUB_TOKEN }}
          generateReleaseNotes: true
          allowUpdates: true
          removeArtifacts: true
          replacesArtifacts: true

  # ---------------------------------------------------------------------------

  # This job is run on tags starting with "v", e.g., "v0.1.0"
  tagged-release:
    name: Release
    runs-on: ubuntu-latest
    if: startsWith(github.ref, 'refs/tags/v')
    needs: [wheels]
    steps:
      - name: Download Artifact
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      # https://github.com/ncipollo/release-action
      - name: GitHub Release
        uses: ncipollo/release-action@v1.12.0
        with:
          prerelease: true
          artifacts: dist/*
          token: ${{ secrets.GITHUB_TOKEN }}
          generateReleaseNotes: true

      - name: Publish to PyPI
        uses: PyO3/maturin-action@v1
        env:
          # These are both set in the GitHub project configuration
          MATURIN_REPOSITORY: ${{ vars.PYPI_REPOSITORY }}
          MATURIN_PYPI_TOKEN: ${{ secrets.MATURIN_PYPI_TOKEN }}
        with:
          command: upload
          args: --skip-existing dist/*

  # ---------------------------------------------------------------------------

  py-test-docs:
    name: Verify the docs build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
          cache: "pip"
          cache-dependency-path: "rerun_py/requirements-doc.txt"

      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -r rerun_py/requirements-doc.txt

      - name: Build via mkdocs
        run: |
          mkdocs build -f rerun_py/mkdocs.yml

  py-docs:
    name: Build and deploy docs
    if: ${{ github.event_name == 'push' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # Don't do a shallow clone

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
          cache: "pip"
          cache-dependency-path: "rerun_py/requirements-doc.txt"

      - name: Install Python dependencies
        run: |
          pip install --upgrade pip
          pip install -r rerun_py/requirements-doc.txt

      - name: Set up git author
        run: |
          remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
          git config --global user.name "${GITHUB_ACTOR}"
          git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # Mike will incrementally update the existing gh-pages branch
      # We then check it out, and reset it to a new orphaned branch, which we force-push to origin
      # to make sure we don't accumulate unnecessary history in gh-pages branch
      - name: Deploy via mike # https://github.com/jimporter/mike
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          git fetch
          mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python -u ${{github.ref_name}} latest
          git checkout gh-pages
          git checkout --orphan gh-pages-orphan
          git commit -m "Update docs for ${GITHUB_SHA}"
          git push origin gh-pages-orphan:gh-pages -f

      # Mike will incrementally update the existing gh-pages branch
      # We then check it out, and reset it to a new orphaned branch, which we force-push to origin
      # to make sure we don't accumulate unnecessary history in gh-pages branch
      - name: Deploy tag via mike # https://github.com/jimporter/mike
        if: github.ref == 'refs/heads/main'
        run: |
          git fetch
          mike deploy -F rerun_py/mkdocs.yml --rebase -b gh-pages --prefix docs/python HEAD
          git checkout gh-pages
          git checkout --orphan gh-pages-orphan
          git commit -m "Update docs for ${GITHUB_SHA}"
          git push origin gh-pages-orphan:gh-pages -f