Skip to content

Commit

Permalink
Enable manual ci.
Browse files Browse the repository at this point in the history
  • Loading branch information
@zrezke
zrezke committed Dec 5, 2024
1 parent e863469 commit 4cf3a22
Showing 1 changed file with 106 additions and 125 deletions.
231 changes: 106 additions & 125 deletions .github/workflows/python.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ on:
- "main"
tags:
- "v*.*.*" # on release tag
workflow_dispatch:

env:
PYTHON_VERSION: "3.8"
Expand All @@ -27,40 +28,43 @@ jobs:
name: Python lints (black, mypy, flake8)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3

- uses: extractions/setup-just@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
just-version: 1.5

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
cache: "pip"
cache-dependency-path: "rerun_py/requirements-lint.txt"

- name: Install Python dependencies
- name: Skip lints
run: |
pip install --upgrade pip
pip install -r rerun_py/requirements-lint.txt
- name: Lint Python
run: |
just py-lint
- name: Check requirements
run: |
just py-requirements
echo "Skipping lints"
# - uses: actions/checkout@v3

# - uses: extractions/setup-just@v1
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# just-version: 1.5

# - name: Set up Python
# uses: actions/setup-python@v4
# with:
# python-version: ${{ env.PYTHON_VERSION }}
# cache: "pip"
# cache-dependency-path: "rerun_py/requirements-lint.txt"

# - name: Install Python dependencies
# run: |
# pip install --upgrade pip
# pip install -r rerun_py/requirements-lint.txt

# - name: Lint Python
# run: |
# just py-lint

# - name: Check requirements
# run: |
# just py-requirements

# ---------------------------------------------------------------------------

matrix-setup:
# Building all the wheels is expensive, so we only run this job when we push (to main or release tags),
# or if the PR has the 'build wheels' label for explicit testing of wheels.
if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels')
if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels') || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest

outputs:
Expand All @@ -85,12 +89,10 @@ jobs:
run: |
matrix=()
if [[ $TAGGED_OR_MAIN ]]; then
# MacOS build is really slow (>30 mins); uses up a lot of CI minutes
matrix+=('{"platform": "macos", "runs_on": "macos-latest"},')
fi
matrix+=('{"platform": "windows", "runs_on": "windows-latest"},')
matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}}')
matrix+=('{"platform": "macos", "runs_on": "macos-latest", "pip": "pip", "python": "python3"},')
matrix+=('{"platform": "windows", "runs_on": "windows-latest", "pip": "pip", "python": "python"},')
matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}, "pip": "pip", "python": "python3"},')
matrix+=('{"platform": "aarch64", "runs_on": ["self-hosted", "linux", "ARM64"], container: {"image": "quay.io/pypa/manylinux_2_28_aarch64"}, "pip": "python3.8 -m pip", "python": "python3.8"}')
echo "Matrix values: ${matrix[@]}"
Expand All @@ -108,6 +110,10 @@ jobs:
container: ${{ matrix.container }}

steps:
- name: Install rust
if: matrix.platform == 'aarch64'
run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH

- uses: actions/checkout@v3

# These should already be in the docker container, but run for good measure. A no-op install
Expand All @@ -121,6 +127,20 @@ jobs:
version: 2.0 # Increment this to pull newer packages
execute_install_scripts: true

- name: Cache DNF packages
if: matrix.platform == 'aarch64'
uses: actions/cache@v2
with:
path: |
/var/cache/dnf
key: ${{ runner.os }}-dnf-aarch64

- name: AARCH64 install packages
if: matrix.platform == 'aarch64'
run: |
dnf makecache --refresh
dnf install -y gtk3-devel openssl-devel
- name: Set up cargo cache
uses: Swatinem/rust-cache@v2
with:
Expand All @@ -133,7 +153,7 @@ jobs:
# The pip-cache setup logic doesn't work in the ubuntu docker container
# That's probably fine since we bake these deps into the container already
- name: Setup python
if: matrix.platform != 'linux'
if: matrix.platform != 'linux' && matrix.platform != 'aarch64'
uses: actions/setup-python@v4
with:
python-version: ${{ env.PYTHON_VERSION }}
Expand All @@ -143,7 +163,14 @@ jobs:
# These should already be in the docker container, but run for good measure. A no-op install
# should be fast, and this way things don't break if we add new packages without rebuilding
# docker
- run: pip install -r rerun_py/requirements-build.txt
- name: "Install build requirements"
run: |
if [ "${{ matrix.platform }}" = "aarch64" ]; then
${{ matrix.pip }} install -r rerun_py/requirements-build.txt && echo "/opt/_internal/cpython-3.8.16/bin" >> $GITHUB_PATH
else
${{ matrix.pip }} install -r rerun_py/requirements-build.txt
fi
shell: bash

# ----------------------------------------------------------------------------------
# Install prerequisites for building the web-viewer Wasm
Expand All @@ -156,45 +183,46 @@ jobs:
# So we only run the script on macos, and then on Windows we do the parts of the script manually.
# On ubuntu, the correct packages are pre-installed in our docker container.

- name: Install prerequisites for building the web-viewer Wasm (non-Windows)
if: matrix.platform == 'macos'
shell: bash
run: ./scripts/setup_web.sh
# - name: Install prerequisites for building the web-viewer Wasm (non-Windows)
# if: matrix.platform == 'macos'
# shell: bash
# run: ./scripts/setup_web.sh

# The first steps of setup_web.sh, for Windows:
- name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows
if: matrix.platform == 'windows'
shell: bash
run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84
# - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows
# if: matrix.platform == 'windows'
# shell: bash
# run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84

# The last step of setup_web.sh, for Windows.
# Since 'winget' is not available within the GitHub runner, we download the package directly:
# See: https://github.com/marketplace/actions/engineerd-configurator
- name: Install binaryen for building the web-viewer Wasm on windows
if: matrix.platform == 'windows'
uses: engineerd/[email protected]
with:
name: "wasm-opt.exe"
url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"
# - name: Install binaryen for building the web-viewer Wasm on windows
# if: matrix.platform == 'windows'
# uses: engineerd/[email protected]
# with:
# name: "wasm-opt.exe"
# url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz"
# pathInArchive: "binaryen-version_111/bin/wasm-opt.exe"

# ----------------------------------------------------------------------------------

- name: Patch Cargo.toml for pre-release
if: github.ref == 'refs/heads/main'
if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch'
# After patching the pre-release version, run cargo check.
# This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing
run: |
python3 scripts/version_util.py --patch_prerelease
${{ matrix.python }} scripts/version_util.py --patch_prerelease
cargo check
- name: Version check for tagged-release
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch'
# This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z
run: |
python3 scripts/version_util.py --check_version
${{ matrix.python }} scripts/version_util.py --check_version
- name: Build Wheel
if: matrix.platform != 'aarch64'
uses: PyO3/maturin-action@v1
with:
maturin-version: "0.14.10"
Expand All @@ -208,19 +236,33 @@ jobs:
--features pypi
--universal2
--out pre-dist
-i ${{ matrix.python }}
- name: Build Wheel (aarch64)
if: matrix.platform == 'aarch64'
run: python3.8 -m maturin build --manifest-path rerun_py/Cargo.toml --release --no-default-features --features pypi --universal2 --out pre-dist

- name: Install built wheel
run: |
pip install depthai_viewer --find-links pre-dist --force-reinstall
${{ matrix.pip }} install depthai_viewer --find-links pre-dist --force-reinstall
- name: Run tests
run: cd rerun_py/tests && pytest
shell: bash
run: |
if [ "${{ matrix.platform }}" = "aarch64" ]; then
cd rerun_py/tests && python3.8 -m pytest
else
cd rerun_py/tests && pytest
fi
- name: Unpack the wheel
shell: bash
run: |
mkdir unpack-dist
wheel unpack pre-dist/*.whl --dest unpack-dist
if [ "${{ matrix.platform }}" == "aarch64" ]; then
python3.8 -m wheel unpack pre-dist/*.whl --dest unpack-dist
else
wheel unpack pre-dist/*.whl --dest unpack-dist
fi
- name: Get the folder name
shell: bash
Expand All @@ -231,7 +273,11 @@ jobs:
shell: bash
run: |
mkdir dist
wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/
if [ "${{ matrix.platform }}" == "aarch64" ]; then
python3.8 -m wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/
else
wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/
fi
- name: Upload wheels
uses: actions/upload-artifact@v3
Expand All @@ -245,7 +291,7 @@ jobs:
pre-release:
name: Pre Release
needs: [wheels]
if: github.ref == 'refs/heads/main'
if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch'
runs-on: "ubuntu-latest"
steps:
- name: Download Artifact
Expand Down Expand Up @@ -292,7 +338,7 @@ jobs:
tagged-release:
name: Release
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/v')
if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch'
needs: [wheels]
steps:
- name: Download Artifact
Expand All @@ -318,68 +364,3 @@ jobs:
with:
command: upload
args: --skip-existing dist/*

# ---------------------------------------------------------------------------

# py-test-docs:
# name: Verify the docs build
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3

# - name: Set up Python
# uses: actions/setup-python@v4
# with:
# python-version: "3.8"
# cache: "pip"
# cache-dependency-path: "rerun_py/requirements-doc.txt"

# - name: Install Python dependencies
# run: |
# pip install --upgrade pip
# pip install -r rerun_py/requirements-doc.txt

# - name: Build via mkdocs
# run: |
# mkdocs build -f rerun_py/mkdocs.yml

# py-docs:
# name: Build and deploy docs
# if: ${{ github.event_name == 'push' }}
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# with:
# fetch-depth: 0 # Don't do a shallow clone

# - name: Set up Python
# uses: actions/setup-python@v4
# with:
# python-version: "3.8"
# cache: "pip"
# cache-dependency-path: "rerun_py/requirements-doc.txt"

# - name: Install Python dependencies
# run: |
# pip install --upgrade pip
# pip install -r rerun_py/requirements-doc.txt

# - name: Set up git author
# run: |
# remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
# git config --global user.name "${GITHUB_ACTOR}"
# git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# - name: Deploy via mike # https://github.com/jimporter/mike
# if: startsWith(github.ref, 'refs/tags/v')
# run: |
# git fetch
# mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages --prefix docs/python -u ${{github.ref_name}} latest

# - name: Deploy tag via mike # https://github.com/jimporter/mike
# if: github.ref == 'refs/heads/main'
# run: |
# git fetch
# mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages --prefix docs/python HEAD

0 comments on commit 4cf3a22

Please sign in to comment.