diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4e4e0e77a9c3..abb337bc96da 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -8,6 +8,7 @@ on: - "main" tags: - "v*.*.*" # on release tag + workflow_dispatch: env: PYTHON_VERSION: "3.8" @@ -27,40 +28,43 @@ jobs: name: Python lints (black, mypy, flake8) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - - uses: extractions/setup-just@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - just-version: 1.5 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: "pip" - cache-dependency-path: "rerun_py/requirements-lint.txt" - - - name: Install Python dependencies + - name: Skip lints run: | - pip install --upgrade pip - pip install -r rerun_py/requirements-lint.txt - - - name: Lint Python - run: | - just py-lint - - - name: Check requirements - run: | - just py-requirements + echo "Skipping lints" + # - uses: actions/checkout@v3 + + # - uses: extractions/setup-just@v1 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # with: + # just-version: 1.5 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: "pip" + # cache-dependency-path: "rerun_py/requirements-lint.txt" + + # - name: Install Python dependencies + # run: | + # pip install --upgrade pip + # pip install -r rerun_py/requirements-lint.txt + + # - name: Lint Python + # run: | + # just py-lint + + # - name: Check requirements + # run: | + # just py-requirements # --------------------------------------------------------------------------- matrix-setup: # Building all the wheels is expensive, so we only run this job when we push (to main or release tags), # or if the PR has the 'build wheels' label for explicit testing of wheels. - if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels') + if: github.event_name == 'push' || contains( github.event.pull_request.labels.*.name, '🛞 build wheels') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest outputs: @@ -85,12 +89,10 @@ jobs: run: | matrix=() - if [[ $TAGGED_OR_MAIN ]]; then - # MacOS build is really slow (>30 mins); uses up a lot of CI minutes - matrix+=('{"platform": "macos", "runs_on": "macos-latest"},') - fi - matrix+=('{"platform": "windows", "runs_on": "windows-latest"},') - matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}}') + matrix+=('{"platform": "macos", "runs_on": "macos-latest", "pip": "pip", "python": "python3"},') + matrix+=('{"platform": "windows", "runs_on": "windows-latest", "pip": "pip", "python": "python"},') + matrix+=('{"platform": "linux", "runs_on": "ubuntu-latest", container: {"image": "rerunio/ci_docker:0.5"}, "pip": "pip", "python": "python3"},') + matrix+=('{"platform": "aarch64", "runs_on": ["self-hosted", "linux", "ARM64"], container: {"image": "quay.io/pypa/manylinux_2_28_aarch64"}, "pip": "python3.8 -m pip", "python": "python3.8"}') echo "Matrix values: ${matrix[@]}" @@ -108,6 +110,10 @@ jobs: container: ${{ matrix.container }} steps: + - name: Install rust + if: matrix.platform == 'aarch64' + run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && echo "$HOME/.cargo/bin" >> $GITHUB_PATH + - uses: actions/checkout@v3 # These should already be in the docker container, but run for good measure. A no-op install @@ -121,6 +127,20 @@ jobs: version: 2.0 # Increment this to pull newer packages execute_install_scripts: true + - name: Cache DNF packages + if: matrix.platform == 'aarch64' + uses: actions/cache@v2 + with: + path: | + /var/cache/dnf + key: ${{ runner.os }}-dnf-aarch64 + + - name: AARCH64 install packages + if: matrix.platform == 'aarch64' + run: | + dnf makecache --refresh + dnf install -y gtk3-devel openssl-devel + - name: Set up cargo cache uses: Swatinem/rust-cache@v2 with: @@ -133,7 +153,7 @@ jobs: # The pip-cache setup logic doesn't work in the ubuntu docker container # That's probably fine since we bake these deps into the container already - name: Setup python - if: matrix.platform != 'linux' + if: matrix.platform != 'linux' && matrix.platform != 'aarch64' uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_VERSION }} @@ -143,7 +163,14 @@ jobs: # These should already be in the docker container, but run for good measure. A no-op install # should be fast, and this way things don't break if we add new packages without rebuilding # docker - - run: pip install -r rerun_py/requirements-build.txt + - name: "Install build requirements" + run: | + if [ "${{ matrix.platform }}" = "aarch64" ]; then + ${{ matrix.pip }} install -r rerun_py/requirements-build.txt && echo "/opt/_internal/cpython-3.8.16/bin" >> $GITHUB_PATH + else + ${{ matrix.pip }} install -r rerun_py/requirements-build.txt + fi + shell: bash # ---------------------------------------------------------------------------------- # Install prerequisites for building the web-viewer Wasm @@ -156,45 +183,46 @@ jobs: # So we only run the script on macos, and then on Windows we do the parts of the script manually. # On ubuntu, the correct packages are pre-installed in our docker container. - - name: Install prerequisites for building the web-viewer Wasm (non-Windows) - if: matrix.platform == 'macos' - shell: bash - run: ./scripts/setup_web.sh + # - name: Install prerequisites for building the web-viewer Wasm (non-Windows) + # if: matrix.platform == 'macos' + # shell: bash + # run: ./scripts/setup_web.sh # The first steps of setup_web.sh, for Windows: - - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows - if: matrix.platform == 'windows' - shell: bash - run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 + # - name: Install wasm32 and wasm-bindgen-cli for building the web-viewer Wasm on windows + # if: matrix.platform == 'windows' + # shell: bash + # run: rustup target add wasm32-unknown-unknown && cargo install wasm-bindgen-cli --version 0.2.84 # The last step of setup_web.sh, for Windows. # Since 'winget' is not available within the GitHub runner, we download the package directly: # See: https://github.com/marketplace/actions/engineerd-configurator - - name: Install binaryen for building the web-viewer Wasm on windows - if: matrix.platform == 'windows' - uses: engineerd/configurator@v0.0.9 - with: - name: "wasm-opt.exe" - url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" - pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" + # - name: Install binaryen for building the web-viewer Wasm on windows + # if: matrix.platform == 'windows' + # uses: engineerd/configurator@v0.0.9 + # with: + # name: "wasm-opt.exe" + # url: "https://github.com/WebAssembly/binaryen/releases/download/version_111/binaryen-version_111-x86_64-windows.tar.gz" + # pathInArchive: "binaryen-version_111/bin/wasm-opt.exe" # ---------------------------------------------------------------------------------- - name: Patch Cargo.toml for pre-release - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' # After patching the pre-release version, run cargo check. # This updates the cargo.lock file with the new version numbers and keeps the wheel build from failing run: | - python3 scripts/version_util.py --patch_prerelease + ${{ matrix.python }} scripts/version_util.py --patch_prerelease cargo check - name: Version check for tagged-release - if: startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch' # This call to version_util.py will assert version from Cargo.toml matches git tagged version vX.Y.Z run: | - python3 scripts/version_util.py --check_version + ${{ matrix.python }} scripts/version_util.py --check_version - name: Build Wheel + if: matrix.platform != 'aarch64' uses: PyO3/maturin-action@v1 with: maturin-version: "0.14.10" @@ -208,19 +236,33 @@ jobs: --features pypi --universal2 --out pre-dist + -i ${{ matrix.python }} + - name: Build Wheel (aarch64) + if: matrix.platform == 'aarch64' + run: python3.8 -m maturin build --manifest-path rerun_py/Cargo.toml --release --no-default-features --features pypi --universal2 --out pre-dist - name: Install built wheel run: | - pip install depthai_viewer --find-links pre-dist --force-reinstall + ${{ matrix.pip }} install depthai_viewer --find-links pre-dist --force-reinstall - name: Run tests - run: cd rerun_py/tests && pytest + shell: bash + run: | + if [ "${{ matrix.platform }}" = "aarch64" ]; then + cd rerun_py/tests && python3.8 -m pytest + else + cd rerun_py/tests && pytest + fi - name: Unpack the wheel shell: bash run: | mkdir unpack-dist - wheel unpack pre-dist/*.whl --dest unpack-dist + if [ "${{ matrix.platform }}" == "aarch64" ]; then + python3.8 -m wheel unpack pre-dist/*.whl --dest unpack-dist + else + wheel unpack pre-dist/*.whl --dest unpack-dist + fi - name: Get the folder name shell: bash @@ -231,7 +273,11 @@ jobs: shell: bash run: | mkdir dist - wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ + if [ "${{ matrix.platform }}" == "aarch64" ]; then + python3.8 -m wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ + else + wheel pack unpack-dist/${{ env.pkg_folder }} --dest dist/ + fi - name: Upload wheels uses: actions/upload-artifact@v3 @@ -245,7 +291,7 @@ jobs: pre-release: name: Pre Release needs: [wheels] - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' runs-on: "ubuntu-latest" steps: - name: Download Artifact @@ -292,7 +338,7 @@ jobs: tagged-release: name: Release runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/v') && github.event_name != 'workflow_dispatch' needs: [wheels] steps: - name: Download Artifact @@ -318,68 +364,3 @@ jobs: with: command: upload args: --skip-existing dist/* - - # --------------------------------------------------------------------------- - - # py-test-docs: - # name: Verify the docs build - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: "3.8" - # cache: "pip" - # cache-dependency-path: "rerun_py/requirements-doc.txt" - - # - name: Install Python dependencies - # run: | - # pip install --upgrade pip - # pip install -r rerun_py/requirements-doc.txt - - # - name: Build via mkdocs - # run: | - # mkdocs build -f rerun_py/mkdocs.yml - - # py-docs: - # name: Build and deploy docs - # if: ${{ github.event_name == 'push' }} - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # with: - # fetch-depth: 0 # Don't do a shallow clone - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: "3.8" - # cache: "pip" - # cache-dependency-path: "rerun_py/requirements-doc.txt" - - # - name: Install Python dependencies - # run: | - # pip install --upgrade pip - # pip install -r rerun_py/requirements-doc.txt - - # - name: Set up git author - # run: | - # remote_repo="https://${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" - # git config --global user.name "${GITHUB_ACTOR}" - # git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com" - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # - name: Deploy via mike # https://github.com/jimporter/mike - # if: startsWith(github.ref, 'refs/tags/v') - # run: | - # git fetch - # mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages --prefix docs/python -u ${{github.ref_name}} latest - - # - name: Deploy tag via mike # https://github.com/jimporter/mike - # if: github.ref == 'refs/heads/main' - # run: | - # git fetch - # mike deploy -F rerun_py/mkdocs.yml -p --rebase -b gh-pages --prefix docs/python HEAD