From d7c867db30897038ff5a39b8566cbc5bbe955a2b Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Tue, 5 Nov 2024 22:06:04 +0100 Subject: [PATCH] Introduce a mechanism of building wheels with extra features (#8010) ### What - Useful for making and sharing early versions of wheels ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/8010?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/8010?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/8010) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --- .github/workflows/adhoc_wheels.yml | 141 ++++++++++++++++++ .../reusable_build_and_upload_wheels.yml | 5 +- rerun_py/Cargo.toml | 3 + scripts/ci/build_and_upload_wheels.py | 9 +- 4 files changed, 154 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/adhoc_wheels.yml diff --git a/.github/workflows/adhoc_wheels.yml b/.github/workflows/adhoc_wheels.yml new file mode 100644 index 000000000000..afa37ec2878d --- /dev/null +++ b/.github/workflows/adhoc_wheels.yml @@ -0,0 +1,141 @@ +name: Adhoc Wheels + +on: + workflow_dispatch: + inputs: + MODE: + type: choice + required: false + options: + - pypi + - pr + - extra + description: "The build mode (`pypi` includes the web viewer, `pr` does not)" + +defaults: + run: + shell: bash + +permissions: + contents: "write" + id-token: "write" + deployments: "write" + +jobs: + # ----------------------------------------------------------------------------------- + # Build rerun-cli (rerun binaries): + + build-rerun-cli-and-upload-linux-arm64: + name: "Linux-arm64: Build & Upload rerun-cli" + uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml + with: + CONCURRENCY: adhoc-wheels-linux-arm64 + PLATFORM: linux-arm64 + secrets: inherit + + build-rerun-cli-and-upload-linux-x64: + name: "Linux-x64: Build & Upload rerun-cli" + uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml + with: + CONCURRENCY: adhoc-wheels-linux-x64 + PLATFORM: linux-x64 + secrets: inherit + + build-rerun-cli-and-upload-macos-x64: + name: "Mac-x64: Build & Upload rerun-cli" + uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml + with: + CONCURRENCY: adhoc-wheels-macos-x64 + PLATFORM: macos-x64 + secrets: inherit + + build-rerun-cli-and-upload-macos-arm64: + name: "Mac-arm64: Build & Upload rerun-cli" + uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml + with: + CONCURRENCY: adhoc-wheels-macos-arm64 + PLATFORM: macos-arm64 + secrets: inherit + + build-rerun-cli-and-upload-windows-x64: + name: "Windows-x64: Build & Upload rerun-cli" + uses: ./.github/workflows/reusable_build_and_upload_rerun_cli.yml + with: + CONCURRENCY: adhoc-wheels-windows-x64 + PLATFORM: windows-x64 + secrets: inherit + + # --------------------------------------------------------------------------- + # Build wheels: + + build-wheel-linux-arm64: + needs: [build-rerun-cli-and-upload-linux-arm64] + name: "Linux-arm64: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: adhoc-wheels-linux-arm64 + PLATFORM: linux-arm64 + WHEEL_ARTIFACT_NAME: linux-arm64-wheel + MODE: ${{ inputs.MODE }} + secrets: inherit + + build-wheel-linux-x64: + needs: [build-rerun-cli-and-upload-linux-x64] + name: "Linux-x64: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: adhoc-wheels-linux-x64 + PLATFORM: linux-x64 + WHEEL_ARTIFACT_NAME: linux-x64-wheel + MODE: ${{ inputs.MODE }} + secrets: inherit + + build-wheel-macos-arm64: + needs: [build-rerun-cli-and-upload-macos-arm64] + name: "Macos-arm64: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: adhoc-wheels-macos-arm64 + PLATFORM: macos-arm64 + WHEEL_ARTIFACT_NAME: macos-arm64-wheel + MODE: ${{ inputs.MODE }} + secrets: inherit + + build-wheel-macos-x64: + needs: [build-rerun-cli-and-upload-macos-x64] + name: "Macos-x64: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: adhoc-wheels-macos-x64 + PLATFORM: macos-x64 + WHEEL_ARTIFACT_NAME: "macos-x64-wheel" + MODE: ${{ inputs.MODE }} + secrets: inherit + + build-wheel-windows-x64: + needs: [build-rerun-cli-and-upload-windows-x64] + name: "Windows-x64: Build & Upload Wheels" + uses: ./.github/workflows/reusable_build_and_upload_wheels.yml + with: + CONCURRENCY: adhoc-wheels-windows-x64 + PLATFORM: windows-x64 + WHEEL_ARTIFACT_NAME: windows-x64-wheel + MODE: "pypi" + secrets: inherit + + # -------------------------------------------------------------------------- + + generate-pip-index: + name: "Generate Pip Index" + needs: + [ + build-wheel-linux-arm64, + build-wheel-linux-x64, + build-wheel-macos-arm64, + build-wheel-macos-x64, + build-wheel-windows-x64, + ] + uses: ./.github/workflows/reusable_pip_index.yml + with: + CONCURRENCY: adhoc-wheels + secrets: inherit diff --git a/.github/workflows/reusable_build_and_upload_wheels.yml b/.github/workflows/reusable_build_and_upload_wheels.yml index 1c92e09cce04..432944479aa3 100644 --- a/.github/workflows/reusable_build_and_upload_wheels.yml +++ b/.github/workflows/reusable_build_and_upload_wheels.yml @@ -40,6 +40,7 @@ on: options: - pypi - pr + - extra description: "The build mode (`pypi` includes the web viewer, `pr` does not)" CONCURRENCY: required: false @@ -226,7 +227,7 @@ jobs: - name: "Build rerun_notebook" # only build the notebook if we are building for pypi and running linux-x64 - if: ${{ inputs.MODE == 'pypi' && inputs.PLATFORM == 'linux-x64' }} + if: ${{ (inputs.MODE == 'pypi' || inputs.MODE == 'extra') && inputs.PLATFORM == 'linux-x64' }} run: | rm -rf dist pixi run js-build-base @@ -234,7 +235,7 @@ jobs: --dir commit/${{ steps.get-sha.outputs.sha }}/wheels - name: Save rerun_notebook wheel artifact - if: ${{ inputs.MODE == 'pypi' && inputs.PLATFORM == 'linux-x64' }} + if: ${{ (inputs.MODE == 'pypi' || inputs.MODE == 'extra') && inputs.PLATFORM == 'linux-x64' }} uses: actions/upload-artifact@v4 with: name: rerun_notebook_wheel diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index f5f1dd1d1798..5abdadb343cc 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -17,6 +17,9 @@ name = "rerun_bindings" # name of the .so library that the Python module will im [features] default = ["extension-module"] +## Extra features that aren't ready to be included in release builds yet. +extra = ["pypi", "remote"] + ## The features we turn on when building the `rerun-sdk` PyPi package ## for . pypi = ["extension-module", "nasm", "web_viewer"] diff --git a/scripts/ci/build_and_upload_wheels.py b/scripts/ci/build_and_upload_wheels.py index 94c0ca149225..8d1fbf69b1b5 100755 --- a/scripts/ci/build_and_upload_wheels.py +++ b/scripts/ci/build_and_upload_wheels.py @@ -56,18 +56,23 @@ def detect_target() -> str: class BuildMode(Enum): PYPI = "pypi" PR = "pr" + EXTRA = "extra" def __str__(self) -> str: return self.value def build_and_upload(bucket: Bucket | None, mode: BuildMode, gcs_dir: str, target: str, compatibility: str) -> None: - if mode is BuildMode.PYPI: - # Only build web viewer when publishing to pypi + # pypi / extra builds require a web build + if mode in (BuildMode.PYPI, BuildMode.EXTRA): run("pixi run rerun-build-web-release") + + if mode is BuildMode.PYPI: maturin_feature_flags = "--no-default-features --features pypi" elif mode is BuildMode.PR: maturin_feature_flags = "--no-default-features --features extension-module" + elif mode is BuildMode.EXTRA: + maturin_feature_flags = "--no-default-features --features extra" dist = f"dist/{target}"