From 64ede94428dd59d47b4605ed73347e5fceecaf64 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Sat, 14 Sep 2024 19:34:51 +0200 Subject: [PATCH] More binary builds v2 (#582) * More binary builds v2 * fix version * fix archs * lets see what happens * fix cd script * typo * wut * ok, need true platform * comment and fix * fix builds * more fix * better test * fix test * fix win * try again * safer * add pushd * cannot install deps with no-index * maybe like this * had an extta popd * sod it * comments * try windows test --- .github/workflows/cd.yml | 112 +++++++++++++++++++++++---------------- download-wgpu-native.py | 50 ++++++++++++----- pyproject.toml | 42 +++++++-------- 3 files changed, 122 insertions(+), 82 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1d7fd18f..dac18a16 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,3 +1,16 @@ +# Github Actions script to produce binary wheels. +# +# Note that a lot of the cibuildwheel config is in pyproject.toml. +# +# We perform one build for each wheel that we generate, i.e. one per architecture. +# In download_wgpu_native.py, we detect CIBW_PLATFORM and CIBW_ARCHS to determine +# the required binary from wgpu-native. +# +# If https://github.com/pypa/cibuildwheel/issues/944 gets implemented, we can build more wheels per build. +# +# Also includes the sdist build that does not include a binary. + + name: CD on: @@ -11,70 +24,75 @@ on: jobs: - # The release builds are done for the platforms that we want to build wheels for. - # We build wheels, test them, and then upload the wheel as an artifact. release-builds: - name: Build wheels on ${{ matrix.os }} + name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }} timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + include: + - platform: windows + arch: AMD64 + os: windows-latest + testable: true + - platform: windows + arch: ARM64 + os: windows-latest + - platform: windows + arch: x86 + os: windows-latest + - platform: macos + arch: arm64 + os: macos-latest + testable: true + - platform: macos + arch: x86_64 + os: macos-13 # last Intel MacOS + cibw_version: '==2.16' # delocation does not work for later versions + - platform: linux + arch: x86_64 + os: ubuntu-latest + testable: true + - platform: linux + arch: aarch64 + os: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up QEMU + if: matrix.platform == 'linux' && matrix.arch == 'aarch64' + uses: docker/setup-qemu-action@v3 with: - python-version: '3.9' + platforms: arm64 - name: Install dev dependencies run: | - python -m pip install --upgrade pip wheel setuptools twine + python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}} - name: Build wheels - # Use v2.16, v2.20 fails the osx builds - uses: pypa/cibuildwheel@v2.16 + run: python -m cibuildwheel --output-dir dist env: - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 - CIBW_ARCHS_LINUX: x86_64 - CIBW_SKIP: cp39-musllinux_x86_64 - with: - output-dir: dist + CIBW_PLATFORM: ${{ matrix.platform }} + CIBW_ARCHS: ${{ matrix.arch }} - name: Twine check run: | twine check dist/* + - name: Test wheel + if: matrix.testable + shell: bash + run: | + rm -rf ./wgpu + filename=$(ls dist/*.whl) + pip install $filename + pushd $HOME + python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' + popd + pip uninstall -y wgpu + git reset --hard HEAD - name: Upload distributions uses: actions/upload-artifact@v4 with: path: dist - name: ${{ matrix.os }}-build + name: ${{ matrix.platform }}-${{ matrix.arch }}-build - # These release builds uses QEMU so that we can build wheels for arm64. - # We build wheels and upload the wheel as an artifact, but we don't test them here. - qemu-release-builds: - name: Build wheels on ubuntu-latest with QEMU - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Build wheels - uses: pypa/cibuildwheel@v2.20 - env: - CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 - CIBW_ARCHS_LINUX: aarch64 - CIBW_SKIP: cp39-musllinux_aarch64 - with: - output-dir: dist - - name: Upload distributions - uses: actions/upload-artifact@v4 - with: - path: dist - name: qemu-build sdist-build: name: Build sdist @@ -99,9 +117,8 @@ jobs: shell: bash run: | rm -rf ./wgpu - pushd $HOME - pip install $GITHUB_WORKSPACE/dist/*.tar.gz - popd + filename=$(ls dist/*.tar.gz) + pip install $filename # don't run tests, we just want to know if the sdist can be installed pip uninstall -y wgpu git reset --hard HEAD @@ -114,10 +131,11 @@ jobs: path: dist name: sdist-build + publish: name: Publish to Github and Pypi runs-on: ubuntu-latest - needs: [release-builds, qemu-release-builds, sdist-build] + needs: [release-builds, sdist-build] if: success() && startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/checkout@v4 diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 69492e29..9f357334 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -66,7 +66,9 @@ def extract_file(zip_filename, member, path): def get_os_string(): - if sys.platform.startswith("win"): + if os.environ.get("CIBUILDWHEEL") == "1" and os.getenv("CIBW_PLATFORM"): + return os.getenv("CIBW_PLATFORM") + elif sys.platform.startswith("win"): return "windows" elif sys.platform.startswith("darwin"): return "macos" @@ -84,24 +86,46 @@ def get_arch(): is_64_bit = sys.maxsize > 2**32 machine = platform.machine() - # See if this is run by cibuildwheel and check to see if ARCHFLAGS is - # specified (only done on macOS). This allows to select the proper binaries. - # For specifics of CIBUILDWHEEL and macOS build envs, see: - # https://github.com/pypa/cibuildwheel/blob/4307b52ff28b631519d38bfa0dd09d6a9b39a81e/cibuildwheel/macos.py#L277 - if os.environ.get("CIBUILDWHEEL") == "1" and "ARCHFLAGS" in os.environ: - archflags = os.environ["ARCHFLAGS"] - return "aarch64" if "arm64" in archflags else "x86_64" - if machine == "armv7l": # Raspberry pi - return "armv7" + detected_arch = "armv7" elif is_64_bit and machine.startswith(("arm", "aarch64")): # Includes MacOS M1, arm linux, ... - return "aarch64" + detected_arch = "aarch64" elif is_64_bit: - return "x86_64" + detected_arch = "x86_64" else: - return "i686" + detected_arch = "i686" + + if os.environ.get("CIBUILDWHEEL") == "1": + # When running in cibuildwheel, we derive the intended arch from + # an env var (the same one that cibuildwheel uses) that we set in cd.yml. + cibw_arch = os.getenv("CIBW_ARCHS") # must be singular + if not cibw_arch: + # Linux builds run on Docker, so env is not visible + cibw_arch = detected_arch + elif "," in cibw_arch: + raise RuntimeError("CIBW_ARCHS must have a single arch") + arch_map = { + "windows": { + "AMD64": "x86_64", + "ARM64": "aarch64", + "x86": "i686", + }, + "macos": { + "arm64": "aarch64", + "x86_64": "x86_64", + }, + "linux": { + "x86_64": "x86_64", + "aarch64": "aarch64", + "i868": "i686", + }, + } + maps_for_os = arch_map[get_os_string()] + return maps_for_os[cibw_arch] + + return detected_arch def main(version, os_string, arch, upstream): diff --git a/pyproject.toml b/pyproject.toml index 3ec58526..81456a66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,33 +1,31 @@ [build-system] -requires = [ - "setuptools>=42", -] +requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" + [tool.cibuildwheel] # we only build on one python version since the wheels are not bound to it -build = "cp39-*" - -# we can't list requests under build-system.requires because -# that step happens _after_ the before-build command +build = "cp312-*" +# Print system info before build +before-all = "uname -a" +# Can't list requests under build-system.requires because that step happens _after_ the before-build command before-build = "pip install requests && python download-wgpu-native.py" - -# this is sufficient to trigger an install of the built wheel +# This is sufficient to trigger an install of the built wheel test-command = "echo Wheel installed" -# this is the minimum supported manylinux version -manylinux-x86_64-image = "manylinux_2_24" -manylinux-i686-image = "manylinux_2_24" -manylinux-aarch64-image = "manylinux_2_24" -manylinux-ppc64le-image = "manylinux_2_24" -manylinux-s390x-image = "manylinux_2_24" -manylinux-pypy_x86_64-image = "manylinux_2_24" -manylinux-pypy_i686-image = "manylinux_2_24" -manylinux-pypy_aarch64-image = "manylinux_2_24" +[tool.cibuildwheel.windows] +# Only for local use, overridden in cd.yml +archs = ["amd64"] [tool.cibuildwheel.macos] -# also create apple silicon wheels -archs = ["x86_64", "arm64"] +# Only for local use, overridden in cd.yml +archs = ["arm64"] -# the upstream binaries are not universal yet -# archs = ["x86_64", "universal2", "arm64"] +[tool.cibuildwheel.linux] +# wgpu-native does not build for musllinux yet +skip = "*musllinux*" +# Use custom images, with minimal version that matches wgpu-native +manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64" +manylinux-aarch64-image = "quay.io/pypa/manylinux_2_28_aarch64" +manylinux-i686-image = "quay.io/pypa/manylinux_2_28_i686" +manylinux-ppc64le-image = "quay.io/pypa/manylinux_2_28_ppc64le"