From 37707756ff6797fcbe30b0a5baab47236562d4f2 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 12:00:26 +0200 Subject: [PATCH 01/23] More binary builds v2 --- .github/workflows/cd.yml | 83 ++++++++++++++++++++++++---------------- pyproject.toml | 40 +++++++++---------- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1d7fd18f..69bcfbeb 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,28 +11,40 @@ on: jobs: + # The release builds are done for the platforms that we want to build wheels for. # We build wheels, test them, and then upload the wheel as an artifact. release-builds: - name: Build wheels on ${{ matrix.os }} + name: Build wheels on ${{ matrix.name }} timeout-minutes: 10 runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + include: + - name: windows + os: windows-latest + archs: 'AMD64,ARM64,x86' + - name: macos arm64 + os: macos-latest + archs: 'aarch64' + - name: macos x86 + os: macos-13 + archs: 'x86_64' + - name: linux + os: ubuntu-latest + archs: 'x86_64,aarch64' steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.12' - name: Install dev dependencies run: | python -m pip install --upgrade pip wheel setuptools twine - name: Build wheels - # Use v2.16, v2.20 fails the osx builds - uses: pypa/cibuildwheel@v2.16 + uses: pypa/cibuildwheel@v2 env: CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 CIBW_ARCHS_LINUX: x86_64 @@ -46,35 +58,37 @@ jobs: uses: actions/upload-artifact@v4 with: path: dist - name: ${{ matrix.os }}-build + name: ${{ matrix.name }}-build + + + # # These release builds uses QEMU so that we can build wheels for arm64. + # # We build wheels and upload the wheel as an artifact, but we don't test them here. + # qemu-release-builds: + # name: Build wheels on ubuntu-latest with QEMU + # timeout-minutes: 10 + # runs-on: ubuntu-latest + # strategy: + # fail-fast: false + # steps: + # - uses: actions/checkout@v4 + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + # with: + # platforms: arm64 + # - name: Build wheels + # uses: pypa/cibuildwheel@v2.20 + # env: + # CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 + # CIBW_ARCHS_LINUX: aarch64 + # CIBW_SKIP: cp39-musllinux_aarch64 + # with: + # output-dir: dist + # - name: Upload distributions + # uses: actions/upload-artifact@v4 + # with: + # path: dist + # name: qemu-build - # These release builds uses QEMU so that we can build wheels for arm64. - # We build wheels and upload the wheel as an artifact, but we don't test them here. - qemu-release-builds: - name: Build wheels on ubuntu-latest with QEMU - timeout-minutes: 10 - runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Build wheels - uses: pypa/cibuildwheel@v2.20 - env: - CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 - CIBW_ARCHS_LINUX: aarch64 - CIBW_SKIP: cp39-musllinux_aarch64 - with: - output-dir: dist - - name: Upload distributions - uses: actions/upload-artifact@v4 - with: - path: dist - name: qemu-build sdist-build: name: Build sdist @@ -114,10 +128,11 @@ jobs: path: dist name: sdist-build + publish: name: Publish to Github and Pypi runs-on: ubuntu-latest - needs: [release-builds, qemu-release-builds, sdist-build] + needs: [release-builds, sdist-build] if: success() && startsWith(github.ref, 'refs/tags/v') steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 3ec58526..4923e85c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,33 +1,33 @@ [build-system] -requires = [ - "setuptools>=42", -] +requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" + [tool.cibuildwheel] + # we only build on one python version since the wheels are not bound to it -build = "cp39-*" +build = "cp312-*" + +# Print system info before build +before-all = "uname -a" -# we can't list requests under build-system.requires because -# that step happens _after_ the before-build command +# Can't list requests under build-system.requires because that step happens _after_ the before-build command before-build = "pip install requests && python download-wgpu-native.py" -# this is sufficient to trigger an install of the built wheel +# This is sufficient to trigger an install of the built wheel test-command = "echo Wheel installed" -# this is the minimum supported manylinux version -manylinux-x86_64-image = "manylinux_2_24" -manylinux-i686-image = "manylinux_2_24" -manylinux-aarch64-image = "manylinux_2_24" -manylinux-ppc64le-image = "manylinux_2_24" -manylinux-s390x-image = "manylinux_2_24" -manylinux-pypy_x86_64-image = "manylinux_2_24" -manylinux-pypy_i686-image = "manylinux_2_24" -manylinux-pypy_aarch64-image = "manylinux_2_24" [tool.cibuildwheel.macos] -# also create apple silicon wheels -archs = ["x86_64", "arm64"] -# the upstream binaries are not universal yet -# archs = ["x86_64", "universal2", "arm64"] +# Only for local use, overridden in cd.yml +archs = ["arm64", "x86_64"] + + +[tool.cibuildwheel.linux] + +# Use custom images, with minimal version that matches wgpu-native +manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64" +manylinux-aarch64-image = "quay.io/pypa/manylinux_2_28_aarch64" +manylinux-i686-image = "quay.io/pypa/manylinux_2_28_i686" +manylinux-ppc64le-image = "quay.io/pypa/manylinux_2_28_ppc64le" From e59b4a479abdc4fbf05cd82319aa3fd083e21bf0 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 12:02:59 +0200 Subject: [PATCH 02/23] fix version --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 69bcfbeb..f0edf382 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -44,7 +44,7 @@ jobs: run: | python -m pip install --upgrade pip wheel setuptools twine - name: Build wheels - uses: pypa/cibuildwheel@v2 + uses: pypa/cibuildwheel@v2.20 env: CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 CIBW_ARCHS_LINUX: x86_64 From aff2f4576b5f5748ce54e8fd0006da72ad719d7f Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 12:07:55 +0200 Subject: [PATCH 03/23] fix archs --- .github/workflows/cd.yml | 4 +--- pyproject.toml | 14 ++++++-------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f0edf382..fedb54ae 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -46,9 +46,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.20 env: - CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 - CIBW_ARCHS_LINUX: x86_64 - CIBW_SKIP: cp39-musllinux_x86_64 + CIBW_ARCHS: ${{ matrix.archs }} with: output-dir: dist - name: Twine check diff --git a/pyproject.toml b/pyproject.toml index 4923e85c..81456a66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,28 +4,26 @@ build-backend = "setuptools.build_meta" [tool.cibuildwheel] - # we only build on one python version since the wheels are not bound to it build = "cp312-*" - # Print system info before build before-all = "uname -a" - # Can't list requests under build-system.requires because that step happens _after_ the before-build command before-build = "pip install requests && python download-wgpu-native.py" - # This is sufficient to trigger an install of the built wheel test-command = "echo Wheel installed" +[tool.cibuildwheel.windows] +# Only for local use, overridden in cd.yml +archs = ["amd64"] [tool.cibuildwheel.macos] - # Only for local use, overridden in cd.yml -archs = ["arm64", "x86_64"] - +archs = ["arm64"] [tool.cibuildwheel.linux] - +# wgpu-native does not build for musllinux yet +skip = "*musllinux*" # Use custom images, with minimal version that matches wgpu-native manylinux-x86_64-image = "quay.io/pypa/manylinux_2_28_x86_64" manylinux-aarch64-image = "quay.io/pypa/manylinux_2_28_aarch64" From fa4d611bbb96cde0ad4ea506ab5ba052d03a3eec Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 13:09:24 +0200 Subject: [PATCH 04/23] lets see what happens --- .github/workflows/cd.yml | 44 ++++++++++++++++++++++++++-------------- download-wgpu-native.py | 36 +++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index fedb54ae..7eb3d053 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -12,28 +12,41 @@ on: jobs: - # The release builds are done for the platforms that we want to build wheels for. - # We build wheels, test them, and then upload the wheel as an artifact. + # We perform one release build for each wheel that we generate, i.e. one per architecture. + # We use CIBW_ARCHS in download_wgpu_native.py to detect what binary is needed. + # If https://github.com/pypa/cibuildwheel/issues/944 gets implemented, we can build more wheels per build. + # Some builds can be properly tested, some cannot. release-builds: - name: Build wheels on ${{ matrix.name }} + name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }} timeout-minutes: 10 - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: + cibw_version: [2.20] include: - - name: windows + - platform: windows + arch: 'AMD64' + os: windows-latest + - platform: windows + arch: 'ARM64' os: windows-latest - archs: 'AMD64,ARM64,x86' - - name: macos arm64 + - platform: windows + arch: 'x86' + os: windows-latest + - platform: macos + arch: 'arm64' os: macos-latest - archs: 'aarch64' - - name: macos x86 + - platform: macos + arch: 'x86_64' os: macos-13 - archs: 'x86_64' - - name: linux + #cibw_version: 2.18 # later versions do not work + - platform: linux + arch: 'x86_64' + os: ubuntu-latest + - platform: linux + arch: 'aarch64' os: ubuntu-latest - archs: 'x86_64,aarch64' steps: - uses: actions/checkout@v4 - name: Set up Python @@ -44,9 +57,10 @@ jobs: run: | python -m pip install --upgrade pip wheel setuptools twine - name: Build wheels - uses: pypa/cibuildwheel@v2.20 + uses: pypa/cibuildwheel@v${{ matrix.cibw_version }} env: - CIBW_ARCHS: ${{ matrix.archs }} + CIBW_PLATFORM: ${{ matrix.platform }} + CIBW_ARCHS: ${{ matrix.arch }} with: output-dir: dist - name: Twine check @@ -56,7 +70,7 @@ jobs: uses: actions/upload-artifact@v4 with: path: dist - name: ${{ matrix.name }}-build + name: ${{ matrix.platform }}-${{ matrix.arch }}-build # # These release builds uses QEMU so that we can build wheels for arm64. diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 83a584e3..2711e1fe 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -63,7 +63,9 @@ def extract_file(zip_filename, member, path): def get_os_string(): - if sys.platform.startswith("win"): + if os.environ.get("CIBUILDWHEEL") == "1" and os.getenv("CIBW_PLATFORM"): + return os.getenv("CIBW_PLATFORM") + elif sys.platform.startswith("win"): return "windows" elif sys.platform.startswith("darwin"): return "macos" @@ -81,15 +83,29 @@ def get_arch(): is_64_bit = sys.maxsize > 2**32 machine = platform.machine() - # See if this is run by cibuildwheel and check to see if ARCHFLAGS is - # specified (only done on macOS). This allows to select the proper binaries. - # For specifics of CIBUILDWHEEL and macOS build envs, see: - # https://github.com/pypa/cibuildwheel/blob/4307b52ff28b631519d38bfa0dd09d6a9b39a81e/cibuildwheel/macos.py#L277 - if os.environ.get("CIBUILDWHEEL") == "1" and "ARCHFLAGS" in os.environ: - archflags = os.environ["ARCHFLAGS"] - return "aarch64" if "arm64" in archflags else "x86_64" - - if machine == "armv7l": + if os.environ.get("CIBUILDWHEEL") == "1": + # When running in cibuildwheel, we derive the intended arch from + # an env var (the same one that cibuildwheel uses) that we set in cd.yml. + cibw_arch = os.getenv("CIBW_ARCHS") # must be singular + map = { + "windows": { + "AMD64": "x86_64", + "ARM64": "aarch64", + "x86": "i868", + }, + "macos": { + "arm64": "aarch64", + "x86_64": "x86_64", + }, + "linux": { + "x86_64": "x86_64", + "aarch64": "aarch64", + "i868": "i868", + }, + } + maps_for_os = arch_map[get_os_string()] + return maps_for_os[cibw_arch] + elif machine == "armv7l": # Raspberry pi return "armv7" elif is_64_bit and machine.startswith(("arm", "aarch64")): From 93fcd95f47e82945b0800975164f523a67489924 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 13:21:25 +0200 Subject: [PATCH 05/23] fix cd script --- .github/workflows/cd.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 7eb3d053..ce61157d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -55,14 +55,12 @@ jobs: python-version: '3.12' - name: Install dev dependencies run: | - python -m pip install --upgrade pip wheel setuptools twine + python -m pip install --upgrade pip wheel setuptools twine cibuildwheel=${{ matrix.cibw_version }} - name: Build wheels - uses: pypa/cibuildwheel@v${{ matrix.cibw_version }} + run: python -m cibuildwheel --output-dir dist env: CIBW_PLATFORM: ${{ matrix.platform }} CIBW_ARCHS: ${{ matrix.arch }} - with: - output-dir: dist - name: Twine check run: | twine check dist/* From 9ddf7b84454e8b9c30197a779b592b51b043fef2 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 13:22:21 +0200 Subject: [PATCH 06/23] typo --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ce61157d..beb529fa 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -55,7 +55,7 @@ jobs: python-version: '3.12' - name: Install dev dependencies run: | - python -m pip install --upgrade pip wheel setuptools twine cibuildwheel=${{ matrix.cibw_version }} + python -m pip install --upgrade pip wheel setuptools twine cibuildwheel==${{ matrix.cibw_version }} - name: Build wheels run: python -m cibuildwheel --output-dir dist env: From 418e6672a6227069438ccc35d2e41ffc3166497d Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 14:45:02 +0200 Subject: [PATCH 07/23] wut --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index beb529fa..ab9e7f36 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - cibw_version: [2.20] + #cibw_version: [2.20] include: - platform: windows arch: 'AMD64' @@ -55,7 +55,7 @@ jobs: python-version: '3.12' - name: Install dev dependencies run: | - python -m pip install --upgrade pip wheel setuptools twine cibuildwheel==${{ matrix.cibw_version }} + python -m pip install --upgrade pip wheel setuptools twine cibuildwheel==2.20 - name: Build wheels run: python -m cibuildwheel --output-dir dist env: From 87f5760a4a43bde80318c71ab6d75366872b5901 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 14:52:03 +0200 Subject: [PATCH 08/23] ok, need true platform --- .github/workflows/cd.yml | 7 +++---- download-wgpu-native.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index ab9e7f36..dc7e85c6 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -19,11 +19,10 @@ jobs: release-builds: name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }} timeout-minutes: 10 - runs-on: ubuntu-latest + runs-on: $${{ matrix.os }} strategy: fail-fast: false matrix: - #cibw_version: [2.20] include: - platform: windows arch: 'AMD64' @@ -40,7 +39,7 @@ jobs: - platform: macos arch: 'x86_64' os: macos-13 - #cibw_version: 2.18 # later versions do not work + cibw_version: '==2.18' # later versions do not work - platform: linux arch: 'x86_64' os: ubuntu-latest @@ -55,7 +54,7 @@ jobs: python-version: '3.12' - name: Install dev dependencies run: | - python -m pip install --upgrade pip wheel setuptools twine cibuildwheel==2.20 + python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}} - name: Build wheels run: python -m cibuildwheel --output-dir dist env: diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 2711e1fe..5c422626 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -87,7 +87,7 @@ def get_arch(): # When running in cibuildwheel, we derive the intended arch from # an env var (the same one that cibuildwheel uses) that we set in cd.yml. cibw_arch = os.getenv("CIBW_ARCHS") # must be singular - map = { + arch_map = { "windows": { "AMD64": "x86_64", "ARM64": "aarch64", From 68e1c206b58339b72b754a5f2b07fcb10e585c3d Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 15:00:05 +0200 Subject: [PATCH 09/23] comment and fix --- .github/workflows/cd.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index dc7e85c6..f865d502 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,3 +1,14 @@ +# Github Actions script to produce binary wheels. +# +# We perform one build for each wheel that we generate, i.e. one per architecture. +# In download_wgpu_native.py, we detect CIBW_PLATFORM and CIBW_ARCHS to determine +# the required binary from wgpu-native. +# +# If https://github.com/pypa/cibuildwheel/issues/944 gets implemented, we can build more wheels per build. +# +# Also includes the sdist build that does not include a binary. + + name: CD on: @@ -11,15 +22,10 @@ on: jobs: - - # We perform one release build for each wheel that we generate, i.e. one per architecture. - # We use CIBW_ARCHS in download_wgpu_native.py to detect what binary is needed. - # If https://github.com/pypa/cibuildwheel/issues/944 gets implemented, we can build more wheels per build. - # Some builds can be properly tested, some cannot. release-builds: name: Build wheel for ${{ matrix.platform }} ${{ matrix.arch }} timeout-minutes: 10 - runs-on: $${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: From f0c909e2fc27aa07b03d5596215941b271ca51c2 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 15:13:10 +0200 Subject: [PATCH 10/23] fix builds --- .github/workflows/cd.yml | 29 +++++++++++++++++------------ download-wgpu-native.py | 8 ++++++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f865d502..347e33c0 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -31,33 +31,38 @@ jobs: matrix: include: - platform: windows - arch: 'AMD64' + arch: AMD64 os: windows-latest - platform: windows - arch: 'ARM64' + arch: ARM64 os: windows-latest - platform: windows - arch: 'x86' + arch: x86 os: windows-latest - platform: macos - arch: 'arm64' + arch: arm64 os: macos-latest - platform: macos - arch: 'x86_64' - os: macos-13 - cibw_version: '==2.18' # later versions do not work + arch: x86_64 + os: macos-12 + #cibw_version: '==2.18' # later versions do not work - platform: linux - arch: 'x86_64' + arch: x86_64 os: ubuntu-latest - platform: linux - arch: 'aarch64' + arch: aarch64 os: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up QEMU + if: matrix.platform == 'linux' && matrix.arch == 'aarch64' + uses: docker/setup-qemu-action@v3 with: - python-version: '3.12' + platforms: arm64 + # - name: Set up Python + # uses: actions/setup-python@v5 + # with: + # python-version: '3.12' - name: Install dev dependencies run: | python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}} diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 5c422626..27ffa3d6 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -87,11 +87,15 @@ def get_arch(): # When running in cibuildwheel, we derive the intended arch from # an env var (the same one that cibuildwheel uses) that we set in cd.yml. cibw_arch = os.getenv("CIBW_ARCHS") # must be singular + if not cibw_arch: + raise RuntimeError("CIBW_ARCHS not set") + elif "," in cibw_arch: + raise RuntimeError("CIBW_ARCHS must have a single arch") arch_map = { "windows": { "AMD64": "x86_64", "ARM64": "aarch64", - "x86": "i868", + "x86": "i686", }, "macos": { "arm64": "aarch64", @@ -100,7 +104,7 @@ def get_arch(): "linux": { "x86_64": "x86_64", "aarch64": "aarch64", - "i868": "i868", + "i868": "i686", }, } maps_for_os = arch_map[get_os_string()] From d23404459f30210cb5d3844cae90595e93f4ea86 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:13:26 +0200 Subject: [PATCH 11/23] more fix --- .github/workflows/cd.yml | 8 ++------ download-wgpu-native.py | 25 ++++++++++++++----------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 347e33c0..461da3b6 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -44,8 +44,8 @@ jobs: os: macos-latest - platform: macos arch: x86_64 - os: macos-12 - #cibw_version: '==2.18' # later versions do not work + os: macos-13 # latest Intel MacOS + cibw_version: '==2.16' # delocation does not work for later versions - platform: linux arch: x86_64 os: ubuntu-latest @@ -59,10 +59,6 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: arm64 - # - name: Set up Python - # uses: actions/setup-python@v5 - # with: - # python-version: '3.12' - name: Install dev dependencies run: | python -m pip install --upgrade pip wheel setuptools twine cibuildwheel${{ matrix.cibw_version}} diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 27ffa3d6..0abbf61c 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -83,12 +83,23 @@ def get_arch(): is_64_bit = sys.maxsize > 2**32 machine = platform.machine() + if machine == "armv7l": + # Raspberry pi + detected_arch = "armv7" + elif is_64_bit and machine.startswith(("arm", "aarch64")): + # Includes MacOS M1, arm linux, ... + detected_arch = "aarch64" + elif is_64_bit: + detected_arch = "x86_64" + else: + detected_arch = "i686" + if os.environ.get("CIBUILDWHEEL") == "1": # When running in cibuildwheel, we derive the intended arch from # an env var (the same one that cibuildwheel uses) that we set in cd.yml. cibw_arch = os.getenv("CIBW_ARCHS") # must be singular if not cibw_arch: - raise RuntimeError("CIBW_ARCHS not set") + cibw_arch = detected_arch # running under virtualisation (e.g. Docker) elif "," in cibw_arch: raise RuntimeError("CIBW_ARCHS must have a single arch") arch_map = { @@ -109,16 +120,8 @@ def get_arch(): } maps_for_os = arch_map[get_os_string()] return maps_for_os[cibw_arch] - elif machine == "armv7l": - # Raspberry pi - return "armv7" - elif is_64_bit and machine.startswith(("arm", "aarch64")): - # Includes MacOS M1, arm linux, ... - return "aarch64" - elif is_64_bit: - return "x86_64" - else: - return "i686" + + return detected_arch def main(version, os_string, arch, upstream): From cec5aaedd75732bc15b4e5caff8f2be110075ff6 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:27:47 +0200 Subject: [PATCH 12/23] better test --- .github/workflows/cd.yml | 36 +++++++----------------------------- download-wgpu-native.py | 3 ++- 2 files changed, 9 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 461da3b6..c2a92b33 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -33,6 +33,7 @@ jobs: - platform: windows arch: AMD64 os: windows-latest + testable: true - platform: windows arch: ARM64 os: windows-latest @@ -42,6 +43,7 @@ jobs: - platform: macos arch: arm64 os: macos-latest + testable: true - platform: macos arch: x86_64 os: macos-13 # latest Intel MacOS @@ -49,6 +51,7 @@ jobs: - platform: linux arch: x86_64 os: ubuntu-latest + testable: true - platform: linux arch: aarch64 os: ubuntu-latest @@ -70,6 +73,10 @@ jobs: - name: Twine check run: | twine check dist/* + - name: Test wheel + if: matrix.testable + run: | + python -c 'import wgpu.backends.wgpu_native' - name: Upload distributions uses: actions/upload-artifact@v4 with: @@ -77,35 +84,6 @@ jobs: name: ${{ matrix.platform }}-${{ matrix.arch }}-build - # # These release builds uses QEMU so that we can build wheels for arm64. - # # We build wheels and upload the wheel as an artifact, but we don't test them here. - # qemu-release-builds: - # name: Build wheels on ubuntu-latest with QEMU - # timeout-minutes: 10 - # runs-on: ubuntu-latest - # strategy: - # fail-fast: false - # steps: - # - uses: actions/checkout@v4 - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v3 - # with: - # platforms: arm64 - # - name: Build wheels - # uses: pypa/cibuildwheel@v2.20 - # env: - # CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64 - # CIBW_ARCHS_LINUX: aarch64 - # CIBW_SKIP: cp39-musllinux_aarch64 - # with: - # output-dir: dist - # - name: Upload distributions - # uses: actions/upload-artifact@v4 - # with: - # path: dist - # name: qemu-build - - sdist-build: name: Build sdist timeout-minutes: 5 diff --git a/download-wgpu-native.py b/download-wgpu-native.py index 0abbf61c..793d9921 100644 --- a/download-wgpu-native.py +++ b/download-wgpu-native.py @@ -99,7 +99,8 @@ def get_arch(): # an env var (the same one that cibuildwheel uses) that we set in cd.yml. cibw_arch = os.getenv("CIBW_ARCHS") # must be singular if not cibw_arch: - cibw_arch = detected_arch # running under virtualisation (e.g. Docker) + # Linux builds run on Docker, so env is not visible + cibw_arch = detected_arch elif "," in cibw_arch: raise RuntimeError("CIBW_ARCHS must have a single arch") arch_map = { From 105f37b802ed555b05224e703cd69bd1192c3cde Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:35:44 +0200 Subject: [PATCH 13/23] fix test --- .github/workflows/cd.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c2a92b33..189d9054 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -76,7 +76,14 @@ jobs: - name: Test wheel if: matrix.testable run: | - python -c 'import wgpu.backends.wgpu_native' + rm -rf ./wgpu + pushd $HOME + pip install $GITHUB_WORKSPACE/dist/*.whl + python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' + # cleanup + popd + pip uninstall -y wgpu + git reset --hard HEAD - name: Upload distributions uses: actions/upload-artifact@v4 with: From e71229b4ebf56b12b3293b0270a1a233f33124fe Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:41:09 +0200 Subject: [PATCH 14/23] fix win --- .github/workflows/cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 189d9054..fd5daab5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -75,6 +75,7 @@ jobs: twine check dist/* - name: Test wheel if: matrix.testable + shell: bash run: | rm -rf ./wgpu pushd $HOME From 0dab58305a542eb72443a580f47e2183d3966618 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:46:13 +0200 Subject: [PATCH 15/23] try again --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index fd5daab5..a2c1cc7e 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -78,8 +78,8 @@ jobs: shell: bash run: | rm -rf ./wgpu - pushd $HOME - pip install $GITHUB_WORKSPACE/dist/*.whl + pushd dist + pip install *.whl python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup popd From 6f2b022c82a8c4eec2121817f2481cffa6fd2a86 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 16:55:11 +0200 Subject: [PATCH 16/23] safer --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index a2c1cc7e..07aee463 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -78,8 +78,8 @@ jobs: shell: bash run: | rm -rf ./wgpu - pushd dist - pip install *.whl + python -m pip install cffi + python -m pip install --no-index --find-links dist wgpu python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup popd From 353aad49388028eaa5721dd95141099e04df30a5 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:01:14 +0200 Subject: [PATCH 17/23] add pushd --- .github/workflows/cd.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 07aee463..75b7ba10 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -80,6 +80,7 @@ jobs: rm -rf ./wgpu python -m pip install cffi python -m pip install --no-index --find-links dist wgpu + pushd $HOME python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup popd @@ -115,9 +116,7 @@ jobs: shell: bash run: | rm -rf ./wgpu - pushd $HOME - pip install $GITHUB_WORKSPACE/dist/*.tar.gz - popd + python -m pip install --no-index --find-links dist wgpu # don't run tests, we just want to know if the sdist can be installed pip uninstall -y wgpu git reset --hard HEAD From ff0683216bf6a0debe8e35fec24d11964cb7a7e2 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:06:00 +0200 Subject: [PATCH 18/23] cannot install deps with no-index --- .github/workflows/cd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 75b7ba10..2a44cbd8 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -79,7 +79,7 @@ jobs: run: | rm -rf ./wgpu python -m pip install cffi - python -m pip install --no-index --find-links dist wgpu + python -m pip install --no-index --no-dependencies --find-links dist wgpu pushd $HOME python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup @@ -116,7 +116,7 @@ jobs: shell: bash run: | rm -rf ./wgpu - python -m pip install --no-index --find-links dist wgpu + python -m pip install --no-index --no-dependencies --find-links dist wgpu # don't run tests, we just want to know if the sdist can be installed pip uninstall -y wgpu git reset --hard HEAD From be9aec7892fc2b37b8bb206a6aa38cc2ae8b09af Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:11:29 +0200 Subject: [PATCH 19/23] maybe like this --- .github/workflows/cd.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2a44cbd8..1b645c7d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -78,9 +78,9 @@ jobs: shell: bash run: | rm -rf ./wgpu - python -m pip install cffi - python -m pip install --no-index --no-dependencies --find-links dist wgpu pushd $HOME + pip install $GITHUB_WORKSPACE/dist/*.whl + popd python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup popd @@ -116,7 +116,9 @@ jobs: shell: bash run: | rm -rf ./wgpu - python -m pip install --no-index --no-dependencies --find-links dist wgpu + pushd $HOME + pip install $GITHUB_WORKSPACE/dist/*.tar.gz + popd # don't run tests, we just want to know if the sdist can be installed pip uninstall -y wgpu git reset --hard HEAD From aeabdca39b06c7205f2a34af6a5996ad0de04cd9 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:16:47 +0200 Subject: [PATCH 20/23] had an extta popd --- .github/workflows/cd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1b645c7d..fd5daab5 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -80,7 +80,6 @@ jobs: rm -rf ./wgpu pushd $HOME pip install $GITHUB_WORKSPACE/dist/*.whl - popd python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' # cleanup popd From 5140fd3f2e80bb850b1755467800ecd51b40fc74 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:22:01 +0200 Subject: [PATCH 21/23] sod it --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index fd5daab5..c0677700 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -33,7 +33,7 @@ jobs: - platform: windows arch: AMD64 os: windows-latest - testable: true + testable: false # pip install dist/*.whl does not work on Windows :/ - platform: windows arch: ARM64 os: windows-latest From a556c77e6711f806b6ee8bee219ce47d07000783 Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Fri, 13 Sep 2024 17:27:20 +0200 Subject: [PATCH 22/23] comments --- .github/workflows/cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c0677700..94204254 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,5 +1,7 @@ # Github Actions script to produce binary wheels. # +# Note that a lot of the cibuildwheel config is in pyproject.toml. +# # We perform one build for each wheel that we generate, i.e. one per architecture. # In download_wgpu_native.py, we detect CIBW_PLATFORM and CIBW_ARCHS to determine # the required binary from wgpu-native. @@ -46,7 +48,7 @@ jobs: testable: true - platform: macos arch: x86_64 - os: macos-13 # latest Intel MacOS + os: macos-13 # last Intel MacOS cibw_version: '==2.16' # delocation does not work for later versions - platform: linux arch: x86_64 From e25118e99eade3f815099b0d8531a45f498b9e1c Mon Sep 17 00:00:00 2001 From: Almar Klein Date: Sat, 14 Sep 2024 14:33:31 +0200 Subject: [PATCH 23/23] try windows test --- .github/workflows/cd.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 94204254..dac18a16 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -35,7 +35,7 @@ jobs: - platform: windows arch: AMD64 os: windows-latest - testable: false # pip install dist/*.whl does not work on Windows :/ + testable: true - platform: windows arch: ARM64 os: windows-latest @@ -80,10 +80,10 @@ jobs: shell: bash run: | rm -rf ./wgpu + filename=$(ls dist/*.whl) + pip install $filename pushd $HOME - pip install $GITHUB_WORKSPACE/dist/*.whl python -c 'import wgpu.backends.wgpu_native; print(wgpu.backends.wgpu_native._ffi.lib_path)' - # cleanup popd pip uninstall -y wgpu git reset --hard HEAD @@ -117,9 +117,8 @@ jobs: shell: bash run: | rm -rf ./wgpu - pushd $HOME - pip install $GITHUB_WORKSPACE/dist/*.tar.gz - popd + filename=$(ls dist/*.tar.gz) + pip install $filename # don't run tests, we just want to know if the sdist can be installed pip uninstall -y wgpu git reset --hard HEAD