Make: Separate PyPi CI jobs

ashvardanian · Jan 24, 2025 · 2de6870 · 2de6870
1 parent d883b9d
commit 2de6870
Showing 1 changed file with 153 additions and 18 deletions.
diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -196,33 +196,168 @@ jobs:
       - name: Test Swift
         run: swift test
 
-  build_wheels:
-    name: Build Python ${{ matrix.python-version }} for ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+  # Compiling Python images is a nightmare, assuming the number of platforms we support
+  # To minimize cross-compilation we use separate runners for each platform
+  # https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+  # https://cibuildwheel.pypa.io/en/stable/#what-does-it-do
+  # Available Linux images: https://cibuildwheel.pypa.io/en/stable/options/#linux-image
+  build_wheels_linux_x86:
+    name: Build Python Wheels (Linux x86)
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ["37", "38", "39", "310", "311", "312", "313"]
+    needs: [test_python]
+        steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
+        env:
+          # Matches: `cp37-manylinux_x86_64` and `cp38-musllinux_x86_64`
+          CIBW_BUILD: cp${{ matrix.python-version }}-*
+          CIBW_PLATFORM: linux
+          CIBW_ARCHS: x86_64
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2014
+          CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_2
+
+  build_wheels_linux_arm:
+    name: Build Python Wheels (Linux ARM)
+    runs-on: ubuntu-22.04-arm64
+    strategy:
+      matrix:
+        python-version: ["37", "38", "39", "310", "311", "312", "313"]
     needs: [test_python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
+        env:
+          # Matches: `cp37-manylinux_aarch64` and `cp38-musllinux_aarch64`
+          CIBW_BUILD: cp${{ matrix.python-version }}-*
+          CIBW_PLATFORM: linux
+          CIBW_ARCHS: aarch64
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2014
+          CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_2
+
+  build_wheels_macos_x86:
+    name: Build Python Wheels (macOS x86)
+    runs-on: macos-13
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: ["37", "38", "39", "310", "311", "312", "313"]
+    needs: [test_python]
     steps:
-      - uses: actions/[email protected]
-      - name: Set up Python
-        uses: actions/[email protected]
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.9"
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: cp${{ matrix.python-version }}-*
+          CIBW_PLATFORM: macos
+          CIBW_ARCHS: x86_64
 
-        # We only need QEMU for Linux builds
-      - name: Setup QEMU
-        if: matrix.os == 'ubuntu-latest'
-        uses: docker/setup-qemu-action@v3
-      - name: Upgrade MSVC tooling
-        if: matrix.os == 'windows-latest'
+  build_wheels_macos_arm:
+    name: Build Python Wheels (macOS ARM)
+    runs-on: macos-14
+    strategy:
+      matrix:
+        python-version: ["37", "38", "39", "310", "311", "312", "313"]
+    needs: [test_python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: cp${{ matrix.python-version }}-*
+          CIBW_PLATFORM: macos
+          CIBW_ARCHS: arm64
+
+  build_wheels_windows:
+    name: Build Python Wheels (Windows)
+    runs-on: windows-2022
+    strategy:
+      matrix:
+        python-version: ["37", "38", "39", "310", "311", "312", "313"]
+        architecture: [AMD64, ARM64, x86]
+    needs: [test_python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Setup MSVC
         uses: microsoft/setup-msbuild@v2
         with:
           vs-version: "17.10"
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.22.0
-      - name: Build wheels
-        run: cibuildwheel --output-dir wheelhouse
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_BUILD: cp${{ matrix.python-version }}-win_${{ matrix.architecture }}
+          CIBW_PLATFORM: windows
+
+  build_wheels_other:
+    name: Build Python Wheels (Other Platforms)
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ["37", "38", "39", "310", "311", "312", "313"]
+    needs: [test_python]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Install Toolchain
+        run: |
+          python -m pip install cibuildwheel
+          cibuildwheel --output-dir wheelhouse
         env:
+          # https://cibuildwheel.pypa.io/en/stable/options/#archs
           CIBW_BUILD: cp${{ matrix.python-version }}-*
+          CIBW_PLATFORM: linux
+          CIBW_ARCHS: ppc64le s390x i686 armv7l
+          CIBW_MANYLINUX_I686_IMAGE: manylinux_2014
+          CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux_2014
+          CIBW_MANYLINUX_S390X_IMAGE: manylinux_2014
+          CIBW_MANYLINUX_ARMV7L_IMAGE: manylinux_2_31
+          CIBW_MUSLLINUX_I686_IMAGE: musllinux_1_2
+          CIBW_MUSLLINUX_PPC64LE_IMAGE: musllinux_1_2
+          CIBW_MUSLLINUX_S390X_IMAGE: musllinux_1_2
+          CIBW_MUSLLINUX_ARMV7L_IMAGE: musllinux_1_2
+
+