Faster graalpy build (#85)

* Revert "Revert "parallelize build on linux"" This reverts commit 0cebe7b. * Revert "Revert "set parallel shell"" This reverts commit 4faaa1b. * Revert "Revert "Revert "set parallel shell""" This reverts commit 659345b. * Revert "Revert "Revert "parallelize build on linux""" This reverts commit e51c764. * split linux into different jobs * tweak * fix syntax
bjia56 · Aug 7, 2024 · d8b9073 · d8b9073
1 parent 912a914
commit d8b9073
Show file tree

Hide file tree

Showing 2 changed files with 128 additions and 118 deletions.
diff --git a/.github/workflows/repackage_graalpy.yml b/.github/workflows/repackage_graalpy.yml
@@ -1,115 +1,117 @@
-name: Repackage GraalPy
-
-on:
-  workflow_dispatch:
-    inputs:
-      graalpy_version:
-        required: true
-        type: string
-      platforms:
-        required: true
-        type: string
-        default: "linux-x86_64,linux-aarch64,macos,windows"
-  workflow_call:
-    inputs:
-      graalpy_version:
-        required: true
-        type: string
-      platforms:
-        required: true
-        type: string
-
-env:
-  image_map: '{"x86_64": "amd64/ubuntu:20.04", "aarch64": "arm64v8/ubuntu:20.04"}'
-
-jobs:
-  build_linux:
-    name: Linux ${{ inputs.graalpy_version }} ${{ matrix.arch }}
-    if: ${{ contains(inputs.platforms, 'linux') }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        arch: [x86_64, aarch64]
-        exclude:
-          - arch: ${{ !contains(inputs.platforms, 'linux-x86_64') && 'x86_64' || '' }}
-          - arch: ${{ !contains(inputs.platforms, 'linux-aarch64') && 'aarch64' || '' }}
-
-    steps:
-      - name: Parse image
-        id: parse_image
-        run: |
-          IMAGE=$(echo ${{ toJSON(env.image_map) }} | jq -r '.["${{ matrix.arch }}"]')
-          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Repackage
-        run: |
-          export DOCKER_IMAGE="${{ steps.parse_image.outputs.image }}"
-          ./scripts/repackage_graalpy.sh ${{ matrix.arch }} ${{ inputs.graalpy_version }} linux
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: graalpy-linux-${{ matrix.arch }}-${{ inputs.graalpy_version }}
-          path: ./graalpy*.zip
-
-  build_windows:
-    name: Windows ${{ inputs.graalpy_version }} x86_64
-    if: ${{ contains(inputs.platforms, 'windows') }}
-    runs-on: windows-latest
-
-    steps:
-      - name: Set git to use LF
-        run: |
-          git config --global core.autocrlf false
-          git config --global core.eol lf
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Repackage
-        shell: bash
-        run: |
-          set -ex
-          ./scripts/repackage_graalpy.sh x86_64 ${{ inputs.graalpy_version }} windows
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: graalpy-windows-x86_64-${{ inputs.graalpy_version }}
-          path: ./graalpy*.zip
-
-  build_macos:
-    name: MacOS ${{ inputs.graalpy_version }} ${{ matrix.arch }}
-    if: ${{ contains(inputs.platforms, 'macos') }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        arch: [x86_64, aarch64]
-        include:
-          - arch: x86_64
-            runner: macos-12
-          - arch: aarch64
-            runner: macos-14
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Repackage
-        run: |
-          set -ex
-          ./scripts/repackage_graalpy.sh ${{ matrix.arch }} ${{ inputs.graalpy_version }} darwin
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: graalpy-darwin-${{ matrix.arch }}-${{ inputs.graalpy_version }}
+name: Repackage GraalPy
+
+on:
+  workflow_dispatch:
+    inputs:
+      graalpy_version:
+        required: true
+        type: string
+      platforms:
+        required: true
+        type: string
+        default: "linux-x86_64,linux-aarch64,macos,windows"
+  workflow_call:
+    inputs:
+      graalpy_version:
+        required: true
+        type: string
+      platforms:
+        required: true
+        type: string
+
+env:
+  image_map: '{"x86_64": "amd64/ubuntu:20.04", "aarch64": "arm64v8/ubuntu:20.04"}'
+
+jobs:
+  build_linux:
+    name: Linux ${{ inputs.graalpy_version }} ${{ matrix.distribution }} ${{ matrix.arch }}
+    if: ${{ contains(inputs.platforms, 'linux') }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, aarch64]
+        distribution: [standard, jvm, community, community-jvm]
+        exclude:
+          - arch: ${{ !contains(inputs.platforms, 'linux-x86_64') && 'x86_64' || '' }}
+          - arch: ${{ !contains(inputs.platforms, 'linux-aarch64') && 'aarch64' || '' }}
+
+    steps:
+      - name: Parse image
+        id: parse_image
+        run: |
+          IMAGE=$(echo ${{ toJSON(env.image_map) }} | jq -r '.["${{ matrix.arch }}"]')
+          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Repackage
+        run: |
+          export DOCKER_IMAGE="${{ steps.parse_image.outputs.image }}"
+          export MATRIX_DISTRIBUTION="${{ matrix.distribution }}"
+          ./scripts/repackage_graalpy.sh ${{ matrix.arch }} ${{ inputs.graalpy_version }} linux
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: graalpy-linux-${{ matrix.distribution }}-${{ matrix.arch }}-${{ inputs.graalpy_version }}
+          path: ./graalpy*.zip
+
+  build_windows:
+    name: Windows ${{ inputs.graalpy_version }} x86_64
+    if: ${{ contains(inputs.platforms, 'windows') }}
+    runs-on: windows-latest
+
+    steps:
+      - name: Set git to use LF
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Repackage
+        shell: bash
+        run: |
+          set -ex
+          ./scripts/repackage_graalpy.sh x86_64 ${{ inputs.graalpy_version }} windows
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: graalpy-windows-x86_64-${{ inputs.graalpy_version }}
+          path: ./graalpy*.zip
+
+  build_macos:
+    name: MacOS ${{ inputs.graalpy_version }} ${{ matrix.arch }}
+    if: ${{ contains(inputs.platforms, 'macos') }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [x86_64, aarch64]
+        include:
+          - arch: x86_64
+            runner: macos-12
+          - arch: aarch64
+            runner: macos-14
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Repackage
+        run: |
+          set -ex
+          ./scripts/repackage_graalpy.sh ${{ matrix.arch }} ${{ inputs.graalpy_version }} darwin
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: graalpy-darwin-${{ matrix.arch }}-${{ inputs.graalpy_version }}
           path: ./graalpy*.zip
diff --git a/scripts/repackage_graalpy.sh b/scripts/repackage_graalpy.sh
@@ -93,7 +93,15 @@ function repackage_graal () {
   echo "::endgroup::"
 }
 
-repackage_graal
-repackage_graal jvm
-repackage_graal community
-repackage_graal community-jvm
+if [[ "${PLATFORM}" == "linux" ]]; then
+  if [[ "${MATRIX_DISTRIBUTION}" == "standard" ]]; then
+    repackage_graal
+  else
+    repackage_graal "${MATRIX_DISTRIBUTION}"
+  fi
+else
+  repackage_graal
+  repackage_graal jvm
+  repackage_graal community
+  repackage_graal community-jvm
+fi