#12273: Move full wheel build on GitHub runners and 22.04 to schedule…

…d job and fix wheel-related workflows to work with TT_FROM_PRECOMPILED (#12535) * #12273: Pass down from-precompiled as an option from eager-package callers so we can control whether or not we use build artifacts in wheel build * #12273: Try changing name of build-artifact so it doesn't conflict? * #12273: Run prepare-metal-run to download artifacts only if they're actually created, which should only be the case if we're building from precompiled * #12273: Try github expression wrapper * #12273: Split into multiple workflows so that we can call build + test wheels from anywhere, and the separate build + test is a separate workflow * #12273: Add push trigger for this branch * #12273: Condition precompiled flow with event name and put back installing metal deps for all cases for host wheel tests. Will see if this breaks something because precompiled binary links to python3.8 lib * #12273: Move host wheel jobs back to github hosted to see if we can start using clean 22.04 environments again. We should move this back to docker when we can * #12273: Only build and test on 22.04 if we're not using precompiled assets as we only build artifacts on 20.04 at the moment. Should be fixed with docker + 22.04 support * #12273: Use fromJSON triclk * #12273: Combine expression and use in both places * #12273: Fix typos for build-and-test-wheels * #12273: Damn man can't write node eh * Revert "#12273: Add push trigger for this branch" This reverts commit e1dc1ae.
tenstorrent · Sep 11, 2024 · 003594f · 003594f
1 parent 74267d4
commit 003594f
Show file tree

Hide file tree

Showing 6 changed files with 71 additions and 39 deletions.
diff --git a/.github/workflows/eager-package-main.yaml → ...orkflows/_build-and-test-wheels-impl.yaml b/.github/workflows/eager-package-main.yaml → ...orkflows/_build-and-test-wheels-impl.yaml
@@ -1,35 +1,41 @@
-name: "[post-commit] Python wheel build and test"
+name: "[internal] Python wheels build and test impl"
 
 on:
-  workflow_dispatch:
   workflow_call:
+    inputs:
+      from-precompiled:
+        description: "Use precompiled assets for wheel build"
+        default: True
+        type: boolean
 
-# TODO: RK - need to change this to every push later once
-# we go public
 jobs:
-  build-eager-package:
+  build-wheels:
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        # Since pre-compiled builds only run on 20.04, we can only test on 20.04 for now
+        # The full 22.04 flow can be tested without precompiled
+        os: ${{ fromJson(inputs.from-precompiled && '["ubuntu-20.04"]' || '["ubuntu-20.04", "ubuntu-22.04"]') }}
         arch: [grayskull, wormhole_b0]
-    uses: ./.github/workflows/eager-package.yaml
+    uses: ./.github/workflows/_build-wheels-impl.yaml
     with:
-      environment: dev
       os: ${{ matrix.os }}
       arch: ${{ matrix.arch }}
-  test-eager-package-host:
-    needs: build-eager-package
+      from-precompiled: ${{ inputs.from-precompiled }}
+  test-wheels-host:
+    needs: build-wheels
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: ${{ fromJson(inputs.from-precompiled && '["ubuntu-20.04"]' || '["ubuntu-20.04", "ubuntu-22.04"]') }}
         runner-hw-info: [
-          {arch: grayskull, type: E150},
-          {arch: wormhole_b0, type: N150}
+          {arch: grayskull},
+          {arch: wormhole_b0}
         ]
-    runs-on: ["cloud-virtual-machine", "${{ matrix.runner-hw-info.type }}", "in-service"]
-    environment: production
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/install-metal-deps
+        with:
+          os: ${{ matrix.os }}
       - uses: actions/download-artifact@v4
         with:
           name: eager-dist-${{ matrix.os }}-${{ matrix.runner-hw-info.arch }}
@@ -41,8 +47,8 @@ jobs:
           source tests/end_to_end_tests/env/bin/activate
           cd tests/end_to_end_tests
           pytest -c conftest.py . -m eager_host_side
-  test-eager-package-silicon:
-    needs: build-eager-package
+  test-wheels-silicon:
+    needs: build-wheels
     strategy:
       matrix:
         os: [ubuntu-20.04]
@@ -52,7 +58,6 @@ jobs:
           {arch: wormhole_b0, type: N300}
         ]
     runs-on: ["cloud-virtual-machine", "${{ matrix.runner-hw-info.type }}", "in-service"]
-    environment: production
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4

diff --git a/.github/workflows/eager-package.yaml → .github/workflows/_build-wheels-impl.yaml b/.github/workflows/eager-package.yaml → .github/workflows/_build-wheels-impl.yaml
@@ -1,22 +1,22 @@
-name: "[internal] Python wheel build and test impl"
+name: "[internal] Python wheels build impl"
 
 on:
   workflow_call:
     inputs:
-      environment:
-        required: True
-        type: string
       os:
         required: True
         type: string
       arch:
         required: True
         type: string
+      from-precompiled:
+        required: True
+        default: True
+        type: boolean
 
 jobs:
-  build-eager-package:
+  build-wheel:
     runs-on: ${{ inputs.os }}
-    environment: ${{ inputs.environment }}
     env:
       TT_METAL_ENV: ${{ vars.TT_METAL_ENV }}
       ARCH_NAME: ${{ inputs.arch }}
@@ -51,13 +51,15 @@ jobs:
         run: |
           echo "DEVICE_CXX=g++" >> $GITHUB_ENV
       - uses: ./.github/actions/prepare-metal-run
+        if: ${{ inputs.from-precompiled }}
         with:
           arch: ${{ inputs.arch }}
       - name: Build Python package distribution
-        run: |
-          TT_FROM_PRECOMPILED=True python -m build
-        # TT_FROM_PRECOMPILED env variable allows us to not re-run `make install` and instead
-        # rely on the artifact that was already compiled.
+        env:
+          # TT_FROM_PRECOMPILED env variable allows us to not re-run the full C++ build and instead
+          # rely on the artifact that was already compiled.
+          TT_FROM_PRECOMPILED: ${{ inputs.from-precompiled && 'True' || 'False' }}
+        run: python -m build
       - name: Upload distribution as artifact
         uses: actions/upload-artifact@v4
         with:

diff --git a/.github/workflows/all-post-commit-workflows.yaml b/.github/workflows/all-post-commit-workflows.yaml
@@ -17,9 +17,11 @@ jobs:
   static-checks:
     uses: ./.github/workflows/all-static-checks.yaml
     secrets: inherit
-  build-wheels:
+  build-and-test-wheels:
     needs: build-artifact
-    uses: ./.github/workflows/eager-package-main.yaml
+    uses: ./.github/workflows/_build-and-test-wheels-impl.yaml
+    with:
+      from-precompiled: true
     secrets: inherit
   build-artifact:
     uses: ./.github/workflows/build-artifact.yaml
@@ -62,7 +64,7 @@ jobs:
       runner-label: ${{ matrix.test-group.runner-label }}
   # Fast Dispatch Unit Tests
   fast-dispatch-unit-tests:
-    needs: build-wheels
+    needs: build-and-test-wheels
     secrets: inherit
     strategy:
       fail-fast: false
@@ -78,7 +80,7 @@ jobs:
       runner-label: ${{ matrix.test-group.runner-label }}
   # TTNN FD Unit tests
   ttnn-unit-tests:
-    needs: build-wheels
+    needs: build-and-test-wheels
     secrets: inherit
     strategy:
       fail-fast: false
@@ -94,7 +96,7 @@ jobs:
       runner-label: ${{ matrix.test-group.runner-label }}
   # FD Model Tests
   models-unit-tests:
-    needs: build-wheels
+    needs: build-and-test-wheels
     secrets: inherit
     strategy:
       fail-fast: false

diff --git a/.github/workflows/blackhole-post-commit.yaml b/.github/workflows/blackhole-post-commit.yaml
@@ -64,8 +64,8 @@ jobs:
 #     needs: build-artifact-profiler
 #     uses: ./.github/workflows/run-profiler-regression.yaml
 #     secrets: inherit
-#   eager-package-main:
-#     uses: ./.github/workflows/eager-package-main.yaml
+#   build-and-test-wheels:
+#     uses: ./.github/workflows/_build-and-test-wheels-impl.yaml
 #     secrets: inherit
 #   build-docs:
 #     needs: build-artifact

diff --git a/.github/workflows/build-and-test-wheels.yaml b/.github/workflows/build-and-test-wheels.yaml
@@ -0,0 +1,23 @@
+name: "[post-commit] Python wheels build and test"
+
+on:
+  workflow_dispatch:
+    inputs:
+      from-precompiled:
+        description: "Use precompiled assets for wheel build"
+        default: True
+        type: boolean
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  build-artifact:
+    if: ${{ github.event_name == 'workflow_dispatch' && inputs.from-precompiled }}
+    uses: ./.github/workflows/build-artifact.yaml
+    secrets: inherit
+  build-and-test-wheels:
+    needs: build-artifact
+    if: ${{ always() }}
+    uses: ./.github/workflows/_build-and-test-wheels-impl.yaml
+    with:
+      from-precompiled: ${{ github.event_name == 'workflow_dispatch' && inputs.from-precompiled }}
diff --git a/.github/workflows/package-and-release.yaml b/.github/workflows/package-and-release.yaml
@@ -110,23 +110,23 @@ jobs:
         with:
           name: release-notes
           path: RELEASE_NOTES.txt
-  build-eager-package:
+  build-wheels:
     needs: create-tag
     strategy:
       matrix:
         os: [ubuntu-20.04]
         arch: [grayskull, wormhole_b0]
-    uses: ./.github/workflows/eager-package.yaml
+    uses: ./.github/workflows/_build-wheels-impl.yaml
     with:
-      environment: dev
       os: ${{ matrix.os }}
       arch: ${{ matrix.arch }}
+      from-precompiled: false
   # Candidate for breaking up
   create-and-upload-draft-release:
     needs: [
       create-tag,
       create-release-notes,
-      build-eager-package,
+      build-wheels,
     ]
     strategy:
       matrix: