#6759: enable watcher on all post-commit pipelines

tenstorrent · Apr 18, 2024 · f3dd7e4 · f3dd7e4
1 parent ef82105
commit f3dd7e4
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 2 deletions.
diff --git a/.github/workflows/build-and-unit-tests.yaml b/.github/workflows/build-and-unit-tests.yaml
@@ -25,6 +25,7 @@ jobs:
       CONFIG: ci
       TT_METAL_SLOW_DISPATCH_MODE: 1
       LOGURU_LEVEL: INFO
+      TT_METAL_WATCHER: 60
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
@@ -43,3 +44,9 @@ jobs:
         run: |
           source ${{ github.workspace }}/python_env/bin/activate
           ./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type post_commit --dispatch-mode slow
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-sd
+          path: generated/watcher/watcher.log
diff --git a/.github/workflows/cpp-post-commit.yaml b/.github/workflows/cpp-post-commit.yaml
@@ -28,6 +28,7 @@ jobs:
       CONFIG: ci
       LOGURU_LEVEL: INFO
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+      TT_METAL_WATCHER: 60
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
       - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
@@ -47,3 +48,9 @@ jobs:
           cd $TT_METAL_HOME
           export PYTHONPATH=$TT_METAL_HOME
           ${{ matrix.test-group.cmd }}
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }}
+          path: generated/watcher/watcher.log
diff --git a/.github/workflows/fast-dispatch-build-and-unit-tests.yaml b/.github/workflows/fast-dispatch-build-and-unit-tests.yaml
@@ -41,6 +41,7 @@ jobs:
       CONFIG: ci
       LOGURU_LEVEL: INFO
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+      TT_METAL_WATCHER: 60
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
       - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
@@ -60,6 +61,12 @@ jobs:
           cd $TT_METAL_HOME
           export PYTHONPATH=$TT_METAL_HOME
           ${{ matrix.test-group.cmd }}
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }}
+          path: generated/watcher/watcher.log
   build-docs:
     strategy:
       # Do not fail-fast because we need to ensure all tests go to completion

diff --git a/.github/workflows/models-post-commit.yaml b/.github/workflows/models-post-commit.yaml
@@ -28,6 +28,7 @@ jobs:
       CONFIG: ci
       LOGURU_LEVEL: INFO
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+      TT_METAL_WATCHER: 60
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
       - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
@@ -47,3 +48,9 @@ jobs:
           cd $TT_METAL_HOME
           export PYTHONPATH=$TT_METAL_HOME
           ${{ matrix.test-group.cmd }}
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }}
+          path: generated/watcher/watcher.log
diff --git a/.github/workflows/run-profiler-regression.yaml b/.github/workflows/run-profiler-regression.yaml
@@ -15,9 +15,9 @@ jobs:
         runner-info: [
           # No GS as tests now require synced starts. GS profiler tests will run on dedicated BMs
           # N150
-          {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"]},
+          {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"], name: N150},
           # N300
-          {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"]},
+          {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"], name: N300},
           # N300 2x4
           {name: "n300-2x4", arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-4", "multi-chip-num-chips-8"]},
         ]
@@ -26,6 +26,7 @@ jobs:
       ARCH_NAME: ${{ matrix.runner-info.arch }}
       CONFIG: ci
       LOGURU_LEVEL: INFO
+      TT_METAL_WATCHER: 60
     environment: dev
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
@@ -40,3 +41,9 @@ jobs:
         timeout-minutes: 30
         run: |
           ./tests/scripts/run_profiler_regressions.sh
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-profiler
+          path: generated/watcher/watcher.log
diff --git a/.github/workflows/ttnn-post-commit.yaml b/.github/workflows/ttnn-post-commit.yaml
@@ -30,6 +30,7 @@ jobs:
       CONFIG: ci
       LOGURU_LEVEL: INFO
       LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib
+      TT_METAL_WATCHER: 60
     runs-on: ${{ matrix.runner-info.runs-on }}
     steps:
       - uses: tenstorrent-metal/metal-workflows/.github/actions/[email protected]
@@ -49,3 +50,9 @@ jobs:
           cd $TT_METAL_HOME
           export PYTHONPATH=$TT_METAL_HOME
           ${{ matrix.test-group.cmd }}
+      - name: Upload watcher log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }}
+          path: generated/watcher/watcher.log