diff --git a/.github/workflows/build-and-unit-tests.yaml b/.github/workflows/build-and-unit-tests.yaml index 98612022c5d5..7b3d88ad675d 100644 --- a/.github/workflows/build-and-unit-tests.yaml +++ b/.github/workflows/build-and-unit-tests.yaml @@ -25,6 +25,7 @@ jobs: CONFIG: ci TT_METAL_SLOW_DISPATCH_MODE: 1 LOGURU_LEVEL: INFO + TT_METAL_WATCHER: 60 LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib runs-on: ${{ matrix.runner-info.runs-on }} steps: @@ -45,3 +46,9 @@ jobs: run: | source $HOME/python_env/bin/activate ./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type post_commit --dispatch-mode slow + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-sd + path: generated/watcher/watcher.log diff --git a/.github/workflows/cpp-post-commit.yaml b/.github/workflows/cpp-post-commit.yaml index 222a0023122e..6a8bab687924 100644 --- a/.github/workflows/cpp-post-commit.yaml +++ b/.github/workflows/cpp-post-commit.yaml @@ -28,6 +28,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -49,3 +50,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log diff --git a/.github/workflows/fast-dispatch-build-and-unit-tests.yaml b/.github/workflows/fast-dispatch-build-and-unit-tests.yaml index cd0bbb2228e6..21f232558082 100644 --- a/.github/workflows/fast-dispatch-build-and-unit-tests.yaml +++ b/.github/workflows/fast-dispatch-build-and-unit-tests.yaml @@ -41,6 +41,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -62,6 +63,12 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log build-docs: strategy: # Do not fail-fast because we need to ensure all tests go to completion diff --git a/.github/workflows/models-post-commit.yaml b/.github/workflows/models-post-commit.yaml index e3a72691c4a7..8c4117c47df6 100644 --- a/.github/workflows/models-post-commit.yaml +++ b/.github/workflows/models-post-commit.yaml @@ -28,6 +28,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -49,3 +50,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log diff --git a/.github/workflows/run-profiler-regression.yaml b/.github/workflows/run-profiler-regression.yaml index cee51bedcdc6..97f35156baf7 100644 --- a/.github/workflows/run-profiler-regression.yaml +++ b/.github/workflows/run-profiler-regression.yaml @@ -15,15 +15,16 @@ jobs: runner-info: [ # No GS as tests now require synced starts. GS profiler tests will run on dedicated BMs # N150 - {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"]}, + {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"], name: N150}, # N300 - {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"]}, + {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"], name: N300}, ] env: TT_METAL_ENV: ${{ vars.TT_METAL_ENV }} ARCH_NAME: ${{ matrix.runner-info.arch }} CONFIG: ci LOGURU_LEVEL: INFO + TT_METAL_WATCHER: 60 environment: dev runs-on: ${{ matrix.runner-info.runs-on }} steps: @@ -38,3 +39,9 @@ jobs: timeout-minutes: 30 run: | ./tests/scripts/run_profiler_regressions.sh + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-profiler + path: generated/watcher/watcher.log diff --git a/.github/workflows/ttnn-post-commit.yaml b/.github/workflows/ttnn-post-commit.yaml index 4acafec0eb97..bfbc7ec6c512 100644 --- a/.github/workflows/ttnn-post-commit.yaml +++ b/.github/workflows/ttnn-post-commit.yaml @@ -30,6 +30,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -51,3 +52,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log