From 5d21b42dffd998e24b531b52d8a2a3d5fccde62c Mon Sep 17 00:00:00 2001 From: Bill Teng Date: Tue, 26 Mar 2024 01:10:52 +0000 Subject: [PATCH] #6759: enable watcher on all post-commit pipelines --- .github/workflows/build-and-unit-tests.yaml | 7 +++++++ .github/workflows/cpp-post-commit.yaml | 7 +++++++ .../workflows/fast-dispatch-build-and-unit-tests.yaml | 7 +++++++ .github/workflows/models-post-commit.yaml | 7 +++++++ .github/workflows/run-profiler-regression.yaml | 11 +++++++++-- .github/workflows/ttnn-post-commit.yaml | 7 +++++++ 6 files changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-unit-tests.yaml b/.github/workflows/build-and-unit-tests.yaml index 73a377090faf..5e6fa4889996 100644 --- a/.github/workflows/build-and-unit-tests.yaml +++ b/.github/workflows/build-and-unit-tests.yaml @@ -25,6 +25,7 @@ jobs: CONFIG: ci TT_METAL_SLOW_DISPATCH_MODE: 1 LOGURU_LEVEL: INFO + TT_METAL_WATCHER: 60 LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib runs-on: ${{ matrix.runner-info.runs-on }} steps: @@ -43,3 +44,9 @@ jobs: run: | source ${{ github.workspace }}/python_env/bin/activate ./tests/scripts/run_tests.sh --tt-arch $ARCH_NAME --pipeline-type post_commit --dispatch-mode slow + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-sd + path: generated/watcher/watcher.log diff --git a/.github/workflows/cpp-post-commit.yaml b/.github/workflows/cpp-post-commit.yaml index f8f262adbb9f..6c5bda0c2f8e 100644 --- a/.github/workflows/cpp-post-commit.yaml +++ b/.github/workflows/cpp-post-commit.yaml @@ -28,6 +28,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -47,3 +48,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log diff --git a/.github/workflows/fast-dispatch-build-and-unit-tests.yaml b/.github/workflows/fast-dispatch-build-and-unit-tests.yaml index 4f3ec3551315..f5175ca245ba 100644 --- a/.github/workflows/fast-dispatch-build-and-unit-tests.yaml +++ b/.github/workflows/fast-dispatch-build-and-unit-tests.yaml @@ -41,6 +41,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -60,6 +61,12 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log build-docs: strategy: # Do not fail-fast because we need to ensure all tests go to completion diff --git a/.github/workflows/models-post-commit.yaml b/.github/workflows/models-post-commit.yaml index 4d8cfbebe189..dd34f2c86960 100644 --- a/.github/workflows/models-post-commit.yaml +++ b/.github/workflows/models-post-commit.yaml @@ -28,6 +28,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -47,3 +48,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log diff --git a/.github/workflows/run-profiler-regression.yaml b/.github/workflows/run-profiler-regression.yaml index 6e7e5c4cd4dc..4dba44bfac5b 100644 --- a/.github/workflows/run-profiler-regression.yaml +++ b/.github/workflows/run-profiler-regression.yaml @@ -15,9 +15,9 @@ jobs: runner-info: [ # No GS as tests now require synced starts. GS profiler tests will run on dedicated BMs # N150 - {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"]}, + {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-1"], name: N150}, # N300 - {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"]}, + {arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-1", "multi-chip-num-chips-2"], name: N300}, # N300 2x4 {name: "n300-2x4", arch: wormhole_b0, runs-on: ["wormhole_b0", "multi-chip-num-pcie-4", "multi-chip-num-chips-8"]}, ] @@ -26,6 +26,7 @@ jobs: ARCH_NAME: ${{ matrix.runner-info.arch }} CONFIG: ci LOGURU_LEVEL: INFO + TT_METAL_WATCHER: 60 environment: dev runs-on: ${{ matrix.runner-info.runs-on }} steps: @@ -40,3 +41,9 @@ jobs: timeout-minutes: 30 run: | ./tests/scripts/run_profiler_regressions.sh + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-profiler + path: generated/watcher/watcher.log diff --git a/.github/workflows/ttnn-post-commit.yaml b/.github/workflows/ttnn-post-commit.yaml index 0726f3a252ef..5cf552d5fb3a 100644 --- a/.github/workflows/ttnn-post-commit.yaml +++ b/.github/workflows/ttnn-post-commit.yaml @@ -30,6 +30,7 @@ jobs: CONFIG: ci LOGURU_LEVEL: INFO LD_LIBRARY_PATH: ${{ github.workspace }}/build/lib + TT_METAL_WATCHER: 60 runs-on: ${{ matrix.runner-info.runs-on }} steps: - uses: tenstorrent-metal/metal-workflows/.github/actions/checkout-with-submodule-lfs@v2.0.0 @@ -49,3 +50,9 @@ jobs: cd $TT_METAL_HOME export PYTHONPATH=$TT_METAL_HOME ${{ matrix.test-group.cmd }} + - name: Upload watcher log + if: always() + uses: actions/upload-artifact@v4 + with: + name: watcher-log-${{ matrix.runner-info.arch }}-${{ matrix.runner-info.name }}-${{ matrix.test-group.name }} + path: generated/watcher/watcher.log