Use tracy in job and reduce code

tenstorrent · Nov 25, 2024 · 5f566a6 · 5f566a6
1 parent bfdc61c
commit 5f566a6
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 31 deletions.
diff --git a/.github/workflows/tg-model-perf-tests-impl.yaml b/.github/workflows/tg-model-perf-tests-impl.yaml
@@ -49,24 +49,13 @@ jobs:
           echo "TT_METAL_HOME=$(pwd)" >> $GITHUB_ENV
           echo "PYTHONPATH=$(pwd)" >> $GITHUB_ENV
       - name: Download profiler build artifact
-        id: download-profiler-artifact
-        if: ${{ matrix.test-group.tracy }}
         uses: actions/download-artifact@v4
         with:
           name: TTMetal_build_${{ matrix.test-group.arch }}_profiler
-        continue-on-error: true
-      - name: Download build artifact
-        id: download-artifact
-        if: ${{ !matrix.test-group.tracy }}
-        uses: actions/download-artifact@v4
-        with:
-          name: TTMetal_build_${{ matrix.test-group.arch }}
       - name: Extract files
-        if: ${{ matrix.test-group.tracy && steps.download-profiler-artifact.outcome == 'success' || !matrix.test-group.tracy }}
         run: tar -xvf ttm_${{ matrix.test-group.arch }}.tar
       - uses: ./.github/actions/install-python-deps
       - name: Run model perf regression tests
-        if: ${{ matrix.test-group.tracy && steps.download-profiler-artifact.outcome == 'success' || !matrix.test-group.tracy }}
         timeout-minutes: 60
         run: |
           source ${{ github.workspace }}/python_env/bin/activate
@@ -75,16 +64,25 @@ jobs:
           ${{ matrix.test-group.cmd }}
       - name: Check perf report exists
         id: check-perf-report
-        if: ${{ !cancelled() && (matrix.test-group.tracy && steps.download-profiler-artifact.outcome == 'success' || !matrix.test-group.tracy) }}
+        if: ${{ !cancelled() }}
         run: |
           TODAY=$(date +%Y_%m_%d)
           PERF_REPORT_FILENAME_MODELS="Models_Perf_${TODAY}.csv"
-          PERF_REPORT_FILENAME_CCL="CCL_Perf_${TODAY}.csv"
-          if [ "${{ matrix.test-info.tracy }}" == "true" ]; then
-            if [ -f "$PERF_REPORT_FILENAME_CCL" ]; then
-              echo "Found CCL Perf report: $PERF_REPORT_FILENAME_CCL"
-              echo "perf_report_filename=$PERF_REPORT_FILENAME_CCL" >> "$GITHUB_OUTPUT"
-            else
+          PERF_REPORT_FILENAME_CCL_ALL_GATHER="CCL_all_gather_Perf_${TODAY}.csv"
+          PERF_REPORT_FILENAME_CCL_REDUCE_SCATTER="CCL_reduce_scatter_Perf_${TODAY}.csv"
+          if [ "${{ matrix.test-group.tracy }}" == "true" ]; then
+            found_reports=false
+            if [ -f "$PERF_REPORT_FILENAME_CCL_ALL_GATHER" ]; then
+              echo "Found CCL AllGather Perf report: $PERF_REPORT_FILENAME_CCL_ALL_GATHER"
+              echo "perf_report_filename_all_gather=$PERF_REPORT_FILENAME_CCL_ALL_GATHER" >> "$GITHUB_OUTPUT"
+              found_reports=true
+            fi
+            if [ -f "$PERF_REPORT_FILENAME_CCL_REDUCE_SCATTER" ]; then
+              echo "Found CCL ReduceScatter Perf report: $PERF_REPORT_FILENAME_CCL_REDUCE_SCATTER"
+              echo "perf_report_filename_reduce_scatter=$PERF_REPORT_FILENAME_CCL_REDUCE_SCATTER" >> "$GITHUB_OUTPUT"
+              found_reports=true
+            fi
+            if [ "$found_reports" = false ]; then
               echo "No CCL perf report found for today."
               exit 1
             fi

diff --git a/.github/workflows/tg-model-perf-tests.yaml b/.github/workflows/tg-model-perf-tests.yaml
@@ -6,18 +6,13 @@ on:
     - cron: "0 */12 * * *" # This cron schedule runs the workflow every 12 hours
 
 jobs:
-  build-artifact:
-    uses: ./.github/workflows/build-artifact.yaml
-    with:
-      arch: '["wormhole_b0"]'
-    secrets: inherit
   build-artifact-profiler:
     uses: ./.github/workflows/build-artifact.yaml
     with:
       arch: '["wormhole_b0"]'
       tracy: true
     secrets: inherit
   tg-model-perf-tests:
-    needs: [build-artifact, build-artifact-profiler]
+    needs: build-artifact-profiler
     secrets: inherit
     uses: ./.github/workflows/tg-model-perf-tests-impl.yaml
diff --git a/tests/scripts/run_tests.sh b/tests/scripts/run_tests.sh
@@ -241,12 +241,6 @@ model_perf_tg_device() {
 
     ./tests/scripts/tg/run_tg_model_perf_tests.sh --pipeline-type "$pipeline_type"
 }
-
-# Run ccl model perf tests
-ccl_perf_tg_device() {
-
-    ./tests/ttnn/unit_tests/operations/ccl/perf/run_all_gather_profile.sh -t tg
-}
 ##########################TG##########################
 
 ##########################TGG##########################
@@ -328,7 +322,7 @@ run_pipeline_tests() {
     elif [[ $pipeline_type == *"model_perf_tg_device" ]]; then
         model_perf_tg_device "$tt_arch" "$pipeline_type" "$dispatch_mode"
     elif [[ $pipeline_type == "ccl_all_gather_perf_tg_device" ]]; then
-        ccl_perf_tg_device
+        ./tests/ttnn/unit_tests/operations/ccl/perf/run_all_gather_profile.sh -t tg
     # TGG pipelines
     elif [[ $pipeline_type == "unit_tgg_device" ]]; then
         unit_tgg_device "$tt_arch" "$pipeline_type" "$dispatch_mode"