From 92f44cb150aae75dc567cfb9a44cea7bf1022ff9 Mon Sep 17 00:00:00 2001
From: Raymond Kim <rkim@tenstorrent.com>
Date: Wed, 11 Dec 2024 16:38:01 -0500
Subject: [PATCH] #0: Skip majority of mamba tests due to instability, take out
 pgm tests to move to a different pipeline, skip failing SD tests

---
 ...atch-full-regressions-and-models-impl.yaml | 28 ++++++++-----------
 .../tests/test_unet_2d_condition_model.py     |  1 +
 .../stable_diffusion/tests/test_upblock_2d.py |  1 +
 3 files changed, 13 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml
index 0af646345b1..8b71190eb2b 100644
--- a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml
+++ b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml
@@ -40,13 +40,6 @@ jobs:
               cmd: tests/scripts/single_card/nightly/run_ttnn.sh,
               timeout: 70
             },
-            {
-              name: "WH N300 pgm dispatch nightly",
-              arch: wormhole_b0,
-              runs-on: ["cloud-virtual-machine", "N300", "in-service"],
-              cmd: ./tests/tt_metal/tt_metal/perf_microbenchmark/dispatch/compare_pgm_dispatch_perf_ci.sh,
-              timeout: 10
-            },
             {
               name: "GS-only models",
               arch: grayskull,
@@ -151,18 +144,19 @@ jobs:
         test-config:
           - model: "stable_diffusion"
             cmd: pytest --timeout 900 -n auto tests/nightly/single_card/stable_diffusion
-          - model: "mamba 1"
-            cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1
-          - model: "mamba 2"
-            cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2
-          - model: "mamba 3"
-            cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3
-          - model: "mamba 4"
-            cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4
+              # Skipping due to issue #15932
+              # - model: "mamba 1"
+              # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1
+              # - model: "mamba 2"
+              # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2
+              # - model: "mamba 3"
+              # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3
+              # - model: "mamba 4"
+              # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4
           - model: "mamba 5"
             cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 5
-          - model: "mamba 6"
-            cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6
+              # - model: "mamba 6"
+              # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6
         card: [N150, N300]
     name: "[Unstable] Nightly ${{ matrix.card }} ${{ matrix.test-config.model }}"
     env:
diff --git a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py
index 72efdb4e178..24b9e22dea2 100644
--- a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py
+++ b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py
@@ -72,6 +72,7 @@ def unsqueeze_all_params_to_4d(params):
         (2, 4, 64, 64),
     ],
 )
+@pytest.mark.skip(reason="#15931: Failing, skip for now")
 def test_unet_2d_condition_model_512x512(device, batch_size, in_channels, input_height, input_width):
     device.enable_program_cache()
 
diff --git a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py
index 0445d58164b..cb9fd165f3f 100644
--- a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py
+++ b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py
@@ -29,6 +29,7 @@
 @pytest.mark.parametrize("res_hidden_states_tuple", [([2, 1280, 8, 8], [2, 1280, 8, 8], [2, 1280, 8, 8])])
 @pytest.mark.parametrize("hidden_states", [[2, 1280, 8, 8]])
 @pytest.mark.parametrize("temb", [[1, 1, 2, 1280]])
+@pytest.mark.skip(reason="#15931: Fails, need to investigate")
 def test_upblock_512x512(reset_seeds, device, res_hidden_states_tuple, hidden_states, temb):
     # TODO
     # setup pytorch model