diff --git a/.github/workflows/_produce-data.yaml b/.github/workflows/_produce-data.yaml index c66c5bb5702..1f13fd8274e 100644 --- a/.github/workflows/_produce-data.yaml +++ b/.github/workflows/_produce-data.yaml @@ -24,6 +24,7 @@ on: - "(Single-card) Demo tests" - "(Single-card) Tests for new models" - "Nightly fast dispatch tests" + - "(Single-card) Nightly model and ttnn tests" - "(Single-card) Tests for new models" - "(T3K) T3000 demo tests" - "(T3K) T3000 model perf tests" diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml index 0af646345b1..8b71190eb2b 100644 --- a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml +++ b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml @@ -40,13 +40,6 @@ jobs: cmd: tests/scripts/single_card/nightly/run_ttnn.sh, timeout: 70 }, - { - name: "WH N300 pgm dispatch nightly", - arch: wormhole_b0, - runs-on: ["cloud-virtual-machine", "N300", "in-service"], - cmd: ./tests/tt_metal/tt_metal/perf_microbenchmark/dispatch/compare_pgm_dispatch_perf_ci.sh, - timeout: 10 - }, { name: "GS-only models", arch: grayskull, @@ -151,18 +144,19 @@ jobs: test-config: - model: "stable_diffusion" cmd: pytest --timeout 900 -n auto tests/nightly/single_card/stable_diffusion - - model: "mamba 1" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1 - - model: "mamba 2" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2 - - model: "mamba 3" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3 - - model: "mamba 4" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4 + # Skipping due to issue #15932 + # - model: "mamba 1" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1 + # - model: "mamba 2" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2 + # - model: "mamba 3" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3 + # - model: "mamba 4" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4 - model: "mamba 5" cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 5 - - model: "mamba 6" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6 + # - model: "mamba 6" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6 card: [N150, N300] name: "[Unstable] Nightly ${{ matrix.card }} ${{ matrix.test-config.model }}" env: diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml index 409833993f4..65bacbceada 100644 --- a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml +++ b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml @@ -1,4 +1,4 @@ -name: Nightly fast dispatch tests +name: "(Single-card) Nightly model and ttnn tests" on: workflow_dispatch: diff --git a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py index 72efdb4e178..24b9e22dea2 100644 --- a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py +++ b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py @@ -72,6 +72,7 @@ def unsqueeze_all_params_to_4d(params): (2, 4, 64, 64), ], ) +@pytest.mark.skip(reason="#15931: Failing, skip for now") def test_unet_2d_condition_model_512x512(device, batch_size, in_channels, input_height, input_width): device.enable_program_cache() diff --git a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py index 0445d58164b..cb9fd165f3f 100644 --- a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py +++ b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py @@ -29,6 +29,7 @@ @pytest.mark.parametrize("res_hidden_states_tuple", [([2, 1280, 8, 8], [2, 1280, 8, 8], [2, 1280, 8, 8])]) @pytest.mark.parametrize("hidden_states", [[2, 1280, 8, 8]]) @pytest.mark.parametrize("temb", [[1, 1, 2, 1280]]) +@pytest.mark.skip(reason="#15931: Fails, need to investigate") def test_upblock_512x512(reset_seeds, device, res_hidden_states_tuple, hidden_states, temb): # TODO # setup pytorch model