From 49d39e9b2ead9e2b4e560ebe12f25fbf9571b39a Mon Sep 17 00:00:00 2001 From: Raymond Kim <109366641+tt-rkim@users.noreply.github.com> Date: Wed, 11 Dec 2024 18:58:57 -0500 Subject: [PATCH] #0: [skip ci] Disable and file issues for flaky nightly fast dispatch tests and rename pipeline (#15947) ### Ticket These tests have been dogging us and we need to be green. ### Problem description Provide context for the problem. ### What's changed Describe the approach used to solve the problem. Summarize the changes made and its impact. ### Checklist - [ ] Post commit CI passes - [ ] Blackhole Post commit (if applicable) - [ ] Model regression CI testing passes (if applicable) - [ ] Device performance regression CI testing passes (if applicable) - [ ] **(For models and ops writers)** Full [new models](https://github.com/tenstorrent/tt-metal/actions/workflows/full-new-models-suite.yaml) tests passes - [ ] New/Existing tests provide coverage for changes --- .github/workflows/_produce-data.yaml | 1 + ...atch-full-regressions-and-models-impl.yaml | 28 ++++++++----------- ...-dispatch-full-regressions-and-models.yaml | 2 +- .../tests/test_unet_2d_condition_model.py | 1 + .../stable_diffusion/tests/test_upblock_2d.py | 1 + 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/_produce-data.yaml b/.github/workflows/_produce-data.yaml index c66c5bb5702..1f13fd8274e 100644 --- a/.github/workflows/_produce-data.yaml +++ b/.github/workflows/_produce-data.yaml @@ -24,6 +24,7 @@ on: - "(Single-card) Demo tests" - "(Single-card) Tests for new models" - "Nightly fast dispatch tests" + - "(Single-card) Nightly model and ttnn tests" - "(Single-card) Tests for new models" - "(T3K) T3000 demo tests" - "(T3K) T3000 model perf tests" diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml index 0af646345b1..8b71190eb2b 100644 --- a/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml +++ b/.github/workflows/fast-dispatch-full-regressions-and-models-impl.yaml @@ -40,13 +40,6 @@ jobs: cmd: tests/scripts/single_card/nightly/run_ttnn.sh, timeout: 70 }, - { - name: "WH N300 pgm dispatch nightly", - arch: wormhole_b0, - runs-on: ["cloud-virtual-machine", "N300", "in-service"], - cmd: ./tests/tt_metal/tt_metal/perf_microbenchmark/dispatch/compare_pgm_dispatch_perf_ci.sh, - timeout: 10 - }, { name: "GS-only models", arch: grayskull, @@ -151,18 +144,19 @@ jobs: test-config: - model: "stable_diffusion" cmd: pytest --timeout 900 -n auto tests/nightly/single_card/stable_diffusion - - model: "mamba 1" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1 - - model: "mamba 2" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2 - - model: "mamba 3" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3 - - model: "mamba 4" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4 + # Skipping due to issue #15932 + # - model: "mamba 1" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 1 + # - model: "mamba 2" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 2 + # - model: "mamba 3" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 3 + # - model: "mamba 4" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 4 - model: "mamba 5" cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 5 - - model: "mamba 6" - cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6 + # - model: "mamba 6" + # cmd: pytest --timeout 900 -n auto tests/nightly/single_card/mamba --splits 6 --group 6 card: [N150, N300] name: "[Unstable] Nightly ${{ matrix.card }} ${{ matrix.test-config.model }}" env: diff --git a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml index 409833993f4..65bacbceada 100644 --- a/.github/workflows/fast-dispatch-full-regressions-and-models.yaml +++ b/.github/workflows/fast-dispatch-full-regressions-and-models.yaml @@ -1,4 +1,4 @@ -name: Nightly fast dispatch tests +name: "(Single-card) Nightly model and ttnn tests" on: workflow_dispatch: diff --git a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py index 72efdb4e178..24b9e22dea2 100644 --- a/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py +++ b/models/demos/wormhole/stable_diffusion/tests/test_unet_2d_condition_model.py @@ -72,6 +72,7 @@ def unsqueeze_all_params_to_4d(params): (2, 4, 64, 64), ], ) +@pytest.mark.skip(reason="#15931: Failing, skip for now") def test_unet_2d_condition_model_512x512(device, batch_size, in_channels, input_height, input_width): device.enable_program_cache() diff --git a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py index 0445d58164b..cb9fd165f3f 100644 --- a/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py +++ b/models/demos/wormhole/stable_diffusion/tests/test_upblock_2d.py @@ -29,6 +29,7 @@ @pytest.mark.parametrize("res_hidden_states_tuple", [([2, 1280, 8, 8], [2, 1280, 8, 8], [2, 1280, 8, 8])]) @pytest.mark.parametrize("hidden_states", [[2, 1280, 8, 8]]) @pytest.mark.parametrize("temb", [[1, 1, 2, 1280]]) +@pytest.mark.skip(reason="#15931: Fails, need to investigate") def test_upblock_512x512(reset_seeds, device, res_hidden_states_tuple, hidden_states, temb): # TODO # setup pytorch model