From 5459004ebb7a6156692851b59a074077bed694c5 Mon Sep 17 00:00:00 2001 From: Eyon Date: Wed, 31 Jan 2024 05:10:50 +0000 Subject: [PATCH] #4003: Look further back in time per op file to see if we had a prior failure --- docs/Makefile | 4 +- docs/requirements-docs.txt | 1 + docs/source/index.rst | 1 + docs/source/ttnn/onboarding.rst | 2 +- docs/source/ttnn/sweeps/acos.rst | 8 + docs/source/ttnn/sweeps/add.rst | 8 + docs/source/ttnn/sweeps/asin.rst | 8 + docs/source/ttnn/sweeps/atan.rst | 8 + docs/source/ttnn/sweeps/average_pool2d.rst | 8 + docs/source/ttnn/sweeps/concat.rst | 8 + docs/source/ttnn/sweeps/cos.rst | 8 + docs/source/ttnn/sweeps/eq.rst | 8 + docs/source/ttnn/sweeps/eqz.rst | 8 + docs/source/ttnn/sweeps/exp.rst | 8 + docs/source/ttnn/sweeps/gelu.rst | 8 + docs/source/ttnn/sweeps/gez.rst | 8 + docs/source/ttnn/sweeps/gt.rst | 8 + docs/source/ttnn/sweeps/gte.rst | 8 + docs/source/ttnn/sweeps/gtz.rst | 8 + docs/source/ttnn/sweeps/index.rst | 66 ++++ docs/source/ttnn/sweeps/layer_norm.rst | 8 + docs/source/ttnn/sweeps/lez.rst | 8 + docs/source/ttnn/sweeps/linear.rst | 8 + docs/source/ttnn/sweeps/lt.rst | 8 + docs/source/ttnn/sweeps/lte.rst | 8 + docs/source/ttnn/sweeps/ltz.rst | 8 + docs/source/ttnn/sweeps/matmul.rst | 8 + docs/source/ttnn/sweeps/max_pool.rst | 8 + docs/source/ttnn/sweeps/mul.rst | 8 + docs/source/ttnn/sweeps/ne.rst | 8 + docs/source/ttnn/sweeps/nez.rst | 8 + docs/source/ttnn/sweeps/relu.rst | 8 + docs/source/ttnn/sweeps/repeat_interleave.rst | 8 + docs/source/ttnn/sweeps/rsqrt.rst | 8 + docs/source/ttnn/sweeps/sin.rst | 8 + docs/source/ttnn/sweeps/softmax.rst | 8 + docs/source/ttnn/sweeps/sub.rst | 8 + docs/source/ttnn/sweeps/tan.rst | 8 + docs/source/ttnn/sweeps/tanh.rst | 8 + .../sweeps/transformer_attention_softmax.rst | 8 + .../sweeps/transformer_concatenate_heads.rst | 8 + ..._split_query_key_value_and_split_heads.rst | 8 + docs/source/ttnn/sweeps/upsample.rst | 8 + docs/spellcheck.sh | 4 +- .../sweep_tests/build_rst_sweep_results.py | 300 ++++++++++++++++++ 45 files changed, 677 insertions(+), 5 deletions(-) create mode 100644 docs/source/ttnn/sweeps/acos.rst create mode 100644 docs/source/ttnn/sweeps/add.rst create mode 100644 docs/source/ttnn/sweeps/asin.rst create mode 100644 docs/source/ttnn/sweeps/atan.rst create mode 100644 docs/source/ttnn/sweeps/average_pool2d.rst create mode 100644 docs/source/ttnn/sweeps/concat.rst create mode 100644 docs/source/ttnn/sweeps/cos.rst create mode 100644 docs/source/ttnn/sweeps/eq.rst create mode 100644 docs/source/ttnn/sweeps/eqz.rst create mode 100644 docs/source/ttnn/sweeps/exp.rst create mode 100644 docs/source/ttnn/sweeps/gelu.rst create mode 100644 docs/source/ttnn/sweeps/gez.rst create mode 100644 docs/source/ttnn/sweeps/gt.rst create mode 100644 docs/source/ttnn/sweeps/gte.rst create mode 100644 docs/source/ttnn/sweeps/gtz.rst create mode 100644 docs/source/ttnn/sweeps/index.rst create mode 100644 docs/source/ttnn/sweeps/layer_norm.rst create mode 100644 docs/source/ttnn/sweeps/lez.rst create mode 100644 docs/source/ttnn/sweeps/linear.rst create mode 100644 docs/source/ttnn/sweeps/lt.rst create mode 100644 docs/source/ttnn/sweeps/lte.rst create mode 100644 docs/source/ttnn/sweeps/ltz.rst create mode 100644 docs/source/ttnn/sweeps/matmul.rst create mode 100644 docs/source/ttnn/sweeps/max_pool.rst create mode 100644 docs/source/ttnn/sweeps/mul.rst create mode 100644 docs/source/ttnn/sweeps/ne.rst create mode 100644 docs/source/ttnn/sweeps/nez.rst create mode 100644 docs/source/ttnn/sweeps/relu.rst create mode 100644 docs/source/ttnn/sweeps/repeat_interleave.rst create mode 100644 docs/source/ttnn/sweeps/rsqrt.rst create mode 100644 docs/source/ttnn/sweeps/sin.rst create mode 100644 docs/source/ttnn/sweeps/softmax.rst create mode 100644 docs/source/ttnn/sweeps/sub.rst create mode 100644 docs/source/ttnn/sweeps/tan.rst create mode 100644 docs/source/ttnn/sweeps/tanh.rst create mode 100644 docs/source/ttnn/sweeps/transformer_attention_softmax.rst create mode 100644 docs/source/ttnn/sweeps/transformer_concatenate_heads.rst create mode 100644 docs/source/ttnn/sweeps/transformer_split_query_key_value_and_split_heads.rst create mode 100644 docs/source/ttnn/sweeps/upsample.rst create mode 100644 tests/ttnn/sweep_tests/build_rst_sweep_results.py diff --git a/docs/Makefile b/docs/Makefile index 1cc8e395119..19eb1458f02 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,7 +16,7 @@ DOCS_VERSION ?= latest BRANCH = $(shell git rev-parse --abbrev-ref HEAD) GITHUB_TOKEN ?= INSERT_TOKEN_HERE -TTNN_SWEEPS_DIR = $(HTMLDIR)/ttnn_sweeps +TTNN_SWEEPS_DIR = $(SOURCEDIR)/ttnn/sweeps # Put it first so that "make" without argument is like "make help". help: @@ -46,7 +46,7 @@ ttnn_sweeps/check_directory: ttnn_sweeps: @echo "Note that GITHUB_TOKEN must be set before calling this" - @cd .. && python tests/ttnn/sweep_tests/build_html_sweep_results.py --dir docs/$(TTNN_SWEEPS_DIR) --token $(GITHUB_TOKEN) + @cd .. && python tests/ttnn/sweep_tests/build_rst_sweep_results.py --dir docs/$(TTNN_SWEEPS_DIR) --token $(GITHUB_TOKEN) server: @echo "Navigate to: \033[4;33mlocalhost:$(PORT)/index.html\033[0m" diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1459d6b63e8..fb3b9f3cf71 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -8,3 +8,4 @@ nbsphinx==0.9.3 sphinxcontrib-jquery==4.1 ipython==8.12.3 pandoc==2.3 +tabulate==0.9.0 diff --git a/docs/source/index.rst b/docs/source/index.rst index 8a152bd91dd..465b2d02e8a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -32,6 +32,7 @@ Welcome to TT-Metalium documentation! ttnn/adding_new_ttnn_operation ttnn/profiling_ttnn_operations ttnn/dependencies/index.rst + ttnn/sweeps/index.rst .. toctree:: :caption: Models diff --git a/docs/source/ttnn/onboarding.rst b/docs/source/ttnn/onboarding.rst index d95219026e2..a2cb652ce87 100644 --- a/docs/source/ttnn/onboarding.rst +++ b/docs/source/ttnn/onboarding.rst @@ -13,7 +13,7 @@ is that the long term benefits will help us maintain our objectives. Please foll * When creating the branch, please follow the pattern of 'TTNN--'. For example, if the issue is 4730, the branch name would be `TTNN-4730-concat-operation` * Use the `fallback` reference implementation for the operation and implement the functionality. * Add the documentation in the rst format for the operation under `ttnn documentation `_ - * Add sweep tests to the branch using the fallback implementation under `ttnn sweep tests `_ + * Add :ref:`sweep tests` to the branch using the fallback implementation under `ttnn sweep tests `_ 3. Update the issue referencing the pull requests after verifying that all the sweep tests run as expected. A TTNN CODEOWNERS will review the PR and verify that the API is acceptable and that the sweep tests reflect the intended functionality. 4. If the pull request (PR) is accepted it will be merge into the main branch and a new branch should be created that adds the implementation. * The fallback implementation for the Operation should be left and will continue to be used for op-by-op PCC comparisons when debugging models (see `--ttnn-enable-debug-decorator`). diff --git a/docs/source/ttnn/sweeps/acos.rst b/docs/source/ttnn/sweeps/acos.rst new file mode 100644 index 00000000000..b2020ee60c7 --- /dev/null +++ b/docs/source/ttnn/sweeps/acos.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_acos: + +acos +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/add.rst b/docs/source/ttnn/sweeps/add.rst new file mode 100644 index 00000000000..3ec31c97358 --- /dev/null +++ b/docs/source/ttnn/sweeps/add.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_add: + +add +==================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width broadcast input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/asin.rst b/docs/source/ttnn/sweeps/asin.rst new file mode 100644 index 00000000000..710191dce57 --- /dev/null +++ b/docs/source/ttnn/sweeps/asin.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_asin: + +asin +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/atan.rst b/docs/source/ttnn/sweeps/atan.rst new file mode 100644 index 00000000000..c8e6b0595b3 --- /dev/null +++ b/docs/source/ttnn/sweeps/atan.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_atan: + +atan +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/average_pool2d.rst b/docs/source/ttnn/sweeps/average_pool2d.rst new file mode 100644 index 00000000000..c4e6f81492e --- /dev/null +++ b/docs/source/ttnn/sweeps/average_pool2d.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_average_pool2d: + +average_pool2d +==================================================================== +==== ======== =========== =============== ================= + .. status exception input_shape dtype +==== ======== =========== =============== ================= +==== ======== =========== =============== ================= diff --git a/docs/source/ttnn/sweeps/concat.rst b/docs/source/ttnn/sweeps/concat.rst new file mode 100644 index 00000000000..a4febd85b1d --- /dev/null +++ b/docs/source/ttnn/sweeps/concat.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_concat: + +concat +==================================================================== +==== =================== ============================================================================================= =================== ================= ============================= ============================= ================ ================= ============================================================================================================================== + .. status exception number_of_tensors rank_of_tensors max_random_size_of_each_dim dimension_to_concatenate_on layout dtype memory_config +==== =================== ============================================================================================= =================== ================= ============================= ============================= ================ ================= ============================================================================================================================== +==== =================== ============================================================================================= =================== ================= ============================= ============================= ================ ================= ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/cos.rst b/docs/source/ttnn/sweeps/cos.rst new file mode 100644 index 00000000000..04ae274d651 --- /dev/null +++ b/docs/source/ttnn/sweeps/cos.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_cos: + +cos +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/eq.rst b/docs/source/ttnn/sweeps/eq.rst new file mode 100644 index 00000000000..3754dbf5d2f --- /dev/null +++ b/docs/source/ttnn/sweeps/eq.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_eq: + +eq +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/eqz.rst b/docs/source/ttnn/sweeps/eqz.rst new file mode 100644 index 00000000000..cac04dfb742 --- /dev/null +++ b/docs/source/ttnn/sweeps/eqz.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_eqz: + +eqz +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/exp.rst b/docs/source/ttnn/sweeps/exp.rst new file mode 100644 index 00000000000..de33a9d25d5 --- /dev/null +++ b/docs/source/ttnn/sweeps/exp.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_exp: + +exp +==================================================================== +==== ======== ========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== ========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== ========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/gelu.rst b/docs/source/ttnn/sweeps/gelu.rst new file mode 100644 index 00000000000..7c77edda6de --- /dev/null +++ b/docs/source/ttnn/sweeps/gelu.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_gelu: + +gelu +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/gez.rst b/docs/source/ttnn/sweeps/gez.rst new file mode 100644 index 00000000000..f730c2f7193 --- /dev/null +++ b/docs/source/ttnn/sweeps/gez.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_gez: + +gez +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/gt.rst b/docs/source/ttnn/sweeps/gt.rst new file mode 100644 index 00000000000..c84422bfc59 --- /dev/null +++ b/docs/source/ttnn/sweeps/gt.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_gt: + +gt +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/gte.rst b/docs/source/ttnn/sweeps/gte.rst new file mode 100644 index 00000000000..3101ee02a8e --- /dev/null +++ b/docs/source/ttnn/sweeps/gte.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_gte: + +gte +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/gtz.rst b/docs/source/ttnn/sweeps/gtz.rst new file mode 100644 index 00000000000..559cd73bddc --- /dev/null +++ b/docs/source/ttnn/sweeps/gtz.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_gtz: + +gtz +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/index.rst b/docs/source/ttnn/sweeps/index.rst new file mode 100644 index 00000000000..c069418dcd3 --- /dev/null +++ b/docs/source/ttnn/sweeps/index.rst @@ -0,0 +1,66 @@ + +.. _ttnn.sweep_tests: + +Sweep Test Results +================== + +Recent New Failures +------------------- + +We have had 0 new failures since the prior run. + +.. toctree:: + :maxdepth: 2 + :hidden: + + + + + + +All Sweep Tests +-------------------- + +These are the sweep tests for commit hash 5d391c6035dbbe59db024c9860bdf9fe9684af2f + +.. toctree:: + :maxdepth: 2 + + acos + add + asin + atan + average_pool2d + concat + cos + eq + eqz + exp + gelu + gez + gt + gte + gtz + layer_norm + lez + linear + lt + lte + ltz + matmul + max_pool + mul + ne + nez + relu + repeat_interleave + rsqrt + sin + softmax + sub + tan + tanh + transformer_attention_softmax + transformer_concatenate_heads + transformer_split_query_key_value_and_split_heads + upsample diff --git a/docs/source/ttnn/sweeps/layer_norm.rst b/docs/source/ttnn/sweeps/layer_norm.rst new file mode 100644 index 00000000000..945fe2c185e --- /dev/null +++ b/docs/source/ttnn/sweeps/layer_norm.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_layer_norm: + +layer_norm +==================================================================== +==== ======== ================================================================================================= ============= ======== ======= ===================== ========= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width use_weight_and_bias epsilon input_dtype input_memory_config output_memory_config +==== ======== ================================================================================================= ============= ======== ======= ===================== ========= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== ================================================================================================= ============= ======== ======= ===================== ========= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/lez.rst b/docs/source/ttnn/sweeps/lez.rst new file mode 100644 index 00000000000..9d4396524a2 --- /dev/null +++ b/docs/source/ttnn/sweeps/lez.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_lez: + +lez +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/linear.rst b/docs/source/ttnn/sweeps/linear.rst new file mode 100644 index 00000000000..b420febbd4e --- /dev/null +++ b/docs/source/ttnn/sweeps/linear.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_linear: + +linear +==================================================================== +==== ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ============= ======== ======== ======== ========== ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes m_size k_size n_size use_bias input_a_dtype input_b_dtype output_dtype input_b_memory_config input_a_memory_config output_memory_config core_grid +==== ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ============= ======== ======== ======== ========== ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== +==== ======== ======================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ============= ======== ======== ======== ========== ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/lt.rst b/docs/source/ttnn/sweeps/lt.rst new file mode 100644 index 00000000000..ee078c7d5ec --- /dev/null +++ b/docs/source/ttnn/sweeps/lt.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_lt: + +lt +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/lte.rst b/docs/source/ttnn/sweeps/lte.rst new file mode 100644 index 00000000000..c2fb073255c --- /dev/null +++ b/docs/source/ttnn/sweeps/lte.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_lte: + +lte +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/ltz.rst b/docs/source/ttnn/sweeps/ltz.rst new file mode 100644 index 00000000000..0d57a2eb97f --- /dev/null +++ b/docs/source/ttnn/sweeps/ltz.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_ltz: + +ltz +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/matmul.rst b/docs/source/ttnn/sweeps/matmul.rst new file mode 100644 index 00000000000..dd06c153e86 --- /dev/null +++ b/docs/source/ttnn/sweeps/matmul.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_matmul: + +matmul +==================================================================== +==== ======== ============================================================================================= ============= ======== ======== ======== ======================= ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes m_size k_size n_size batch_matrix_multiply input_a_dtype input_b_dtype output_dtype input_b_memory_config input_a_memory_config output_memory_config core_grid +==== ======== ============================================================================================= ============= ======== ======== ======== ======================= ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== +==== ======== ============================================================================================= ============= ======== ======== ======== ======================= ================= ================= ================= ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/max_pool.rst b/docs/source/ttnn/sweeps/max_pool.rst new file mode 100644 index 00000000000..9ebd8ba9e80 --- /dev/null +++ b/docs/source/ttnn/sweeps/max_pool.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_max_pool: + +max_pool +==================================================================== +==== ======== ================================ ================== ============= ========= ======== ========== ========= ================== + .. status exception act_shape kernel_size padding stride dilation nblocks dtype +==== ======== ================================ ================== ============= ========= ======== ========== ========= ================== +==== ======== ================================ ================== ============= ========= ======== ========== ========= ================== diff --git a/docs/source/ttnn/sweeps/mul.rst b/docs/source/ttnn/sweeps/mul.rst new file mode 100644 index 00000000000..d62ef8581c2 --- /dev/null +++ b/docs/source/ttnn/sweeps/mul.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_mul: + +mul +==================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width broadcast input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/ne.rst b/docs/source/ttnn/sweeps/ne.rst new file mode 100644 index 00000000000..791a3491638 --- /dev/null +++ b/docs/source/ttnn/sweeps/ne.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_ne: + +ne +==================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============= ======== ======= ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/nez.rst b/docs/source/ttnn/sweeps/nez.rst new file mode 100644 index 00000000000..aefe627f904 --- /dev/null +++ b/docs/source/ttnn/sweeps/nez.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_nez: + +nez +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/relu.rst b/docs/source/ttnn/sweeps/relu.rst new file mode 100644 index 00000000000..3ed91abf400 --- /dev/null +++ b/docs/source/ttnn/sweeps/relu.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_relu: + +relu +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/repeat_interleave.rst b/docs/source/ttnn/sweeps/repeat_interleave.rst new file mode 100644 index 00000000000..cbe59220852 --- /dev/null +++ b/docs/source/ttnn/sweeps/repeat_interleave.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_repeat_interleave: + +repeat_interleave +==================================================================== +==== =================== ======================================================================================================== ====================== ================ ============================= ======================== ================ ================= ============================================================================================================================== + .. status exception make_repeat_a_tensor rank_of_tensor max_random_size_of_each_dim dimension_to_repeat_on layout dtype memory_config +==== =================== ======================================================================================================== ====================== ================ ============================= ======================== ================ ================= ============================================================================================================================== +==== =================== ======================================================================================================== ====================== ================ ============================= ======================== ================ ================= ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/rsqrt.rst b/docs/source/ttnn/sweeps/rsqrt.rst new file mode 100644 index 00000000000..b7e5887f7fc --- /dev/null +++ b/docs/source/ttnn/sweeps/rsqrt.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_rsqrt: + +rsqrt +==================================================================== +==== ======== ============================================================================================ ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== ============================================================================================ ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== ============================================================================================ ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/sin.rst b/docs/source/ttnn/sweeps/sin.rst new file mode 100644 index 00000000000..7af66b31994 --- /dev/null +++ b/docs/source/ttnn/sweeps/sin.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_sin: + +sin +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/softmax.rst b/docs/source/ttnn/sweeps/softmax.rst new file mode 100644 index 00000000000..8ab7bd88d5d --- /dev/null +++ b/docs/source/ttnn/sweeps/softmax.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_softmax: + +softmax +==================================================================== +==== ======== ============================================================================================== ============= ======== ======= ===== ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width dim input_dtype input_memory_config output_memory_config +==== ======== ============================================================================================== ============= ======== ======= ===== ================= ============================================================================================================================== ============================================================================================================================== +==== ======== ============================================================================================== ============= ======== ======= ===== ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/sub.rst b/docs/source/ttnn/sweeps/sub.rst new file mode 100644 index 00000000000..062e1db7bfd --- /dev/null +++ b/docs/source/ttnn/sweeps/sub.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_sub: + +sub +==================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width broadcast input_a_dtype input_b_dtype input_a_layout input_b_layout input_b_memory_config input_a_memory_config output_memory_config +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== +==== ======== ========================================================================================== ============= ======== ======= =========== ================= ================= ================ ================ ============================================================================================================================== ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/tan.rst b/docs/source/ttnn/sweeps/tan.rst new file mode 100644 index 00000000000..b6f8dc1031e --- /dev/null +++ b/docs/source/ttnn/sweeps/tan.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_tan: + +tan +==================================================================== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config layout +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== +==== ======== =========== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== =========== diff --git a/docs/source/ttnn/sweeps/tanh.rst b/docs/source/ttnn/sweeps/tanh.rst new file mode 100644 index 00000000000..6684b65602e --- /dev/null +++ b/docs/source/ttnn/sweeps/tanh.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_tanh: + +tanh +==================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_sizes height width input_dtype input_memory_config output_memory_config +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========================================================================================== ============= ======== ======= ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/transformer_attention_softmax.rst b/docs/source/ttnn/sweeps/transformer_attention_softmax.rst new file mode 100644 index 00000000000..c35705eb425 --- /dev/null +++ b/docs/source/ttnn/sweeps/transformer_attention_softmax.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_transformer_attention_softmax: + +transformer_attention_softmax +==================================================================== +==== ======== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ============ =========== =============== ====================== ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_size num_heads sequence_size target_sequence_size input_dtype input_memory_config output_memory_config +==== ======== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ============ =========== =============== ====================== ================= ============================================================================================================================== ============================================================================================================================== +==== ======== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= ============ =========== =============== ====================== ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/transformer_concatenate_heads.rst b/docs/source/ttnn/sweeps/transformer_concatenate_heads.rst new file mode 100644 index 00000000000..47922defb74 --- /dev/null +++ b/docs/source/ttnn/sweeps/transformer_concatenate_heads.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_transformer_concatenate_heads: + +transformer_concatenate_heads +==================================================================== +==== ======== =========== ============ =========== =============== =========== ================= ============================================================================================================================== ============================================================================================================================== + .. status exception batch_size num_heads sequence_size head_size input_dtype input_memory_config output_memory_config +==== ======== =========== ============ =========== =============== =========== ================= ============================================================================================================================== ============================================================================================================================== +==== ======== =========== ============ =========== =============== =========== ================= ============================================================================================================================== ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/transformer_split_query_key_value_and_split_heads.rst b/docs/source/ttnn/sweeps/transformer_split_query_key_value_and_split_heads.rst new file mode 100644 index 00000000000..4cbaab295ae --- /dev/null +++ b/docs/source/ttnn/sweeps/transformer_split_query_key_value_and_split_heads.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_transformer_split_query_key_value_and_split_heads: + +transformer_split_query_key_value_and_split_heads +==================================================================== +==== ======== =========== ============ =============== =========== =========== ================= ============================================================================================================================== + .. status exception batch_size sequence_size num_heads head_size input_dtype input_memory_config +==== ======== =========== ============ =============== =========== =========== ================= ============================================================================================================================== +==== ======== =========== ============ =============== =========== =========== ================= ============================================================================================================================== diff --git a/docs/source/ttnn/sweeps/upsample.rst b/docs/source/ttnn/sweeps/upsample.rst new file mode 100644 index 00000000000..3a8912b596c --- /dev/null +++ b/docs/source/ttnn/sweeps/upsample.rst @@ -0,0 +1,8 @@ +.. _ttnn.sweep_test_upsample: + +upsample +==================================================================== +==== ======== =========== ============ === === === ========= ========= + .. status exception batch_size c h w scale_h scale_w +==== ======== =========== ============ === === === ========= ========= +==== ======== =========== ============ === === === ========= ========= diff --git a/docs/spellcheck.sh b/docs/spellcheck.sh index 350ec8d41d1..98fd29f3a85 100755 --- a/docs/spellcheck.sh +++ b/docs/spellcheck.sh @@ -12,7 +12,7 @@ then exit -1 fi cd ${TT_METAL_HOME}/docs/ -for i in `find ./source/ -iname '*.rst'`; +for i in `find ./source/ -type d -name 'sweeps' -prune -o -iname '*.rst'`; do echo "Checking $i" if [ $INTERACTIVE_MODE -gt 0 ]; @@ -27,7 +27,7 @@ do echo "-------------------------------" echo "There are typos in the file: $i" echo "Please update text in $i, or update personal dictionary as case maybe" - echo "-------------------------------" + echo "-------------------------------" exit -1 else echo "Skipping empty file $i" diff --git a/tests/ttnn/sweep_tests/build_rst_sweep_results.py b/tests/ttnn/sweep_tests/build_rst_sweep_results.py new file mode 100644 index 00000000000..8a3f666db7a --- /dev/null +++ b/tests/ttnn/sweep_tests/build_rst_sweep_results.py @@ -0,0 +1,300 @@ +# SPDX-FileCopyrightText: © 2023 Tenstorrent Inc. + +# SPDX-License-Identifier: Apache-2.0 + + +import argparse +import requests +import tempfile +import pathlib +import zipfile +import pandas as pd +from loguru import logger +from dataclasses import dataclass +from tabulate import tabulate +import os +import shutil + + +def get_list_of_runs(): + params = {"per_page": 15} + url = "https://api.github.com/repos/tenstorrent-metal/tt-metal/actions/workflows/ttnn-run-sweeps.yaml/runs" + headers = {"Accept": "application/vnd.github.v3+json"} + response = requests.get(url, headers=headers, params=params) + if response.status_code == 200: + runs = response.json() + else: + raise RuntimeError(f"Error fetching workflow runs: {response.status_code}:{response.text}") + + return runs + + +def download_artifacts(token, artifacts_url, temp_dir_path, directory_index): + response = requests.get(artifacts_url) + headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"} + if response.status_code == 200: + artifacts_data = response.json() + if artifacts_data["artifacts"]: + artifact = artifacts_data["artifacts"][0] + artifact_download_url = artifact["archive_download_url"] + artifact_response = requests.get(artifact_download_url, headers=headers) + if artifact_response.status_code == 200: + (temp_dir_path / str(directory_index)).mkdir(parents=True, exist_ok=True) + artifact_zip = temp_dir_path / str(directory_index) / "artifact.zip" + with open(artifact_zip, "wb") as file: + file.write(artifact_response.content) + logger.info(f"{artifacts_url} downloaded successfully.") + return True + else: + raise RuntimeError("Failed to download the artifact.") + else: + print(f"No artifacts found. Is there a run in progress for {artifacts_url} ?") + else: + raise RuntimeError(f"Failed to fetch artifacts list. {response.status_code}:{response.text}") + return False + + +def read_csv_from_zip(zip_file, file_name): + with zip_file.open(file_name) as f: + df = pd.read_csv(f) + if not df.empty and len(df.columns) > 1: + # remove first unamed column which is just the index. + # This will be displayed by tabulate. + df = df.iloc[:, 1:] + return df + + +def trim_column(texte, longueur): + if len(texte) > longueur: + return texte[-longueur + 3 :] + return texte + + +def get_subset_for_status(recent_df, prior_df, status): + failed_recent = recent_df[recent_df["status"] == status] + matching_prior_status = prior_df["status"] == status + failed_prior = prior_df[matching_prior_status] + return failed_recent, failed_prior + + +def extract_only_recent_changes(failed_recent, failed_prior): + run_id_column_name = failed_recent.columns[0] + newly_failed = failed_recent[~failed_recent[run_id_column_name].isin(failed_prior[run_id_column_name])] + for column in newly_failed.columns: + newly_failed[column] = newly_failed[column].apply(lambda x: trim_column(str(x), 10)) + return newly_failed + + +def build_new_failures(recent_df, prior_df): + failed_recent, failed_prior = get_subset_for_status(recent_df, prior_df, "failed") + return extract_only_recent_changes(failed_recent, failed_prior) + + +def build_new_crashes(recent_df, prior_df): + failed_recent, failed_prior = get_subset_for_status(recent_df, prior_df, "crashed") + return extract_only_recent_changes(failed_recent, failed_prior) + + +def delete_directory_contents(dir_path): + for item in os.listdir(dir_path): + item_path = os.path.join(dir_path, item) + if os.path.isfile(item_path) or os.path.islink(item_path): + os.unlink(item_path) + elif os.path.isdir(item_path): + shutil.rmtree(item_path) + + +@dataclass +class OperationFailure: + file_name: str + failure_file_name: str + commit_hash_with_failure: str + commit_hash_prior_to_failure: str + failures: int + + +def diff_results(temp_dir_path, most_recent_run_index, total_runs, directory_for_rst_pages): + directory_for_rst_pages = pathlib.Path(directory_for_rst_pages) + rst_failure_files = [] + rst_files = [] + failures_since_last_run = 0 + + recent_zip = temp_dir_path / str(most_recent_run_index) / "artifact.zip" + most_recent_commit_hash = "" + commit_hash_file = temp_dir_path / str(most_recent_run_index) / "commit_hash.txt" + with open(commit_hash_file, "r") as file: + most_recent_commit_hash = file.read() + + new_failures = {} + + with zipfile.ZipFile(recent_zip, "r") as zip1: + # We want to put the latest csv from the most recent run into html files + zip1_files = set(zip1.namelist()) + for file_name in zip1_files: + test_name = pathlib.Path(file_name).stem + if file_name.endswith(".csv"): + recent_df = read_csv_from_zip(zip1, file_name) + for col in recent_df.columns: + recent_df[col] = recent_df[col].apply(lambda x: str(x).replace("\t", " ").replace("\n", " ")) + rst_table = tabulate(recent_df, headers="keys", tablefmt="rst") + rst_page_name = directory_for_rst_pages / f"{test_name}.rst" + with open(rst_page_name, "w") as f: + f.writelines(f".. _ttnn.sweep_test_{test_name}:\n") + f.writelines("\n") + f.writelines(f"{test_name}\n") + f.writelines("====================================================================\n") + f.write(rst_table) + new_failures[test_name] = OperationFailure( + f"{test_name}.rst", f"{test_name}_failure.rst", most_recent_commit_hash, "", 0 + ) + rst_files.append(test_name) + + # Now we need to check and see which differences started showing up relative to the most recent run per operation file + for test_name in new_failures: + commit_hash = most_recent_commit_hash + prior_run_index = most_recent_run_index + 1 + while new_failures[test_name].failures == 0 and prior_run_index < total_runs - 1: + prior_zip = temp_dir_path / str(prior_run_index) / "artifact.zip" + with zipfile.ZipFile(prior_zip, "r") as zip2: + for file_name in zip2.namelist(): + if file_name.endswith(f"{test_name}.csv"): + test_name = pathlib.Path(file_name).stem + recent_df = read_csv_from_zip(zip1, file_name) + prior_df = read_csv_from_zip(zip2, file_name) + failures_df = build_new_failures(recent_df, prior_df) + crashes_df = build_new_crashes(recent_df, prior_df) + combined_test_results_df = pd.concat([failures_df, crashes_df]) + if len(combined_test_results_df) > 0: + failures_since_last_run = failures_since_last_run + len(combined_test_results_df) + new_failures[test_name].failures = combined_test_results_df.size + new_failures[test_name].failure_file_name = f"{test_name}_failure.rst" + new_failures[test_name].commit_hash_with_failure = commit_hash + + rst_table = tabulate(combined_test_results_df, headers="keys", tablefmt="rst") + rst_page_name = directory_for_rst_pages / f"{test_name}_failure.rst" + with open(rst_page_name, "w") as f: + f.writelines(f".. _ttnn.sweep_test_failure_{test_name}:\n") + f.writelines("\n") + f.writelines(f"{test_name}\n") + f.writelines( + "====================================================================\n" + ) + f.write(rst_table) + rst_failure_files.append(new_failures[test_name]) + + commit_hash_file = temp_dir_path / str(prior_run_index) / "commit_hash.txt" + with open(commit_hash_file, "r") as file: + commit_hash = file.read() + new_failures[test_name].commit_hash_prior_to_failure = commit_hash + + prior_run_index = prior_run_index + 1 + + rst_template = """ +.. _ttnn.sweep_tests: + +Sweep Test Results +================== + +Recent New Failures +------------------- + +We have had {failures_since_last_run} new failures since the prior run. + +.. toctree:: + :maxdepth: 2 + :hidden: + + {toctree_failure_filenames} + +{sweep_test_failure_entries} + + +All Sweep Tests +-------------------- + +These are the sweep tests for commit hash {most_recent_commit_hash} + +.. toctree:: + :maxdepth: 2 + + {toctree_entries} +""" + + sweep_test_failure_entries = "\n".join( + [ + f"* :ref:`{op_failure.file_name.split('.')[0]} ` " + f"-> ( {op_failure.commit_hash_prior_to_failure} .. {op_failure.commit_hash_with_failure} ]" + for op_failure in rst_failure_files + ] + ) + sweep_test_failure_entries = sweep_test_failure_entries.lstrip() + + toctree_failure_filenames = "\n ".join( + [op_failure.failure_file_name.replace(".rst", "") for op_failure in rst_failure_files] + ) + + toctree_entries = "\n ".join(sorted(rst_files)) + + complete_rst = rst_template.format( + most_recent_commit_hash=most_recent_commit_hash, + failures_since_last_run=failures_since_last_run, + toctree_failure_filenames=toctree_failure_filenames, + sweep_test_failure_entries=sweep_test_failure_entries, + toctree_entries=toctree_entries, + ) + + rst_page_name = directory_for_rst_pages / "index.rst" + with open(rst_page_name, "w") as file: + file.write(complete_rst) + + logger.info(f"Built {rst_page_name}") + + +def download_from_pipeline(token, directory_for_rst_pages): + """ + Download the results of the sweeps from the GitHub pipeline. + + :param token: Provide your GitHub token. + """ + + runs = get_list_of_runs() + if len(runs["workflow_runs"]) < 3: + # Note that if the run is in progress, there will not be any artifacts available yet on the most recent run. + raise RuntimeError("We need at least three runs to compare the changes in the sweep tests") + + total_runs = len(runs["workflow_runs"]) + if runs["workflow_runs"][0]["status"] == "completed": + most_recent_run_index = 0 + else: # a run is in progress so we just use the prior two for the first comparison + most_recent_run_index = 1 + + directory_index = 0 + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir_path = pathlib.Path(temp_dir) + for i in range(most_recent_run_index, total_runs): + most_recent_run = runs["workflow_runs"][i] + most_recent_artifact_url = most_recent_run["artifacts_url"] + commit_hash = most_recent_run["head_sha"] + if download_artifacts(token, most_recent_artifact_url, temp_dir_path, directory_index): + commit_hash_file = temp_dir_path / str(directory_index) / "commit_hash.txt" + with open(commit_hash_file, "w") as file: + file.write(commit_hash) + directory_index = directory_index + 1 + + total_runs = directory_index + delete_directory_contents(directory_for_rst_pages) + diff_results(temp_dir_path, 0, total_runs, directory_for_rst_pages) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--token") + parser.add_argument("--dir") + token = parser.parse_args().token + directory_for_rst_pages = parser.parse_args().dir + + download_from_pipeline(token, directory_for_rst_pages) + + +if __name__ == "__main__": + main()