#4003: Look further back in time per op file to see if we had a prior… (

#5037)

* #4003: Look further back in time per op file to see if we had a prior failure

* #4003: placeholder ttnn_sweeps index.rst

* #4003: Integrate build_rst_sweep_results.py into build docs flow to generate automated RSTs during build time

---------

Co-authored-by: Vincent Tang <[email protected]>
Co-authored-by: Raymond Kim <[email protected]>

.github/workflows/docs-latest-public.yaml

@@ -49,20 +49,13 @@ jobs:
       - uses: ./.github/actions/install-python-deps
       - name: Build Docs
         timeout-minutes: 15
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         run: |
           source ${{ github.workspace }}/python_env/bin/activate
           cd $TT_METAL_HOME
           export PYTHONPATH=$TT_METAL_HOME
           ./tests/scripts/run_build_docs.sh
-      - name: Build additional ttnn sweeps docs
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        run: |
-          export PYTHONPATH=$(pwd)
-          source ${{ github.workspace }}/python_env/bin/activate
-          cd docs/
-          make ttnn_sweeps/check_directory
-          make ttnn_sweeps
       - name: Prepare artifact - move output
         run: |
           mkdir gh_pages

docs/Makefile

@@ -15,13 +15,13 @@ PORT          	?= 8888
 DOCS_VERSION       	?= latest
 
 GITHUB_TOKEN    ?= INSERT_TOKEN_HERE
-TTNN_SWEEPS_DIR = $(HTMLDIR)/ttnn/ttnn_sweeps
+TTNN_SWEEPS_DIR = source/ttnn/ttnn_sweeps
 
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help default clean html publish build_doxygen sphinx_build_dir html/ttnn html/tt-metalium ttnn_sweeps ttnn_sweeps/check_directory
+.PHONY: help default clean html publish build_doxygen sphinx_build_dir html/ttnn html/tt-metalium ttnn_sweeps
 
 default: html
 
@@ -49,17 +49,9 @@ html: html/tt-metalium html/ttnn
 	mv -f $(TTNN_BUILDDIR) $(HTMLDIR)/ttnn
 	cp source/index.html $(HTMLDIR)/
 
-ttnn_sweeps/check_directory:
-	@if [ -d "$(TTNN_SWEEPS_DIR)" ]; then \
-		echo "Error: ttnn sweeps dir $(TTNN_SWEEPS_DIR) exists already."; \
-		exit 1; \
-	else \
-		mkdir -p $(TTNN_SWEEPS_DIR); \
-	fi
-
 ttnn_sweeps:
 	@echo "Note that GITHUB_TOKEN must be set before calling this"
-	@cd .. && python tests/ttnn/sweep_tests/build_html_sweep_results.py --dir docs/$(TTNN_SWEEPS_DIR) --token $(GITHUB_TOKEN)
+	@cd .. && python tests/ttnn/sweep_tests/build_rst_sweep_results.py --dir docs/$(TTNN_SWEEPS_DIR) --token $(GITHUB_TOKEN)
 
 server:
 	@echo "Navigate to: \033[4;33mlocalhost:$(PORT)/index.html\033[0m"

docs/requirements-docs.txt

@@ -8,3 +8,4 @@ nbsphinx==0.9.3
 sphinxcontrib-jquery==4.1
 ipython==8.12.3
 pandoc==2.3
+tabulate==0.9.0

docs/source/ttnn/index.rst

@@ -20,6 +20,7 @@ Welcome to TT-NN documentation!
    ttnn/profiling_ttnn_operations
    ttnn/dependencies/index.rst
    ttnn/demos
+   ttnn_sweeps/index.rst
 
 .. toctree::
    :caption: Models

docs/source/ttnn/ttnn/onboarding.rst

@@ -13,7 +13,7 @@ is that the long term benefits will help us maintain our objectives. Please foll
     * When creating the branch, please follow the pattern of 'TTNN-<Issue Number>-<brief description>'.  For example, if the issue is 4730, the branch name would be `TTNN-4730-concat-operation`
     * Use the `fallback` reference implementation for the operation and implement the functionality.
     * Add the documentation in the rst format for the operation under `ttnn documentation <https://github.com/tenstorrent/tt-metal/tree/main/docs/source/ttnn/ttnn>`_
-    * Add sweep tests to the branch using the fallback implementation under `ttnn sweep tests <https://github.com/tenstorrent/tt-metal/tree/main/tests/ttnn/sweep_tests/sweeps>`_
+    * Add :ref:`sweep tests<ttnn.sweep_tests>` to the branch using the fallback implementation under `ttnn sweep tests <https://github.com/tenstorrent/tt-metal/tree/main/tests/ttnn/sweep_tests/sweeps>`_
 3. Update the issue referencing the pull requests after verifying that all the sweep tests run as expected.  A TTNN CODEOWNERS will review the PR and verify that the API is acceptable and that the sweep tests reflect the intended functionality.
 4. If the pull request (PR) is accepted it will be merge into the main branch and a new branch should be created that adds the implementation.
     * The fallback implementation for the Operation should be left and will continue to be used for op-by-op PCC comparisons when debugging models (see `--ttnn-enable-debug-decorator`).

docs/source/ttnn/ttnn_sweeps/index.rst

@@ -0,0 +1,6 @@
+.. _ttnn.sweep_tests:
+
+Placeholder title
+=================
+
+You must generate ttnn_sweeps here.

docs/spellcheck.sh

@@ -12,7 +12,7 @@ then
     exit -1
 fi
 cd ${TT_METAL_HOME}/docs/
-for i in `find ./source/ -iname '*.rst'`; 
+for i in `find ./source/ -type d -name 'sweeps' -prune -o -iname '*.rst'`;
 do
     echo "Checking $i"
     if [ $INTERACTIVE_MODE -gt 0 ];
@@ -27,7 +27,7 @@ do
                 echo "-------------------------------"
                 echo "There are typos in the file: $i"
                 echo "Please update text in $i, or update personal dictionary as case maybe"
-                echo "-------------------------------"            
+                echo "-------------------------------"
                 exit -1
             else
                 echo "Skipping empty file $i"

tests/scripts/run_build_docs.sh

@@ -11,4 +11,5 @@ echo "Checking docs build..."
 cd $TT_METAL_HOME/docs
 python -m pip install -r requirements-docs.txt
 make clean
+make ttnn_sweeps
 make html