diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 1459d6b63e8d..fb3b9f3cf710 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -8,3 +8,4 @@ nbsphinx==0.9.3 sphinxcontrib-jquery==4.1 ipython==8.12.3 pandoc==2.3 +tabulate==0.9.0 diff --git a/tests/ttnn/sweep_tests/build_html_sweep_results.py b/tests/ttnn/sweep_tests/build_html_sweep_results.py index 6df4a4aa23b3..deb1383c8ac9 100644 --- a/tests/ttnn/sweep_tests/build_html_sweep_results.py +++ b/tests/ttnn/sweep_tests/build_html_sweep_results.py @@ -10,17 +10,19 @@ import zipfile import pandas as pd from loguru import logger +from dataclasses import dataclass +from tabulate import tabulate def get_list_of_runs(): - params = {"per_page": 3} + params = {"per_page": 15} url = "https://api.github.com/repos/tenstorrent-metal/tt-metal/actions/workflows/ttnn-run-sweeps.yaml/runs" headers = {"Accept": "application/vnd.github.v3+json"} response = requests.get(url, headers=headers, params=params) if response.status_code == 200: runs = response.json() else: - raise RuntimeError(f"Error fetching workflow runs: {response.status_code}") + raise RuntimeError(f"Error fetching workflow runs: {response.status_code}:{response.text}") return runs @@ -43,7 +45,7 @@ def download_artifacts(token, artifacts_url, output_path): else: raise RuntimeError("No artifacts found. Is there a run in progress?") else: - raise RuntimeError("Failed to fetch artifacts list.") + raise RuntimeError(f"Failed to fetch artifacts list. {response.status_code}:{response.text}") def read_csv_from_zip(zip_file, file_name): @@ -81,75 +83,127 @@ def build_new_crashes(recent_df, prior_df): return extract_only_recent_changes(failed_recent, failed_prior) -def diff_results(recent_zip, prior_zip, directory_for_html_pages, commit_hash): +@dataclass +class OperationFailure: + file_name: str + failure_file_name: str + commit_hash_with_failure: str + commit_hash_prior_to_failure: str + failures: int + + +def diff_results(temp_dir_path, most_recent_run_index, total_runs, directory_for_html_pages): directory_for_html_pages = pathlib.Path(directory_for_html_pages) - html_files = [] - html_failure_files = [] + rst_failure_files = [] + rst_files = [] failures_since_last_run = 0 - with zipfile.ZipFile(recent_zip, "r") as zip1, zipfile.ZipFile(prior_zip, "r") as zip2: + + recent_zip = temp_dir_path / str(most_recent_run_index) / "artifact.zip" + most_recent_commit_hash = "" + commit_hash_file = temp_dir_path / str(most_recent_run_index) / "commit_hash.txt" + with open(commit_hash_file, "r") as file: + most_recent_commit_hash = file.read() + + new_failures = {} + + with zipfile.ZipFile(recent_zip, "r") as zip1: + # We want to put the latest csv from the most recent run into html files zip1_files = set(zip1.namelist()) - zip2_files = set(zip2.namelist()) - common_files = zip1_files.intersection(zip2_files) - # pd.set_option("display.max_rows", None) - # pd.set_option("display.max_columns", None) - # pd.set_option("display.width", None) - # pd.set_option("display.max_colwidth", 10) - for file_name in common_files: + for file_name in zip1_files: test_name = pathlib.Path(file_name).stem if file_name.endswith(".csv"): recent_df = read_csv_from_zip(zip1, file_name) - html_table = recent_df.to_html() - html_page_name = directory_for_html_pages / f"{test_name}.html" - with open(html_page_name, "w") as f: - f.write(html_table) - html_files.append(f"{test_name}.html") - prior_df = read_csv_from_zip(zip2, file_name) - failures_df = build_new_failures(recent_df, prior_df) - crashes_df = build_new_crashes(recent_df, prior_df) - combined_test_resutls_df = pd.concat([failures_df, crashes_df]) - if combined_test_resutls_df.size > 0: - failures_since_last_run = failures_since_last_run + combined_test_resutls_df.size - html_table = combined_test_resutls_df.to_html() - html_page_name = directory_for_html_pages / f"{test_name}_failure.html" - with open(html_page_name, "w") as f: - f.write(html_table) - html_failure_files.append(f"{test_name}_failure.html") - - html_template = """ - - - - Sweep Test Results - - - -

Sweep Tests

-

We have had {failures_since_last_run} failures since the prior run.

-

Commit Hash: {commit_hash}

-
- {iframes} - - - """ - - iframe_tags = "".join( - [f'

{file.split(".")[0]}

' for file in html_failure_files] + rst_table = tabulate(recent_df, headers="keys", tablefmt="rst") + rst_page_name = directory_for_html_pages / f"{test_name}.rst" + with open(rst_page_name, "w") as f: + f.write(rst_table) + new_failures[test_name] = OperationFailure( + f"{test_name}.rst", f"{test_name}_failure.rst", most_recent_commit_hash, "", 0 + ) + rst_files.append(test_name) + + # Now we need to check and see which differences started showing up relative to the most recent run per operation file + for test_name in new_failures: + commit_hash = most_recent_commit_hash + prior_run_index = most_recent_run_index + 1 + while new_failures[test_name].failures == 0 and prior_run_index < total_runs - 1: + prior_zip = temp_dir_path / str(prior_run_index) / "artifact.zip" + with zipfile.ZipFile(prior_zip, "r") as zip2: + for file_name in zip2.namelist(): + if file_name.endswith(f"{test_name}.csv"): + test_name = pathlib.Path(file_name).stem + recent_df = read_csv_from_zip(zip1, file_name) + prior_df = read_csv_from_zip(zip2, file_name) + failures_df = build_new_failures(recent_df, prior_df) + crashes_df = build_new_crashes(recent_df, prior_df) + combined_test_results_df = pd.concat([failures_df, crashes_df]) + if len(combined_test_results_df) > 0: + failures_since_last_run = failures_since_last_run + len(combined_test_results_df) + new_failures[test_name].failures = combined_test_results_df.size + new_failures[test_name].failure_file_name = f"{test_name}_failure.rst" + new_failures[test_name].commit_hash_with_failure = commit_hash + + rst_table = tabulate(combined_test_results_df, headers="keys", tablefmt="rst") + rst_page_name = directory_for_html_pages / f"{test_name}_failure.rst" + with open(rst_page_name, "w") as f: + f.write(rst_table) + rst_failure_files.append(new_failures[test_name]) + + commit_hash_file = temp_dir_path / str(prior_run_index) / "commit_hash.txt" + with open(commit_hash_file, "r") as file: + commit_hash = file.read() + new_failures[test_name].commit_hash_prior_to_failure = commit_hash + + prior_run_index = prior_run_index + 1 + + rst_template = """ +Sweep Test Results +================== + +Recent Failures +--------------- + +We have had {failures_since_last_run} failures since the prior run. + +.. contents:: + :local: + :depth: 1 + +{links} + +All Sweep Tests +-------------------- + +These are the sweep tests for commit hash {most_recent_commit_hash} + +{sweep_test_results} + +""" + + link_tags = "\n".join( + [ + f"* `{op_failure.file_name.split('.')[0]} <{op_failure.failure_file_name}>`_ " + f"-> ( {op_failure.commit_hash_prior_to_failure} .. {op_failure.commit_hash_with_failure} ]" + for op_failure in rst_failure_files + ] ) - complete_html = html_template.format( - commit_hash=commit_hash, failures_since_last_run=failures_since_last_run, iframes=iframe_tags + link_tags = link_tags.lstrip() + + sweep_test_link_tags = "\n".join([f"* `{file} <{file}>`_" for file in rst_files]) + sweep_test_link_tags = sweep_test_link_tags.lstrip() + + complete_rst = rst_template.format( + most_recent_commit_hash=most_recent_commit_hash, + failures_since_last_run=failures_since_last_run, + links=link_tags, + sweep_test_results=sweep_test_link_tags, ) - html_page_name = directory_for_html_pages / f"index.html" - with open(html_page_name, "w") as file: - file.write(complete_html) - logger.info(f"Built {html_page_name}") + rst_page_name = directory_for_html_pages / "index.rst" + with open(rst_page_name, "w") as file: + file.write(complete_rst) + + logger.info(f"Built {rst_page_name}") def download_from_pipeline(token, directory_for_html_pages): @@ -161,27 +215,29 @@ def download_from_pipeline(token, directory_for_html_pages): runs = get_list_of_runs() if len(runs["workflow_runs"]) < 3: - # Note that if the run is in progress, there will not be any artifacts avaiable yet on the most recent run. + # Note that if the run is in progress, there will not be any artifacts available yet on the most recent run. raise RuntimeError("We need at least three runs to compare the changes in the sweep tests") + total_runs = len(runs["workflow_runs"]) if runs["workflow_runs"][0]["status"] == "completed": - most_recent_run = runs["workflow_runs"][0] - prior_run = runs["workflow_runs"][1] - else: - most_recent_run = runs["workflow_runs"][1] - prior_run = runs["workflow_runs"][2] - - most_recent_artifact_url = most_recent_run["artifacts_url"] - commit_hash = most_recent_run["head_sha"] - prior_artifact_url = prior_run["artifacts_url"] + most_recent_run_index = 0 + else: # a run is in progress so we just use the prior two for the first comparison + most_recent_run_index = 1 with tempfile.TemporaryDirectory() as temp_dir: temp_dir_path = pathlib.Path(temp_dir) - recent_zip = temp_dir_path / "recent.zip" - prior_zip = temp_dir_path / "prior.zip" - download_artifacts(token, most_recent_artifact_url, output_path=recent_zip) - download_artifacts(token, prior_artifact_url, output_path=prior_zip) - diff_results(recent_zip, prior_zip, directory_for_html_pages, commit_hash) + for i in range(most_recent_run_index, total_runs): + most_recent_run = runs["workflow_runs"][i] + most_recent_artifact_url = most_recent_run["artifacts_url"] + (temp_dir_path / str(i)).mkdir(parents=True, exist_ok=True) + artifact_zip = temp_dir_path / str(i) / "artifact.zip" + commit_hash = most_recent_run["head_sha"] + download_artifacts(token, most_recent_artifact_url, output_path=artifact_zip) + commit_hash_file = temp_dir_path / str(i) / "commit_hash.txt" + with open(commit_hash_file, "w") as file: + file.write(commit_hash) + + diff_results(temp_dir_path, most_recent_run_index, total_runs, directory_for_html_pages) def main(): @@ -190,7 +246,9 @@ def main(): parser.add_argument("--dir") token = parser.parse_args().token directory_for_html_pages = parser.parse_args().dir - download_from_pipeline(token, directory_for_html_pages) + + # download_from_pipeline(token, directory_for_html_pages) + diff_results(pathlib.Path("/tmp/saved_downloads"), 0, 10, directory_for_html_pages) if __name__ == "__main__":