diff --git a/tests/scripts/run_profiler_regressions.sh b/tests/scripts/run_profiler_regressions.sh index 87006cf0dbc..744423a1ea1 100755 --- a/tests/scripts/run_profiler_regressions.sh +++ b/tests/scripts/run_profiler_regressions.sh @@ -124,13 +124,6 @@ run_profiling_no_reset_test(){ remove_default_log_locations } -run_post_proc_test(){ - source python_env/bin/activate - export PYTHONPATH=$TT_METAL_HOME - - pytest $PROFILER_TEST_SCRIPTS_ROOT/test_device_logs.py -vvv -} - cd $TT_METAL_HOME # @@ -140,8 +133,6 @@ if [[ $1 == "PROFILER" ]]; then run_profiling_test elif [[ $1 == "PROFILER_NO_RESET" ]]; then run_profiling_no_reset_test -elif [[ $1 == "POST_PROC" ]]; then - run_post_proc_test else run_profiling_test run_post_proc_test diff --git a/tests/tt_metal/tools/profiler/device_log_run.py b/tests/tt_metal/tools/profiler/device_log_run.py deleted file mode 100644 index a2f4f7bfba3..00000000000 --- a/tests/tt_metal/tools/profiler/device_log_run.py +++ /dev/null @@ -1,84 +0,0 @@ -# SPDX-FileCopyrightText: © 2023 Tenstorrent Inc. - -# SPDX-License-Identifier: Apache-2.0 - -import os, sys -from filecmp import dircmp, cmp -from pathlib import Path -from difflib import Differ -import re -import fileinput - -import jsbeautifier -from loguru import logger - -from tt_metal.tools.profiler.common import TT_METAL_HOME, PROFILER_SCRIPTS_ROOT, PROFILER_ARTIFACTS_DIR - -TT_METAL_PATH = TT_METAL_HOME / "tt_metal" -GOLDEN_OUTPUTS_DIR = TT_METAL_PATH / "third_party/lfs/profiler/tests/golden/device/outputs" - -RE_RANDOM_ID_STRINGS = [r'if \(document.getElementById\("{0}"\)\) {{', r' Plotly.newPlot\("{0}", \[{{'] -DIFF_LINE_COUNT_LIMIT = 40 - - -def replace_random_id(line): - for randomIDStr in RE_RANDOM_ID_STRINGS: - match = re.search(f"^{randomIDStr.format('.*')}$", line) - if match: - return randomIDStr.format("random_id_replaced_for_automation").replace("\\", "") - return line - - -def filter_device_analysis_data(testOutputFolder): - testFiles = os.scandir(testOutputFolder) - for testFile in testFiles: - if "device_analysis_data.json" in testFile.name: - testFilePath = f"{testOutputFolder}/{testFile.name}" - for line in fileinput.input(testFilePath, inplace=True): - if "deviceInputLog" not in line: - print(line, end="") - - -def run_device_log_compare_golden(test): - goldenPath = GOLDEN_OUTPUTS_DIR / test - underTestPath = PROFILER_ARTIFACTS_DIR / "output/device" - - ret = os.system( - f"cd {PROFILER_SCRIPTS_ROOT} && ./process_device_log.py -d {goldenPath}/profile_log_device.csv --no-print-stats --no-artifacts" - ) - assert ret == 0, f"Log process script crashed with exit code {ret}" - - filter_device_analysis_data(underTestPath) - - dcmp = dircmp(goldenPath, underTestPath) - - for diffFile in dcmp.diff_files: - goldenFile = Path(f"{goldenPath}/{diffFile}") - underTestFile = Path(f"{underTestPath}/{diffFile}") - - diffStr = f"\n{diffFile}\n" - with open(goldenFile) as golden, open(underTestFile) as underTest: - differ = Differ() - lineCount = 0 - for line in differ.compare(golden.readlines(), underTest.readlines()): - if line[0] in ["-", "+", "?"]: - diffStr += line - lineCount += 1 - if lineCount > DIFF_LINE_COUNT_LIMIT: - diffStr += ( - "[NOTE: limited lines on log output, run locally without line count limits for more info]" - ) - break - logger.error(diffStr) - - assert not dcmp.diff_files, f"{dcmp.diff_files} cannot be different from golden" - assert not dcmp.right_only, f"New output files: {dcmp.right_only}" - assert not dcmp.left_only, f"Golden files not present in output: {dcmp.left_only}" - assert not dcmp.funny_files, f"Unreadable files: {dcmp.funny_files}" - - -def run_test(func): - def test(): - run_device_log_compare_golden(func.__name__) - - return test diff --git a/tests/tt_metal/tools/profiler/populate_golden.py b/tests/tt_metal/tools/profiler/populate_golden.py deleted file mode 100755 index b27f63ac2e9..00000000000 --- a/tests/tt_metal/tools/profiler/populate_golden.py +++ /dev/null @@ -1,114 +0,0 @@ -#! /usr/bin/env python3 - -# SPDX-FileCopyrightText: © 2023 Tenstorrent Inc. - -# SPDX-License-Identifier: Apache-2.0 - - -import os, sys -import csv -import re - -import click -from loguru import logger - -from device_log_run import filter_device_analysis_data - -from tt_metal.tools.profiler.common import TT_METAL_HOME, PROFILER_SCRIPTS_ROOT, PROFILER_ARTIFACTS_DIR, rm - -TT_METAL_PATH = TT_METAL_HOME / "tt_metal" -GOLDEN_OUTPUTS_DIR = TT_METAL_PATH / "third_party/lfs/profiler/tests/golden/device/outputs" -GOLDEN_LOGS_DIR = TT_METAL_PATH / "third_party/lfs/profiler/tests/golden/device/logs" -TEST_DEVICE_LOGS_PATH = TT_METAL_HOME / "tests/tt_metal/tools/profiler/test_device_logs.py" - -TEST_FILE_IMPORTS = """# SPDX-FileCopyrightText: © 2023 Tenstorrent Inc. - -# SPDX-License-Identifier: Apache-2.0 - -# THIS FILE IS AUTO-GENERATED -# Refer to the profiler README to learn how to generate this file using populate_golden.py - -from tests.tt_metal.tools.profiler.device_log_run import run_test -""" - -TEST_FILE_METHOD = """ -@run_test -def test_{}(): - pass -""" - - -@click.command() -@click.option("-w", "--wipe", default=False, is_flag=True, help="Wipe the golden outputs folder") -def main(wipe): - if wipe: - rm(GOLDEN_OUTPUTS_DIR) - os.mkdir(f"{GOLDEN_OUTPUTS_DIR}") - - testNames = [] - for logPath in os.listdir(GOLDEN_LOGS_DIR): - correctFormat = re.search("^profile_log_device_.*.csv$", logPath) - if correctFormat: - testName = logPath.split(".")[0].split("profile_log_device_")[-1].lower() - testNames.append((logPath, testName)) - testNames.sort() - - with open(f"{TEST_DEVICE_LOGS_PATH}", "w") as testsMethods: - print(TEST_FILE_IMPORTS, file=testsMethods) - - for testNum, (logPath, testName) in enumerate(testNames): - if not os.path.isdir(f"{GOLDEN_OUTPUTS_DIR}/test_{testName}"): - logger.info(f"Generating {testName}") - - csvPath = f"{GOLDEN_LOGS_DIR}/{logPath}" - - csvIsOld = False - lines = [] - with open(csvPath, "r") as deviceLog: - lines = deviceLog.readlines() - if "stat value" not in lines[1]: - csvIsOld = True - - if csvIsOld: - infoHead = lines[0] - csvHead = "PCIe slot, core_x, core_y, RISC processor type, timer_id, time[cycles since reset], stat value, Run ID, zone name, zone phase, source line, source file\n" - lines = lines[1:] - with open(csvPath, "w") as deviceLog: - for line in lines: - deviceLog.write(line) - - with open(csvPath, "r") as deviceLog: - csvRows = csv.DictReader(deviceLog) - with open("./tmp.csv", "w") as tmpLog: - tmpLog.write(infoHead) - tmpLog.write(csvHead) - newFields = csvRows.fieldnames - newFields = newFields[:5] + [" stat value"] + newFields[5:] - csvTmp = csv.DictWriter(tmpLog, newFields) - - for row in csvRows: - row[" stat value"] = 0 - csvTmp.writerow(row) - - os.system(f"cp tmp.csv {csvPath}") - - os.mkdir(f"{GOLDEN_OUTPUTS_DIR}/test_{testName}") - os.system(f"cp {GOLDEN_LOGS_DIR}/{logPath} {GOLDEN_OUTPUTS_DIR}/test_{testName}/profile_log_device.csv") - - ret = os.system( - f"cd {PROFILER_SCRIPTS_ROOT} && ./process_device_log.py -d {GOLDEN_OUTPUTS_DIR}/test_{testName}/profile_log_device.csv --no-artifacts --no-print-stats" - ) - assert ret == 0, f"Log process script crashed with exit code {ret}" - - os.system(f"cp {PROFILER_ARTIFACTS_DIR}/output/device/*.* {GOLDEN_OUTPUTS_DIR}/test_{testName}/") - filter_device_analysis_data(f"{GOLDEN_OUTPUTS_DIR}/test_{testName}/") - - # Remove line ending from the last test - if testNum == (len(testNames) - 1): - print(TEST_FILE_METHOD[:-1].format(testName), file=testsMethods) - else: - print(TEST_FILE_METHOD.format(testName), file=testsMethods) - - -if __name__ == "__main__": - main() diff --git a/tests/tt_metal/tools/profiler/test_device_logs.py b/tests/tt_metal/tools/profiler/test_device_logs.py deleted file mode 100644 index 315417be7e5..00000000000 --- a/tests/tt_metal/tools/profiler/test_device_logs.py +++ /dev/null @@ -1,93 +0,0 @@ -# SPDX-FileCopyrightText: © 2023 Tenstorrent Inc. - -# SPDX-License-Identifier: Apache-2.0 - -# THIS FILE IS AUTO-GENERATED -# Refer to the profiler README to learn how to generate this file using populate_golden.py - -from tests.tt_metal.tools.profiler.device_log_run import run_test - - -@run_test -def test_add_two_ints(): - pass - - -@run_test -def test_circular_trace_order(): - pass - - -@run_test -def test_compute(): - pass - - -@run_test -def test_full_buffer_nc_b_risc(): - pass - - -@run_test -def test_full_buffer_nc_b_t0_t1_t2_risc(): - pass - - -@run_test -def test_full_buffer_nc_risc(): - pass - - -@run_test -def test_grayskull_new_header(): - pass - - -@run_test -def test_large(): - pass - - -@run_test -def test_matmul(): - pass - - -@run_test -def test_matmul_small(): - pass - - -@run_test -def test_missing_brisc(): - pass - - -@run_test -def test_missing_ncrisc(): - pass - - -@run_test -def test_missing_all_defualt_markers(): - pass - - -@run_test -def test_multi_core_multi_launch(): - pass - - -@run_test -def test_very_long_launch_delta(): - pass - - -@run_test -def test_wormhole_multi_core(): - pass - - -@run_test -def test_wormhole_single_core(): - pass diff --git a/tt_metal/third_party/lfs b/tt_metal/third_party/lfs index 2ba5a297391..e82667eb9bd 160000 --- a/tt_metal/third_party/lfs +++ b/tt_metal/third_party/lfs @@ -1 +1 @@ -Subproject commit 2ba5a2973915aa1c5afbf6f1524e87e0c43bc058 +Subproject commit e82667eb9bdd42bcd9a0fe256e0081563624772a diff --git a/tt_metal/tools/profiler/process_device_log.py b/tt_metal/tools/profiler/process_device_log.py index 11210d5fce3..da7885acad2 100755 --- a/tt_metal/tools/profiler/process_device_log.py +++ b/tt_metal/tools/profiler/process_device_log.py @@ -507,7 +507,9 @@ def get_duration(riscData, analysis): desMarker = {"risc": risc, "zone_name": timerID["zone_name"]} if desMarker == analysis["marker"]: totalDuration += statData - return [dict(duration_type=analysis["marker"], duration_cycles=totalDuration)] + if totalDuration: + return [dict(duration_type=analysis["marker"], duration_cycles=totalDuration)] + return [] def adjacent_LF_analysis(riscData, analysis):