[CI] Add new shape and with/without ukernel for benchmark tracking (#909

) Now prints performance numbers for shapes (M, N, K) = (512, 512, 4096) and (512, 4096, 512) and (4096, 512, 512) with and without ukernels.
nod-ai · Nov 21, 2024 · 59d991e · 59d991e
1 parent d7bf670
commit 59d991e
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 13 deletions.
diff --git a/build_tools/ci/cpu_comparison/performance_summarizer.py b/build_tools/ci/cpu_comparison/performance_summarizer.py
@@ -6,15 +6,22 @@
     import sys
 
     if len(sys.argv) != 2:
-        print("Usage: python3 performance_summarizer.py <path_to_log_file>")
+        print(
+            "Usage: python3 performance_summarizer.py <path_to_log_file>. This will strip out the performance numbers from the log file and print a summary."
+        )
         sys.exit(1)
     path = sys.argv[1]
     with open(path, "r") as f:
         lines = f.readlines()
     print("============================")
+    first_print = True
     for line in lines:
         if "Run #1" in line:
-            print(line.split()[-1])
+            if not first_print:
+                print("\n" + line.split()[-1])
+            else:
+                print(line.split()[-1])
+            first_print = False
         if "IREE_AMDAIE" in line:
-            print(line)
+            print(line.strip())
     print("============================")
diff --git a/build_tools/ci/cpu_comparison/run.py b/build_tools/ci/cpu_comparison/run.py
@@ -241,6 +241,7 @@ def __init__(
         run_on_target=["npu1_4col"],
         additional_labels=None,
         aie_compilation_flags=None,
+        n_repeats=1,
     ):
         super().__init__(
             run_on_target=run_on_target,
@@ -252,7 +253,7 @@ def __init__(
             acc_type=acc_type,
             tile_pipeline="pack-peel",
             use_ukernel=use_ukernel,
-            n_repeats=1,
+            n_repeats=n_repeats,
         )
 
         self.name = f"vanilla_matmul_{M}_{N}_{K}_{input_type}_{acc_type}"
@@ -1093,16 +1094,27 @@ def __init__(self):
             )
         )
 
-        self.register(
-            VanillaMatmul(
-                512,
-                512,
-                4096,
-                "bf16",
-                "f32",
-                additional_labels=["Performance"],
+        # Some bf16 Performance tests:
+        for M, N, K, use_ukernel in [
+            (512, 512, 4096, False),
+            (512, 512, 4096, True),
+            (512, 4096, 512, False),
+            (512, 4096, 512, True),
+            (4096, 512, 512, False),
+            (4096, 512, 512, True),
+        ]:
+            self.register(
+                VanillaMatmul(
+                    M,
+                    N,
+                    K,
+                    "bf16",
+                    "f32",
+                    additional_labels=["Performance"],
+                    use_ukernel=use_ukernel,
+                    n_repeats=2,
+                )
             )
-        )
 
         # MultipleDispatches tests:
         for name in ["two_matmul_switching", "matmul_f32_8_8_4", "matmul_f32_8_4_8"]: