From 59d991ee1c63d89ca5a31fd18d10c7b74f938e62 Mon Sep 17 00:00:00 2001
From: James Newling <james.newling@gmail.com>
Date: Thu, 21 Nov 2024 04:23:44 -0800
Subject: [PATCH] [CI] Add new shape and with/without ukernel for benchmark
 tracking (#909)

Now prints performance numbers for shapes

(M, N, K) = (512, 512, 4096) and (512, 4096, 512) and (4096, 512, 512)

with and without ukernels.
---
 .../cpu_comparison/performance_summarizer.py  | 13 ++++++--
 build_tools/ci/cpu_comparison/run.py          | 32 +++++++++++++------
 2 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/build_tools/ci/cpu_comparison/performance_summarizer.py b/build_tools/ci/cpu_comparison/performance_summarizer.py
index 146be4aa7..401b06401 100644
--- a/build_tools/ci/cpu_comparison/performance_summarizer.py
+++ b/build_tools/ci/cpu_comparison/performance_summarizer.py
@@ -6,15 +6,22 @@
     import sys
 
     if len(sys.argv) != 2:
-        print("Usage: python3 performance_summarizer.py <path_to_log_file>")
+        print(
+            "Usage: python3 performance_summarizer.py <path_to_log_file>. This will strip out the performance numbers from the log file and print a summary."
+        )
         sys.exit(1)
     path = sys.argv[1]
     with open(path, "r") as f:
         lines = f.readlines()
     print("============================")
+    first_print = True
     for line in lines:
         if "Run #1" in line:
-            print(line.split()[-1])
+            if not first_print:
+                print("\n" + line.split()[-1])
+            else:
+                print(line.split()[-1])
+            first_print = False
         if "IREE_AMDAIE" in line:
-            print(line)
+            print(line.strip())
     print("============================")
diff --git a/build_tools/ci/cpu_comparison/run.py b/build_tools/ci/cpu_comparison/run.py
index 189348639..5060c027d 100755
--- a/build_tools/ci/cpu_comparison/run.py
+++ b/build_tools/ci/cpu_comparison/run.py
@@ -241,6 +241,7 @@ def __init__(
         run_on_target=["npu1_4col"],
         additional_labels=None,
         aie_compilation_flags=None,
+        n_repeats=1,
     ):
         super().__init__(
             run_on_target=run_on_target,
@@ -252,7 +253,7 @@ def __init__(
             acc_type=acc_type,
             tile_pipeline="pack-peel",
             use_ukernel=use_ukernel,
-            n_repeats=1,
+            n_repeats=n_repeats,
         )
 
         self.name = f"vanilla_matmul_{M}_{N}_{K}_{input_type}_{acc_type}"
@@ -1093,16 +1094,27 @@ def __init__(self):
             )
         )
 
-        self.register(
-            VanillaMatmul(
-                512,
-                512,
-                4096,
-                "bf16",
-                "f32",
-                additional_labels=["Performance"],
+        # Some bf16 Performance tests:
+        for M, N, K, use_ukernel in [
+            (512, 512, 4096, False),
+            (512, 512, 4096, True),
+            (512, 4096, 512, False),
+            (512, 4096, 512, True),
+            (4096, 512, 512, False),
+            (4096, 512, 512, True),
+        ]:
+            self.register(
+                VanillaMatmul(
+                    M,
+                    N,
+                    K,
+                    "bf16",
+                    "f32",
+                    additional_labels=["Performance"],
+                    use_ukernel=use_ukernel,
+                    n_repeats=2,
+                )
             )
-        )
 
         # MultipleDispatches tests:
         for name in ["two_matmul_switching", "matmul_f32_8_8_4", "matmul_f32_8_4_8"]: