#13329: Run Mnist perf test for 100 iter and assert accuracy check in…

… demo

models/demos/mnist/README.md

@@ -28,5 +28,3 @@ The demo receives inputs from respective dataset MNIST.
 ## Additional Information
 
 If you encounter issues when running the model, ensure that device has support for all required operations.
-
-### Owner: [sabira-mcw](https://github.com/sabira-mcw)

models/demos/mnist/demo/demo.py

@@ -49,7 +49,7 @@ def run_demo_dataset(device, batch_size, iterations, model_location_generator):
             ttnn_predictions.append(predicted_label[i])
             logger.info(f"Iter: {iters} Sample {i}:")
             logger.info(f"Expected Label: {dataset_predictions[i]}")
-            logger.info(f"Predicted Label: {ttnn_predictions[i]}")
+            logger.info(f"TT Predicted Label: {ttnn_predictions[i]}")
 
             if dataset_predictions[i] == ttnn_predictions[i]:
                 dataset_ttnn_correct += 1
@@ -61,6 +61,7 @@ def run_demo_dataset(device, batch_size, iterations, model_location_generator):
 
     accuracy = correct / (batch_size * iterations)
     logger.info(f"ImageNet Inference Accuracy for {batch_size}x{iterations} Samples : {accuracy}")
+    assert accuracy >= 0.96875, f"Expected accuracy : {0.96875} Actual accuracy: {accuracy}"
 
 
 @pytest.mark.parametrize("device_params", [{"l1_small_size": 32768}], indirect=True)

models/demos/mnist/tests/test_perf_mnist.py

@@ -27,11 +27,11 @@
 def get_expected_times(tt_mnist):
     if is_grayskull():
         return {
-            tt_mnist: (3.54, 0.00905),
+            tt_mnist: (3.54, 0.005),
         }[tt_mnist]
     elif is_wormhole_b0():
         return {
-            tt_mnist: (8.14, 0.0081),
+            tt_mnist: (3.89, 0.005),
         }[tt_mnist]
 
 
@@ -63,7 +63,7 @@ def test_performance_mnist(device, batch_size, tt_mnist, model_location_generato
 
     test_input = ttnn.from_torch(x, dtype=ttnn.bfloat16, device=device)
     durations = []
-    for _ in range(2):
+    for _ in range(100):
         start = time.time()
 
         ttnn_output = tt_mnist.mnist(
@@ -111,7 +111,7 @@ def test_perf_device_bare_metal(batch_size, reset_seeds):
     num_iterations = 1
     margin = 0.03
     if is_grayskull():
-        expected_perf = 588743.96
+        expected_perf = 653017.5
     elif is_wormhole_b0():
         expected_perf = 1338730.2
 
@@ -122,7 +122,7 @@ def test_perf_device_bare_metal(batch_size, reset_seeds):
     expected_perf_cols = {inference_time_key: expected_perf}
 
     post_processed_results = run_device_perf(command, subdir, num_iterations, cols, batch_size)
-    expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols)
+    expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols, assert_on_fail=True)
     prep_device_perf_report(
         model_name=f"tt_mnist{batch_size}",
         batch_size=batch_size,