#13329: Run Mnist perf test for 100 iter and assert accuracy check in…

… demo

models/demos/mnist/README.md

@@ -28,5 +28,3 @@ The demo receives inputs from respective dataset MNIST.
 ## Additional Information
 
 If you encounter issues when running the model, ensure that device has support for all required operations.
-
-### Owner: [sabira-mcw](https://github.com/sabira-mcw)

models/demos/mnist/demo/demo.py

@@ -5,17 +5,15 @@
 import pytest
 import torch
 import ttnn
-
 from torchvision import transforms, datasets
 from loguru import logger
-from models.utility_functions import (
-    disable_persistent_kernel_cache,
-)
 from torch.utils.data import DataLoader
-from models.demos.mnist.reference.mnist import MnistModel
-from models.demos.mnist.tt import tt_mnist
 
 from ttnn.model_preprocessing import preprocess_model_parameters
+from models.utility_functions import disable_persistent_kernel_cache
+
+from models.demos.mnist.reference.mnist import MnistModel
+from models.demos.mnist.tt import tt_mnist
 
 
 def run_demo_dataset(device, batch_size, iterations, model_location_generator):
@@ -49,18 +47,17 @@ def run_demo_dataset(device, batch_size, iterations, model_location_generator):
             ttnn_predictions.append(predicted_label[i])
             logger.info(f"Iter: {iters} Sample {i}:")
             logger.info(f"Expected Label: {dataset_predictions[i]}")
-            logger.info(f"Predicted Label: {ttnn_predictions[i]}")
-
+            logger.info(f"TT Predicted Label: {ttnn_predictions[i]}")
             if dataset_predictions[i] == ttnn_predictions[i]:
                 dataset_ttnn_correct += 1
                 correct += 1
         dataset_ttnn_accuracy = dataset_ttnn_correct / (batch_size)
         logger.info(
             f"ImageNet Inference Accuracy for iter {iters} of {batch_size} input samples : {dataset_ttnn_accuracy}"
         )
-
     accuracy = correct / (batch_size * iterations)
     logger.info(f"ImageNet Inference Accuracy for {batch_size}x{iterations} Samples : {accuracy}")
+    assert accuracy >= 0.96875, f"Expected accuracy : {0.96875} Actual accuracy: {accuracy}"
 
 
 @pytest.mark.parametrize("device_params", [{"l1_small_size": 32768}], indirect=True)

models/demos/mnist/tests/test_perf_mnist.py

@@ -9,29 +9,32 @@
 from loguru import logger
 from torchvision import transforms, datasets
 from torch.utils.data import DataLoader
+
+from ttnn.model_preprocessing import preprocess_model_parameters
+from models.perf.perf_utils import prep_perf_report
 from models.utility_functions import (
     enable_persistent_kernel_cache,
     disable_persistent_kernel_cache,
 )
-from models.perf.perf_utils import prep_perf_report
-from ttnn.model_preprocessing import preprocess_model_parameters
-from models.demos.mnist.reference.mnist import MnistModel
 from models.utility_functions import is_grayskull, is_wormhole_b0
 from models.perf.device_perf_utils import run_device_perf, check_device_perf, prep_device_perf_report
+
+from models.demos.mnist.reference.mnist import MnistModel
 from models.demos.mnist.tt import tt_mnist
 
+
 transform = transforms.Compose([transforms.ToTensor()])
 test_dataset = datasets.MNIST(root="./data", train=False, transform=None, download=True)
 
 
 def get_expected_times(tt_mnist):
     if is_grayskull():
         return {
-            tt_mnist: (3.54, 0.00905),
+            tt_mnist: (3.54, 0.005),
         }[tt_mnist]
     elif is_wormhole_b0():
         return {
-            tt_mnist: (8.14, 0.0081),
+            tt_mnist: (3.89, 0.005),
         }[tt_mnist]
 
 
@@ -60,10 +63,9 @@ def test_performance_mnist(device, batch_size, tt_mnist, model_location_generato
     test_dataset = datasets.MNIST(root="./data", train=False, transform=transform, download=True)
     dataloader = DataLoader(test_dataset, batch_size=batch_size)
     x, labels = next(iter(dataloader))
-
     test_input = ttnn.from_torch(x, dtype=ttnn.bfloat16, device=device)
     durations = []
-    for _ in range(2):
+    for _ in range(100):
         start = time.time()
 
         ttnn_output = tt_mnist.mnist(
@@ -75,7 +77,6 @@ def test_performance_mnist(device, batch_size, tt_mnist, model_location_generato
         end = time.time()
         durations.append(end - start)
         enable_persistent_kernel_cache()
-
     inference_and_compile_time, *inference_times = durations
     inference_time = sum(inference_times) / len(inference_times)
     expected_compile_time, expected_inference_time = get_expected_times(tt_mnist)
@@ -111,9 +112,9 @@ def test_perf_device_bare_metal(batch_size, reset_seeds):
     num_iterations = 1
     margin = 0.03
     if is_grayskull():
-        expected_perf = 588743.96
+        expected_perf = 653017.5
     elif is_wormhole_b0():
-        expected_perf = 1338730.2
+        expected_perf = 1383185.64944
 
     command = f"pytest tests/ttnn/integration_tests/mnist/test_mnist.py::test_mnist"
     cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"]
@@ -122,7 +123,7 @@ def test_perf_device_bare_metal(batch_size, reset_seeds):
     expected_perf_cols = {inference_time_key: expected_perf}
 
     post_processed_results = run_device_perf(command, subdir, num_iterations, cols, batch_size)
-    expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols)
+    expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols, assert_on_fail=True)
     prep_device_perf_report(
         model_name=f"tt_mnist{batch_size}",
         batch_size=batch_size,

models/demos/mnist/tt/tt_mnist.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import ttnn
-import torch
 
 
 def mnist(device, batch_size, x, parameters):

tests/scripts/single_card/run_single_card_demo_tests.sh

@@ -34,7 +34,7 @@ run_common_func_tests() {
   # ConvNet Mnist
   pytest --disable-warnings models/demos/convnet_mnist/demo/demo.py --timeout 600; fail+=$?
 
-  #mnist
+  # Mnist
   pytest --disable-warnings models/demos/mnist/demo/demo.py --timeout 600; fail+=$?
 
   return $fail

tests/ttnn/integration_tests/mnist/test_mnist.py

@@ -5,13 +5,15 @@
 import torch
 import ttnn
 import pytest
-from models.demos.mnist.reference.mnist import MnistModel
-from models.demos.mnist.tt import tt_mnist
-from torch.utils.data import DataLoader
 from torchvision import transforms, datasets
+from torch.utils.data import DataLoader
+
 from tests.ttnn.utils_for_testing import assert_with_pcc
 from ttnn.model_preprocessing import preprocess_model_parameters
 
+from models.demos.mnist.reference.mnist import MnistModel
+from models.demos.mnist.tt import tt_mnist
+
 
 @pytest.mark.parametrize("device_params", [{"l1_small_size": 32768}], indirect=True)
 @pytest.mark.parametrize(
@@ -40,4 +42,5 @@ def test_mnist(reset_seeds, device, batch_size, model_location_generator):
     tt_output = tt_mnist.mnist(device, batch_size, x, parameters)
 
     tt_output = ttnn.to_torch(tt_output)
+    print(tt_output.shape)
     assert_with_pcc(torch_output, tt_output, 0.99)