From 61b92b6f03bad9b4875906d5954a67c1b6a98b6d Mon Sep 17 00:00:00 2001 From: sabira-mcw Date: Fri, 22 Nov 2024 08:06:22 +0000 Subject: [PATCH] #13329: Run Mnist perf test for 100 iter and assert accuracy check in demo --- models/demos/mnist/README.md | 2 -- models/demos/mnist/demo/demo.py | 15 +++++------- models/demos/mnist/tests/test_perf_mnist.py | 23 ++++++++++--------- models/demos/mnist/tt/tt_mnist.py | 1 - .../single_card/run_single_card_demo_tests.sh | 2 +- .../integration_tests/mnist/test_mnist.py | 9 +++++--- 6 files changed, 25 insertions(+), 27 deletions(-) diff --git a/models/demos/mnist/README.md b/models/demos/mnist/README.md index a85cb3234811..fa575a109671 100644 --- a/models/demos/mnist/README.md +++ b/models/demos/mnist/README.md @@ -28,5 +28,3 @@ The demo receives inputs from respective dataset MNIST. ## Additional Information If you encounter issues when running the model, ensure that device has support for all required operations. - -### Owner: [sabira-mcw](https://github.com/sabira-mcw) diff --git a/models/demos/mnist/demo/demo.py b/models/demos/mnist/demo/demo.py index d0ba9a21c88d..55c712f9a7c3 100644 --- a/models/demos/mnist/demo/demo.py +++ b/models/demos/mnist/demo/demo.py @@ -5,17 +5,15 @@ import pytest import torch import ttnn - from torchvision import transforms, datasets from loguru import logger -from models.utility_functions import ( - disable_persistent_kernel_cache, -) from torch.utils.data import DataLoader -from models.demos.mnist.reference.mnist import MnistModel -from models.demos.mnist.tt import tt_mnist from ttnn.model_preprocessing import preprocess_model_parameters +from models.utility_functions import disable_persistent_kernel_cache + +from models.demos.mnist.reference.mnist import MnistModel +from models.demos.mnist.tt import tt_mnist def run_demo_dataset(device, batch_size, iterations, model_location_generator): @@ -49,8 +47,7 @@ def run_demo_dataset(device, batch_size, iterations, model_location_generator): ttnn_predictions.append(predicted_label[i]) logger.info(f"Iter: {iters} Sample {i}:") logger.info(f"Expected Label: {dataset_predictions[i]}") - logger.info(f"Predicted Label: {ttnn_predictions[i]}") - + logger.info(f"TT Predicted Label: {ttnn_predictions[i]}") if dataset_predictions[i] == ttnn_predictions[i]: dataset_ttnn_correct += 1 correct += 1 @@ -58,9 +55,9 @@ def run_demo_dataset(device, batch_size, iterations, model_location_generator): logger.info( f"ImageNet Inference Accuracy for iter {iters} of {batch_size} input samples : {dataset_ttnn_accuracy}" ) - accuracy = correct / (batch_size * iterations) logger.info(f"ImageNet Inference Accuracy for {batch_size}x{iterations} Samples : {accuracy}") + assert accuracy >= 0.96875, f"Expected accuracy : {0.96875} Actual accuracy: {accuracy}" @pytest.mark.parametrize("device_params", [{"l1_small_size": 32768}], indirect=True) diff --git a/models/demos/mnist/tests/test_perf_mnist.py b/models/demos/mnist/tests/test_perf_mnist.py index f737a31c468e..c60efe5f16e6 100644 --- a/models/demos/mnist/tests/test_perf_mnist.py +++ b/models/demos/mnist/tests/test_perf_mnist.py @@ -9,17 +9,20 @@ from loguru import logger from torchvision import transforms, datasets from torch.utils.data import DataLoader + +from ttnn.model_preprocessing import preprocess_model_parameters +from models.perf.perf_utils import prep_perf_report from models.utility_functions import ( enable_persistent_kernel_cache, disable_persistent_kernel_cache, ) -from models.perf.perf_utils import prep_perf_report -from ttnn.model_preprocessing import preprocess_model_parameters -from models.demos.mnist.reference.mnist import MnistModel from models.utility_functions import is_grayskull, is_wormhole_b0 from models.perf.device_perf_utils import run_device_perf, check_device_perf, prep_device_perf_report + +from models.demos.mnist.reference.mnist import MnistModel from models.demos.mnist.tt import tt_mnist + transform = transforms.Compose([transforms.ToTensor()]) test_dataset = datasets.MNIST(root="./data", train=False, transform=None, download=True) @@ -27,11 +30,11 @@ def get_expected_times(tt_mnist): if is_grayskull(): return { - tt_mnist: (3.54, 0.00905), + tt_mnist: (3.54, 0.005), }[tt_mnist] elif is_wormhole_b0(): return { - tt_mnist: (8.14, 0.0081), + tt_mnist: (3.89, 0.005), }[tt_mnist] @@ -60,10 +63,9 @@ def test_performance_mnist(device, batch_size, tt_mnist, model_location_generato test_dataset = datasets.MNIST(root="./data", train=False, transform=transform, download=True) dataloader = DataLoader(test_dataset, batch_size=batch_size) x, labels = next(iter(dataloader)) - test_input = ttnn.from_torch(x, dtype=ttnn.bfloat16, device=device) durations = [] - for _ in range(2): + for _ in range(100): start = time.time() ttnn_output = tt_mnist.mnist( @@ -75,7 +77,6 @@ def test_performance_mnist(device, batch_size, tt_mnist, model_location_generato end = time.time() durations.append(end - start) enable_persistent_kernel_cache() - inference_and_compile_time, *inference_times = durations inference_time = sum(inference_times) / len(inference_times) expected_compile_time, expected_inference_time = get_expected_times(tt_mnist) @@ -111,9 +112,9 @@ def test_perf_device_bare_metal(batch_size, reset_seeds): num_iterations = 1 margin = 0.03 if is_grayskull(): - expected_perf = 588743.96 + expected_perf = 653017.5 elif is_wormhole_b0(): - expected_perf = 1338730.2 + expected_perf = 1383185.64944 command = f"pytest tests/ttnn/integration_tests/mnist/test_mnist.py::test_mnist" cols = ["DEVICE FW", "DEVICE KERNEL", "DEVICE BRISC KERNEL"] @@ -122,7 +123,7 @@ def test_perf_device_bare_metal(batch_size, reset_seeds): expected_perf_cols = {inference_time_key: expected_perf} post_processed_results = run_device_perf(command, subdir, num_iterations, cols, batch_size) - expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols) + expected_results = check_device_perf(post_processed_results, margin, expected_perf_cols, assert_on_fail=True) prep_device_perf_report( model_name=f"tt_mnist{batch_size}", batch_size=batch_size, diff --git a/models/demos/mnist/tt/tt_mnist.py b/models/demos/mnist/tt/tt_mnist.py index b6ee5b72e6a2..6a4da567aad4 100644 --- a/models/demos/mnist/tt/tt_mnist.py +++ b/models/demos/mnist/tt/tt_mnist.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 import ttnn -import torch def mnist(device, batch_size, x, parameters): diff --git a/tests/scripts/single_card/run_single_card_demo_tests.sh b/tests/scripts/single_card/run_single_card_demo_tests.sh index 934e8122fe3d..aec300876413 100755 --- a/tests/scripts/single_card/run_single_card_demo_tests.sh +++ b/tests/scripts/single_card/run_single_card_demo_tests.sh @@ -34,7 +34,7 @@ run_common_func_tests() { # ConvNet Mnist pytest --disable-warnings models/demos/convnet_mnist/demo/demo.py --timeout 600; fail+=$? - #mnist + # Mnist pytest --disable-warnings models/demos/mnist/demo/demo.py --timeout 600; fail+=$? return $fail diff --git a/tests/ttnn/integration_tests/mnist/test_mnist.py b/tests/ttnn/integration_tests/mnist/test_mnist.py index fcb8fefbc441..e4ea5143a08a 100644 --- a/tests/ttnn/integration_tests/mnist/test_mnist.py +++ b/tests/ttnn/integration_tests/mnist/test_mnist.py @@ -5,13 +5,15 @@ import torch import ttnn import pytest -from models.demos.mnist.reference.mnist import MnistModel -from models.demos.mnist.tt import tt_mnist -from torch.utils.data import DataLoader from torchvision import transforms, datasets +from torch.utils.data import DataLoader + from tests.ttnn.utils_for_testing import assert_with_pcc from ttnn.model_preprocessing import preprocess_model_parameters +from models.demos.mnist.reference.mnist import MnistModel +from models.demos.mnist.tt import tt_mnist + @pytest.mark.parametrize("device_params", [{"l1_small_size": 32768}], indirect=True) @pytest.mark.parametrize( @@ -40,4 +42,5 @@ def test_mnist(reset_seeds, device, batch_size, model_location_generator): tt_output = tt_mnist.mnist(device, batch_size, x, parameters) tt_output = ttnn.to_torch(tt_output) + print(tt_output.shape) assert_with_pcc(torch_output, tt_output, 0.99)