[benchmarks] Remove --strict-compatible argument. (pytorch#6491)

amithrm · Mar 1, 2024 · 1e93b22 · 1e93b22
1 parent b41a812
commit 1e93b22
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 32 deletions.
diff --git a/benchmarks/experiment_runner.py b/benchmarks/experiment_runner.py
@@ -120,8 +120,7 @@ def generate_and_run_all_configs(self):
         #
         # Otherwise, we should go ahead and execute it.
         if (not self._args.no_skip and not self.model_loader.is_compatible(
-            benchmark_model, benchmark_experiment,
-            self._args.strict_compatible)):
+            benchmark_model, benchmark_experiment)):
           logger.warning("SKIP incompatible model and experiment configs.")
           self._save_results(benchmark_experiment.to_dict(),
                              benchmark_model.to_dict(), {"error": "SKIP"})
@@ -883,11 +882,6 @@ def __str__(self):
       action="store_true",
       help="""If set, verifies the model output with PT Eager mode, and saves relative error to the output file."""
   )
-  parser.add_argument(
-      "--strict-compatible",
-      action="store_true",
-      help="Strictly skips some models including models without installation file or causing stackdump.",
-  )
   parser.add_argument(
       "--no-skip",
       action="store_true",

diff --git a/benchmarks/torchbench_model.py b/benchmarks/torchbench_model.py
@@ -135,25 +135,6 @@
     ],
 }
 
-# This strict deny list denies tests that hold for too long and timeoout.
-STRICT_DENY_LIST = {
-    **{
-        "opacus_cifar10": [{
-            "accelerator": "tpu",
-        },],  # stackdump issue in TPU
-        "pytorch_stargan": [{
-            "accelerator": "tpu",
-        },],  # stackdump issue in TPU
-        "soft_actor_critic": [{
-            "accelerator": "tpu",
-        },],  # stackdump issue in TPU
-        "speech_transformer": [{
-            "accelerator": "tpu",
-        },],  # stackdump issue in TPU
-    },
-    **DENY_LIST
-}
-
 
 class TorchBenchModelLoader(ModelLoader):
 
@@ -234,12 +215,8 @@ def list_model_configs(self):
 
     return model_configs
 
-  def is_compatible(self,
-                    dummy_benchmark_model,
-                    benchmark_experiment,
-                    use_strict_deny=False):
+  def is_compatible(self, dummy_benchmark_model, benchmark_experiment):
     name = dummy_benchmark_model.model_name
-    deny_list = STRICT_DENY_LIST if use_strict_deny else DENY_LIST
 
     if name in self.skip["skip"]:
       return False
@@ -258,7 +235,7 @@ def is_compatible(self,
     def is_attr_eq(k, v):
       return getattr(benchmark_experiment, k) == v
 
-    for deny_experiment_config in deny_list.get(name, []):
+    for deny_experiment_config in DENY_LIST.get(name, []):
       if all(is_attr_eq(k, v) for k, v in deny_experiment_config.items()):
         return False