Add support for test config overrides

README.md

@@ -62,6 +62,21 @@ python3 test-runner/wasi_test_runner.py
     -r adapters/wasmtime.py # path to a runtime adapter
 ```
 
+You can also optionally override test configs with the `--config-override`
+option:
+
+```bash
+python3 test-runner/wasi_test_runner.py                                                  \
+    -t ./tests/assemblyscript/testsuite/ # path to folders containing .wasm test files   \
+       ./tests/c/testsuite/                                                              \
+       ./tests/rust/testsuite/                                                           \
+    --config-override examples/config_override.json                                      \
+    -r adapters/wasmtime.py # path to a runtime adapter
+```
+
+This can be useful for passing additional environment variables to certain
+tests without modifying the test config files.
+
 The default executable in the adapter used for test execution can be
 overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
 [adapters/](adapters/), and might not work with 3rd party adapters.

examples/config_override.json

@@ -0,0 +1,16 @@
+{
+    "WASI C tests": {
+        "stat-dev-ino": {
+            "dirs": [
+                "fs-tests.dir"
+            ],
+            "env": {
+                "TEST_VAR": "test"
+            },
+            "args": [
+                "fs-tests.dir",
+                "test_arg"
+            ]
+        }
+    }
+}

test-runner/tests/test_test_suite_runner.py

@@ -67,7 +67,9 @@ def test_runner_end_to_end() -> None:
     filters = [filt]
 
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite(
+            "my-path", runtime, validators, reporters, filters, None  # type: ignore
+        )
 
     # Assert manifest was read correctly
     assert suite.name == "test-suite"
@@ -104,6 +106,6 @@ def test_runner_end_to_end() -> None:
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
-    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
+    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [], None)
 
     assert suite.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -1,6 +1,6 @@
 import argparse
 import sys
-from typing import List
+from typing import List, Optional
 
 
 from .runtime_adapter import RuntimeAdapter
@@ -11,6 +11,7 @@
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
 from .validators import exit_code_validator, stdout_validator, Validator
+from .override import ConfigOverride, JSONConfigOverride
 
 
 def main() -> int:
@@ -46,6 +47,11 @@ def main() -> int:
         default=False,
         help="Disables color for console output reporter.",
     )
+    parser.add_argument(
+        "--config-override",
+        required=False,
+        help="Location of JSON file containing overrides for the config used for each test",
+    )
 
     options = parser.parse_args()
 
@@ -59,12 +65,17 @@ def main() -> int:
     for filt in options.exclude_filter:
         filters.append(JSONTestExcludeFilter(filt))
 
+    override: Optional[ConfigOverride] = None
+    if options.config_override:
+        override = JSONConfigOverride(options.config_override)
+
     return run_all_tests(
         RuntimeAdapter(options.runtime_adapter),
         options.test_suite,
         validators,
         reporters,
         filters,
+        override,
     )
 
 

test-runner/wasi_test_runner/harness.py

@@ -1,24 +1,27 @@
-from typing import List
+from typing import List, Optional
 
 from .filters import TestFilter
 from .reporters import TestReporter
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
 from .validators import Validator
+from .override import ConfigOverride
 
 
+# pylint: disable-msg=too-many-arguments
 def run_all_tests(
     runtime: RuntimeAdapter,
     test_suite_paths: List[str],
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    override: Optional[ConfigOverride],
 ) -> int:
     ret = 0
 
     for test_suite_path in test_suite_paths:
         test_suite = run_tests_from_test_suite(
-            test_suite_path, runtime, validators, reporters, filters,
+            test_suite_path, runtime, validators, reporters, filters, override
         )
         for reporter in reporters:
             reporter.report_test_suite(test_suite)

test-runner/wasi_test_runner/override.py

@@ -0,0 +1,38 @@
+import json
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+from .test_case import (
+    Config,
+)
+
+
+class ConfigOverride(ABC):
+    @abstractmethod
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        pass
+
+
+class JSONConfigOverride(ConfigOverride):
+    overrides_dict: Dict[str, Dict[str, Dict[str, Any]]]
+
+    def __init__(self, overrides_path: str) -> None:
+        with open(overrides_path, encoding="utf-8") as file:
+            self.overrides_dict = json.load(file)
+
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        test_suite_overrides = self.overrides_dict.get(test_suite_name)
+
+        if test_suite_overrides is None:
+            return None
+
+        test_override = test_suite_overrides.get(test_name)
+
+        if test_override is None:
+            return None
+
+        return Config.from_dict(test_override)

test-runner/wasi_test_runner/test_case.py

@@ -37,10 +37,19 @@ class Config(NamedTuple):
 
     @classmethod
     def from_file(cls: Type[T], config_file: str) -> T:
-        default = cls()
-
         with open(config_file, encoding="utf-8") as file:
             dict_config = json.load(file)
+            return cls.from_dict(dict_config)
+
+    @classmethod
+    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
+        for field_name in dict_config:
+            if field_name not in cls._fields:
+                logging.warning("Unknown field in the config file: %s", field_name)
+
+    @classmethod
+    def from_dict(cls: Type[T], dict_config: Dict[str, Any]) -> T:
+        default = cls()
 
         cls._validate_dict(dict_config)
 
@@ -53,12 +62,6 @@ def from_file(cls: Type[T], config_file: str) -> T:
             stdout=dict_config.get("stdout", default.stdout),
         )
 
-    @classmethod
-    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
-        for field_name in dict_config:
-            if field_name not in cls._fields:
-                logging.warning("Unknown field in the config file: %s", field_name)
-
 
 class TestCase(NamedTuple):
     name: str

test-runner/wasi_test_runner/test_suite_runner.py

@@ -6,7 +6,7 @@
 import time
 
 from datetime import datetime
-from typing import List, cast
+from typing import List, Optional, cast
 
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
@@ -19,14 +19,17 @@
 from .reporters import TestReporter
 from .test_suite import TestSuite
 from .validators import Validator
+from .override import ConfigOverride
 
 
+# pylint: disable-msg=too-many-arguments, too-many-locals
 def run_tests_from_test_suite(
     test_suite_path: str,
     runtime: RuntimeAdapter,
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    config_override: Optional[ConfigOverride],
 ) -> TestSuite:
     test_cases: List[TestCase] = []
     test_start = datetime.now()
@@ -45,7 +48,14 @@ def run_tests_from_test_suite(
                 test_case = _skip_single_test(runtime, validators, test_path)
                 break
         else:
-            test_case = _execute_single_test(runtime, validators, test_path)
+            test_config_override = (
+                config_override.get_test_override(test_suite_name, test_name)
+                if config_override
+                else None
+            )
+            test_case = _execute_single_test(
+                runtime, validators, test_path, test_config_override
+            )
         test_cases.append(test_case)
         for reporter in reporters:
             reporter.report_test(test_case)
@@ -73,9 +83,12 @@ def _skip_single_test(
 
 
 def _execute_single_test(
-    runtime: RuntimeAdapter, validators: List[Validator], test_path: str
+    runtime: RuntimeAdapter,
+    validators: List[Validator],
+    test_path: str,
+    config_override: Optional[Config],
 ) -> TestCase:
-    config = _read_test_config(test_path)
+    config = config_override or _read_test_config(test_path)
     test_start = time.time()
     test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
     elapsed = time.time() - test_start