Add support for test config overrides

It can be useful to override the config for certain tests. The main use
case for this is to override environment variables which control test
assertions (e.g. NO_DANGLING_FILESYSTEM, ERRNO_MODE_WINDOWS). When
running the tests in CI, it would be nicer to support this without
manually modifying the JSON test config files.

README.md

@@ -62,6 +62,21 @@ python3 test-runner/wasi_test_runner.py
     -r adapters/wasmtime.py # path to a runtime adapter
 ```
 
+You can also optionally override test configs with the `--config-override`
+option:
+
+```bash
+python3 test-runner/wasi_test_runner.py                                                  \
+    -t ./tests/assemblyscript/testsuite/ # path to folders containing .wasm test files   \
+       ./tests/c/testsuite/                                                              \
+       ./tests/rust/testsuite/                                                           \
+    --config-override examples/config_override.json                                      \
+    -r adapters/wasmtime.py # path to a runtime adapter
+```
+
+This can be useful for passing additional environment variables to certain
+tests without modifying the test config files.
+
 The default executable in the adapter used for test execution can be
 overridden using `TEST_RUNTIME_EXE` variable. This only works with adapters defined in
 [adapters/](adapters/), and might not work with 3rd party adapters.

examples/config_override.json

@@ -0,0 +1,16 @@
+{
+    "WASI C tests": {
+        "stat-dev-ino": {
+            "dirs": [
+                "fs-tests.dir"
+            ],
+            "env": {
+                "TEST_VAR": "test"
+            },
+            "args": [
+                "fs-tests.dir",
+                "test_arg"
+            ]
+        }
+    }
+}

test-runner/tests/test_test_suite_runner.py

@@ -67,7 +67,9 @@ def test_runner_end_to_end() -> None:
     filters = [filt]
 
     with patch("glob.glob", return_value=test_paths):
-        suite = tsr.run_tests_from_test_suite("my-path", runtime, validators, reporters, filters)  # type: ignore
+        suite = tsr.run_tests_from_test_suite(
+            "my-path", runtime, validators, reporters, filters, None  # type: ignore
+        )
 
     # Assert manifest was read correctly
     assert suite.name == "test-suite"
@@ -104,6 +106,6 @@ def test_runner_end_to_end() -> None:
 
 @patch("os.path.exists", Mock(return_value=False))
 def test_runner_should_use_path_for_name_if_manifest_does_not_exist() -> None:
-    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [])
+    suite = tsr.run_tests_from_test_suite("my-path", Mock(), [], [], [], None)
 
     assert suite.name == "my-path"

test-runner/wasi_test_runner/__main__.py

@@ -1,6 +1,6 @@
 import argparse
 import sys
-from typing import List
+from typing import List, Optional
 
 
 from .runtime_adapter import RuntimeAdapter
@@ -11,6 +11,7 @@
 from .reporters.console import ConsoleTestReporter
 from .reporters.json import JSONTestReporter
 from .validators import exit_code_validator, stdout_validator, Validator
+from .override import ConfigOverride, JSONConfigOverride
 
 
 def main() -> int:
@@ -46,6 +47,11 @@ def main() -> int:
         default=False,
         help="Disables color for console output reporter.",
     )
+    parser.add_argument(
+        "--config-override",
+        required=False,
+        help="Location of JSON file containing overrides for the config used for each test",
+    )
 
     options = parser.parse_args()
 
@@ -59,12 +65,17 @@ def main() -> int:
     for filt in options.exclude_filter:
         filters.append(JSONTestExcludeFilter(filt))
 
+    override: Optional[ConfigOverride] = None
+    if options.config_override:
+        override = JSONConfigOverride(options.config_override)
+
     return run_all_tests(
         RuntimeAdapter(options.runtime_adapter),
         options.test_suite,
         validators,
         reporters,
         filters,
+        override,
     )
 
 

test-runner/wasi_test_runner/harness.py

@@ -1,24 +1,27 @@
-from typing import List
+from typing import List, Optional
 
 from .filters import TestFilter
 from .reporters import TestReporter
 from .test_suite_runner import run_tests_from_test_suite
 from .runtime_adapter import RuntimeAdapter
 from .validators import Validator
+from .override import ConfigOverride
 
 
+# pylint: disable-msg=too-many-arguments
 def run_all_tests(
     runtime: RuntimeAdapter,
     test_suite_paths: List[str],
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    override: Optional[ConfigOverride],
 ) -> int:
     ret = 0
 
     for test_suite_path in test_suite_paths:
         test_suite = run_tests_from_test_suite(
-            test_suite_path, runtime, validators, reporters, filters,
+            test_suite_path, runtime, validators, reporters, filters, override
         )
         for reporter in reporters:
             reporter.report_test_suite(test_suite)

test-runner/wasi_test_runner/override.py

@@ -0,0 +1,38 @@
+import json
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+from .test_case import (
+    Config,
+)
+
+
+class ConfigOverride(ABC):
+    @abstractmethod
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        pass
+
+
+class JSONConfigOverride(ConfigOverride):
+    overrides_dict: Dict[str, Dict[str, Dict[str, Any]]]
+
+    def __init__(self, overrides_path: str) -> None:
+        with open(overrides_path, encoding="utf-8") as file:
+            self.overrides_dict = json.load(file)
+
+    def get_test_override(
+        self, test_suite_name: str, test_name: str
+    ) -> Optional[Config]:
+        test_suite_overrides = self.overrides_dict.get(test_suite_name)
+
+        if test_suite_overrides is None:
+            return None
+
+        test_override = test_suite_overrides.get(test_name)
+
+        if test_override is None:
+            return None
+
+        return Config.from_dict(test_override)

test-runner/wasi_test_runner/test_case.py

@@ -37,10 +37,19 @@ class Config(NamedTuple):
 
     @classmethod
     def from_file(cls: Type[T], config_file: str) -> T:
-        default = cls()
-
         with open(config_file, encoding="utf-8") as file:
             dict_config = json.load(file)
+            return cls.from_dict(dict_config)
+
+    @classmethod
+    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
+        for field_name in dict_config:
+            if field_name not in cls._fields:
+                logging.warning("Unknown field in the config file: %s", field_name)
+
+    @classmethod
+    def from_dict(cls: Type[T], dict_config: Dict[str, Any]) -> T:
+        default = cls()
 
         cls._validate_dict(dict_config)
 
@@ -53,12 +62,6 @@ def from_file(cls: Type[T], config_file: str) -> T:
             stdout=dict_config.get("stdout", default.stdout),
         )
 
-    @classmethod
-    def _validate_dict(cls: Type[T], dict_config: Dict[str, Any]) -> None:
-        for field_name in dict_config:
-            if field_name not in cls._fields:
-                logging.warning("Unknown field in the config file: %s", field_name)
-
 
 class TestCase(NamedTuple):
     name: str

test-runner/wasi_test_runner/test_suite_runner.py

@@ -6,7 +6,7 @@
 import time
 
 from datetime import datetime
-from typing import List, cast
+from typing import List, Optional, cast
 
 from .filters import TestFilter
 from .runtime_adapter import RuntimeAdapter
@@ -19,14 +19,17 @@
 from .reporters import TestReporter
 from .test_suite import TestSuite
 from .validators import Validator
+from .override import ConfigOverride
 
 
+# pylint: disable-msg=too-many-arguments, too-many-locals
 def run_tests_from_test_suite(
     test_suite_path: str,
     runtime: RuntimeAdapter,
     validators: List[Validator],
     reporters: List[TestReporter],
     filters: List[TestFilter],
+    config_override: Optional[ConfigOverride],
 ) -> TestSuite:
     test_cases: List[TestCase] = []
     test_start = datetime.now()
@@ -45,7 +48,14 @@ def run_tests_from_test_suite(
                 test_case = _skip_single_test(runtime, validators, test_path)
                 break
         else:
-            test_case = _execute_single_test(runtime, validators, test_path)
+            test_config_override = (
+                config_override.get_test_override(test_suite_name, test_name)
+                if config_override
+                else None
+            )
+            test_case = _execute_single_test(
+                runtime, validators, test_path, test_config_override
+            )
         test_cases.append(test_case)
         for reporter in reporters:
             reporter.report_test(test_case)
@@ -73,9 +83,12 @@ def _skip_single_test(
 
 
 def _execute_single_test(
-    runtime: RuntimeAdapter, validators: List[Validator], test_path: str
+    runtime: RuntimeAdapter,
+    validators: List[Validator],
+    test_path: str,
+    config_override: Optional[Config],
 ) -> TestCase:
-    config = _read_test_config(test_path)
+    config = config_override or _read_test_config(test_path)
     test_start = time.time()
     test_output = runtime.run_test(test_path, config.args, config.env, config.dirs)
     elapsed = time.time() - test_start