Use e3 facilities for test skipping

testsuite/README.md

@@ -43,7 +43,7 @@ All tests are based on running a Python script. There are these test drivers:
 - `docker-wrapper`: run in a pristine docker Ubuntu image in shared build mode.
 
 # Environment variables
-The following variables can be used to modify test/testsuite behavior.
+The following variables can be used to modify testsuite behavior.
 
 For `ALIRE_DISABLE_*` variables, their mere existence activates their function,
 no matter their value, or lack of one.

testsuite/drivers/driver/base_driver.py

@@ -0,0 +1,54 @@
+import os
+from typing import Any, Dict
+import e3
+
+import drivers.helpers
+
+from e3.testsuite.control import YAMLTestControlCreator
+from e3.testsuite.driver.classic import ClassicTestDriver
+
+
+class BaseDriver(ClassicTestDriver):
+    """
+    A common base driver from which the rest inherit, so common functionality
+    can be shared by all drivers. In particular, enabling/disabling via the e3
+    control feature.
+    """
+
+    # Environment variables that can be used to modify the testsuite behavior
+    # in the control field of the test YAML file.
+    # E.g.:
+    #     control:
+    #         - [SKIP, "skip_docker", "Docker is disabled"]
+    MODIFIERS = {
+          "distro"  : 'ALIRE_DISABLE_DISTRO'
+        , "docker"  : 'ALIRE_DISABLE_DOCKER'
+        , "local"   : 'ALIRE_ENABLE_LOCAL_TESTS'
+        , "network" : 'ALIRE_DISABLE_NETWORK_TESTS'
+    }
+
+    # In the constructor, prepare the features map based on the environment
+    # variables.
+    def __init__(self, env: e3.env.Env, test_env: Dict[str, Any]):
+        super().__init__(env, test_env)
+
+        # Prepare the features map based on the environment variables
+        self.skip = {}
+
+        for modifier, env_var in BaseDriver.MODIFIERS.items():
+            affirming = "ENABLE" in env_var
+            key = f"skip_{modifier}"
+
+            if env_var in os.environ:
+                self.skip[key] = not affirming
+            else:
+                self.skip[key] = affirming
+
+        # Hardcode OSes
+        for osname in ["windows", "linux", "macos"]:
+            self.skip[f"skip_{osname}"] = not getattr(drivers.helpers, f"on_{osname}")()
+
+    @property
+    def test_control_creator(self):
+        # Give our custom dictionary to the control filter
+        return YAMLTestControlCreator(self.skip)

testsuite/drivers/driver/docker_nested.py

@@ -10,7 +10,7 @@
 
 from drivers import alr
 from drivers.alr import run_alr
-from drivers.helpers import MODIFIERS
+from drivers.driver.base_driver import BaseDriver
 
 
 def main():
@@ -36,7 +36,7 @@ def main():
     # Set up the environment
 
     # Copy any received modifiers
-    for modifier in [m for m in MODIFIERS if m in test_env]:
+    for modifier in [m for m in BaseDriver.MODIFIERS.items() if m in test_env]:
         os.environ[modifier] = test_env[modifier]
 
     # alr path

testsuite/drivers/driver/docker_wrapper.py

@@ -5,9 +5,10 @@
 from importlib import import_module
 from typing import Tuple
 
-from drivers.helpers import FileLock, on_linux, MODIFIERS
-from e3.testsuite.driver.classic import (ClassicTestDriver,
-                                         TestAbortWithFailure,
+from drivers.driver.base_driver import BaseDriver
+from drivers.helpers import FileLock, on_linux
+
+from e3.testsuite.driver.classic import (TestAbortWithFailure,
                                          TestSkip)
 
 DOCKERFILE = "Dockerfile"
@@ -88,7 +89,7 @@ def build_image() -> None:
             labels={LABEL_HASH : compute_dockerfile_hash()})
 
 
-class DockerWrapperDriver(ClassicTestDriver):
+class DockerWrapperDriver(BaseDriver):
 
     # This is a workaround for Windows, where attempting to use rlimit by e3-core
     # causes permission errors. TODO: remove once e3-core has a proper solution.
@@ -104,7 +105,7 @@ def run(self):
 
         # Augment the test environment with local modifiers that could
         # impact the wrapped test, if defined
-        for modifier in [m for m in MODIFIERS if m in os.environ]:
+        for modifier in [val for _, val in self.MODIFIERS.items() if val in os.environ]:
             self.test_env[modifier] = os.environ[modifier]
 
         # Run our things

testsuite/drivers/driver/python_script.py

@@ -4,12 +4,13 @@
 import sys
 
 from drivers.alr import prepare_env, prepare_indexes, run_alr
-from e3.testsuite.driver.classic import (ClassicTestDriver,
-                                         TestAbortWithFailure,
+from drivers.driver.base_driver import BaseDriver
+
+from e3.testsuite.driver.classic import (TestAbortWithFailure,
                                          TestSkip)
 
 
-class PythonScriptDriver(ClassicTestDriver):
+class PythonScriptDriver(BaseDriver):
     """
     Test driver to run a "test.py" Python script.
 
@@ -142,6 +143,9 @@ def run(self):
                                  self.test_env.get('build-mode',
                                                    DEFAULT_MODE))
         # One of 'shared', 'sandboxed', or 'both'
+        if mode not in ["shared", "sandboxed", "both"]:
+            raise ValueError(f"Invalid build mode: {mode}, must be one of "
+                             "'shared', 'sandboxed', or 'both'")
 
         # If mode is "both", track original files for later
         if mode == "both":

testsuite/drivers/helpers.py

@@ -11,12 +11,6 @@
 from subprocess import run
 from zipfile import ZipFile
 
-# Environment variables that can be used to modify the testsuite behavior
-MODIFIERS : list = [
-      'ALIRE_DISABLE_DISTRO'
-    , 'ALIRE_DISABLE_DOCKER'
-    , 'ALIRE_DISABLE_NETWORK_TESTS'
-]
 
 # Return the entries (sorted) under a given folder, both folders and files
 # Optionally, return only those matching regex. Uses '/' always as separator.

testsuite/run-dev.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# This script is used to run the testsuite with some extra tests enabled,
+# intended only for the main developers in their local machines.
+
+export ALR_TESTSUITE_ALLOW=1        # So `alr` doesn't raise
+export ALIRE_ENABLE_LOCAL_TESTS=1   # So they're actually run
+
+clear
+./run.py -M1 "$@"

testsuite/run.py

@@ -22,9 +22,11 @@
 
 class Testsuite(e3.testsuite.Testsuite):
     tests_subdir = 'tests'
+
+    # Available drivers
     test_driver_map = {
-        'python-script': PythonScriptDriver,
-        'docker-wrapper': DockerWrapperDriver
+        'python-script'  : PythonScriptDriver,
+        'docker-wrapper' : DockerWrapperDriver
         }
 
     def add_options(self, parser):

testsuite/skels/no-index/test.yaml

@@ -1,7 +1,14 @@
 driver: python-script
 build_mode: both  # one of shared, sandboxed, both (default)
+control: # Used to disable test depending on one of:
+    - [SKIP, "skip_distro",  "Unknown distro testing disabled"]
+    - [SKIP, "skip_docker",  "Docker-hosted tests disabled"]
+    - [SKIP, "skip_network", "Network-requiring tests disabled"]
+    - [SKIP, "skip_linux",   "Test is Linux-only"]
+    - [SKIP, "skip_macos",   "Test is macOS-only"]
+    - [SKIP, "skip_windows", "Test is Windows-only"]
 indexes:
     compiler_only_index: {}
     # Note that shared builds require a detected compiler to be able to compute
     # build hashes, which is needed for many subcommands: build, get, printenv,
-    # update...
+    # update...

testsuite/tests/dockerized/misc/default-cache/test.yaml

@@ -1,4 +1,6 @@
 driver: docker-wrapper
+control:
+    - [SKIP, "skip_docker", "Docker disabled"]
 indexes:
     toolchain_index:
         copy_crates_src: True

testsuite/tests/dockerized/misc/failed-auto-update/test.yaml

@@ -1,2 +1,5 @@
 driver: docker-wrapper
+control:
+    - [SKIP, "skip_docker",  "Docker disabled"]
+    - [SKIP, "skip_network", "Network disabled"]
 indexes: {}

testsuite/tests/index/external-msys2/test.yaml

@@ -1,4 +1,6 @@
 driver: python-script
+control:
+    - [SKIP, "skip_windows", "Windows-only"]
 indexes:
     my_index:
         in_fixtures: false