Docker driver for tests in default configuration (#1358)

.vscode/launch.json

@@ -0,0 +1,28 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        },
+        {
+            "name": "Python: run.py",
+            "type": "python",
+            "request": "launch",
+            "cwd": "${workspaceFolder}/testsuite",
+            "program": "run.py",
+            "args": [
+                "default-cache" // Replace with the test you want to debug
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

scripts/ci-github.sh

@@ -14,7 +14,7 @@ pushd $( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 popd
 
 # Build alr
-export ALIRE_OS=$(get_OS) 
+export ALIRE_OS=$(get_OS)
 gprbuild -j0 -p -P alr_env
 
 # Disable distro detection if supported
@@ -63,7 +63,7 @@ fi
 echo Python version: $($run_python --version)
 echo Pip version: $($run_pip --version)
 
-$run_pip install --upgrade e3-testsuite
+$run_pip install --upgrade -r requirements.txt
 echo Python search paths:
 $run_python -c "import sys; print('\n'.join(sys.path))"
 

testsuite/Dockerfile

@@ -0,0 +1,11 @@
+# This docker image is used in tests with the `docker_wrapper` driver.
+
+FROM alire/gnat:ubuntu-lts
+
+RUN useradd -m -s /bin/bash user && \
+    chown user:user /home/user
+
+RUN pip3 install e3-testsuite
+
+WORKDIR /testsuite
+USER user

testsuite/drivers/alr.py

@@ -44,7 +44,7 @@ def prepare_env(config_dir, env):
     mkdir(config_dir)
     env['ALR_CONFIG'] = config_dir
     #  We pass config location explicitly in the following calls since env is
-    #  not yet applied.
+    #  not yet applied (it's just a dict to be passed later to subprocess)
 
     # Disable autoconfig of the community index, to prevent unintended use of
     # it in tests, besides the overload of fetching it

testsuite/drivers/asserts.py

@@ -45,7 +45,7 @@ def assert_eq(expected, actual, label=None):
 
 def assert_contents(dir: str, expected, regex: str = ""):
     """
-    Check that entries in dir filtered by regex match the list in contents
+    Check that entries in dir filtered by regex match the list in expected
     """
     real = contents(dir, regex)
     assert real == expected, \

testsuite/drivers/driver/docker_nested.py

@@ -0,0 +1,76 @@
+"""
+This is run inside a docker container, so we aren't in the context of e3. We
+prepare the environment and run the test script directly.
+"""
+
+import json
+import os
+import subprocess
+import sys
+
+from drivers import alr
+from drivers.alr import run_alr
+
+
+def main():
+    testsuite_root = "/testsuite"  # Must match docker volume mount
+    home = os.environ["HOME"]
+    work_dir = "/tmp/test"
+
+    # We receive the test environment prepared by e3 as JSON via environment.
+    # With this test_env dictionary we are mostly in the same situation as in
+    # the Python driver.
+    test_env = json.loads(os.environ['ALIRE_TEST_ENV'])
+
+    # Find the test sources inside docker. The test_env we got still has the
+    # host paths.
+
+    test_dir = test_env['test_dir']  # The test source dir to find test.py
+    # Strip the prefix that comes from the host filesystem
+    test_dir = test_dir[test_dir.find(testsuite_root):]
+
+    # Create a pristine folder for the test to run in
+    os.mkdir(work_dir)
+
+    # Set up the environment
+
+    # alr path
+    os.environ["ALR_PATH"] = "/usr/bin/alr" # Must match docker volume mount
+
+    # Disable autoconfig of the community index, to prevent unintended use
+    run_alr("config", "--global", "--set", "index.auto_community", "false")
+
+    # Disable selection of toolchain. Tests that
+    # require a configured compiler will have to set it up explicitly.
+    run_alr("toolchain", "--disable-assistant")
+
+    # Disable warning on old index, to avoid having to update index versions
+    # when they're still compatible.
+    run_alr("config", "--global", "--set", "warning.old_index", "false")
+
+    # indexes to use
+    if 'indexes' in test_env:
+        alr.prepare_indexes(
+            config_dir=home + "/.config/alire",
+            working_dir=work_dir,
+            index_descriptions=test_env.get('indexes', {}))
+
+    # Give access to our Python helpers
+    env = os.environ.copy()
+    python_path = env.get('PYTHONPATH', '')
+    env['PYTHONPATH'] = python_path  # Ensure it exists, even if empty
+    env['PYTHONPATH'] += os.path.pathsep + testsuite_root  # And add ours
+
+    # Run the test
+    try:
+        subprocess.run(["python3", test_dir + "/test.py"],
+                       cwd=work_dir, env=env, check=True)
+    except:
+        # No need to let the exception to muddle the reporting, the output of
+        # the test with any exception is already in the log that the
+        # docker_wrapper will print.
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

testsuite/drivers/driver/docker_wrapper.py

@@ -0,0 +1,137 @@
+import hashlib
+import json
+import os
+import subprocess
+from importlib import import_module
+from typing import Tuple
+
+from drivers.helpers import FileLock, on_linux
+from e3.testsuite.driver.classic import (ClassicTestDriver,
+                                         TestAbortWithFailure,
+                                         TestSkip)
+
+DOCKERFILE = "Dockerfile"
+PLACEHOLDER = "TO_BE_COMPUTED"
+TAG = "alire_testsuite"
+ENV_FLAG = "ALIRE_DOCKER_ENABLED" # Set to "False" to disable docker tests
+LABEL_HASH = "hash"
+
+
+def is_docker_available() -> bool:
+    # Restrict docker testing only to Linux
+    if not on_linux():
+        return False
+
+    # Detect explicitly disabled
+    if os.environ.get(ENV_FLAG, "True").lower() in ["0", "false"]:
+        return False
+
+    # Detect python library
+    try:
+        import_module("docker")
+    except ImportError:
+        return False
+
+    # Detect executable
+    try:
+        subprocess.run(["docker", "--version"], check=True,
+                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return False
+
+    return True
+
+
+def get_client():
+    if is_docker_available():
+        import docker
+        return docker.from_env()
+    else:
+        return None
+
+
+def compute_dockerfile_hash() -> str:
+    with open(DOCKERFILE, 'r') as file:
+        content = file.read()
+    return hashlib.sha256(content.encode('utf-8')).hexdigest()
+
+
+def already_built() -> Tuple[str, str]:
+    file_hash = compute_dockerfile_hash()
+
+    # Check existing images
+    client = get_client()
+    for image in client.images.list():
+        if LABEL_HASH in image.labels and image.labels[LABEL_HASH] == file_hash:
+            return image, file_hash
+
+    return None, file_hash
+
+
+def build_image() -> None:
+    # We need to lock here as multiple docker tests might attempt to create the
+    # image at the same time
+    with FileLock("/tmp/alire_testsuite_docker.lock"):
+        image, file_hash = already_built()
+
+        if image:
+            return
+
+        # Do the actual build
+        get_client().images.build(
+            path="..", dockerfile=f"testsuite/{DOCKERFILE}", rm=True, tag=TAG,
+            labels={LABEL_HASH : compute_dockerfile_hash()})
+
+
+class DockerWrapperDriver(ClassicTestDriver):
+
+    # This is a workaround for Windows, where attempting to use rlimit by e3-core
+    # causes permission errors. TODO: remove once e3-core has a proper solution.
+    @property
+    def default_process_timeout(self):
+        return None
+
+    def run(self):
+        if not is_docker_available():
+            raise TestSkip('Docker testing is disabled or not available')
+
+        build_image()
+
+        # Run our things
+        try:
+            container = get_client().containers.run(
+                # Regular image launching
+                image=TAG, tty=True, stdin_open=True, detach=True,
+
+                # Pass the test environment to the container as JSON
+                environment={"ALIRE_TEST_ENV": json.dumps(self.test_env)},
+
+                # Bind the testsuite directory as read-only and the alr executable
+                volumes={ os.path.abspath(".") : { "bind": "/testsuite", "mode": "ro" }
+                          , os.path.abspath("..") + "/bin/alr" : { "bind": "/usr/bin/alr", "mode": "ro" }
+                         },
+
+                # In the container, launch the script that will setup the test
+                command=
+                    "/bin/python3 -c 'from drivers.driver import docker_nested;"
+                    "docker_nested.main()'")
+
+            # Wait for the container to finish and retrieve its output
+            result = container.wait()
+            output = container.logs().decode()
+
+            if (code := result["StatusCode"]) != 0:
+                self.result.log += f'Docker command failed with exit code {code} and output:\n{output}'
+                raise TestAbortWithFailure(
+                    f"Docker command failed with exit code {code}")
+
+        finally:
+            # Don't leave dead containers around
+            if 'container' in locals() and container:
+                container.remove()
+
+            # Check that the test succeeded inside the docker container
+            out_lines = output.splitlines()
+            if not out_lines or out_lines[-1] != 'SUCCESS':
+                self.result.log += f'missing SUCCESS output line:\n{output}'
+                raise TestAbortWithFailure('missing SUCCESS output line')

testsuite/drivers/python_script.py → testsuite/drivers/driver/python_script.py

@@ -40,8 +40,8 @@ def run(self):
 
         # Also give it access to our Python helpers
         python_path = env.get('PYTHONPATH', '')
-        path_for_drivers = os.path.abspath(
-            os.path.dirname(os.path.dirname(__file__)))
+        parent = os.path.dirname
+        path_for_drivers = os.path.abspath(parent(parent(parent(__file__))))
         env['PYTHONPATH'] = '{}{}{}'.format(
             path_for_drivers, os.path.pathsep, python_path
         ) if python_path else path_for_drivers

testsuite/drivers/helpers.py

@@ -64,6 +64,10 @@ def check_line_in(filename, line):
                 repr(line), filename, content_of(filename))
 
 
+def on_linux():
+    return platform.system() == "Linux"
+
+
 def on_macos():
     return platform.system() == "Darwin"
 
@@ -240,3 +244,31 @@ def replace_in_file(filename : str, old : str, new : str):
     old_contents = content_of(filename)
     with open(filename, "wt") as file:
         file.write(old_contents.replace(old, new))
+
+
+class FileLock():
+    """
+    A filesystem-level lock for tests executed from different threads but
+    without shared memory space. Only used on Linux.
+    """
+    def __init__(self, lock_file_path):
+        if not on_linux():
+            raise Exception("FileLock is only supported on Linux")
+
+        self.lock_file_path = lock_file_path
+
+    def __enter__(self):
+        # Create the lock file if it doesn't exist
+        open(self.lock_file_path, 'a').close()
+
+        # Reopen in read mode
+        self.lock_file = open(self.lock_file_path, 'r')
+        # Acquire the file lock or wait for it
+        import fcntl
+        fcntl.flock(self.lock_file.fileno(), fcntl.LOCK_EX)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Release the file lock
+        import fcntl
+        fcntl.flock(self.lock_file.fileno(), fcntl.LOCK_UN)
+        self.lock_file.close()

testsuite/requirements.txt

@@ -0,0 +1,2 @@
+docker
+e3-testsuite

testsuite/run.py

@@ -18,12 +18,16 @@
 from e3.testsuite.result import TestStatus
 
 from drivers.helpers import on_windows
-from drivers.python_script import PythonScriptDriver
+from drivers.driver.python_script import PythonScriptDriver
+from drivers.driver.docker_wrapper import DockerWrapperDriver
 
 
 class Testsuite(e3.testsuite.Testsuite):
     tests_subdir = 'tests'
-    test_driver_map = {'python-script': PythonScriptDriver}
+    test_driver_map = {
+        'python-script': PythonScriptDriver,
+        'docker-wrapper': DockerWrapperDriver
+        }
 
     def add_options(self, parser):
         super().add_options(parser)

testsuite/tests/dockerized/misc/default-cache/test.py

@@ -0,0 +1,22 @@
+"""
+Check inside a pristine environment that the default cache is located where
+it should.
+"""
+
+import os
+
+from drivers.alr import alr_with, init_local_crate
+
+# Forcing the deployment of a binary crate triggers the use of the global
+# cache, which should be created at the expected location.
+init_local_crate()
+alr_with("gnat_native")
+
+home = os.environ["HOME"]
+
+assert \
+    os.path.isdir(f"{home}/.cache/alire/dependencies/gnat_native_8888.0.0_99fa3a55"), \
+    "Default cache not found at the expected location"
+
+
+print('SUCCESS')

testsuite/tests/dockerized/misc/default-cache/test.yaml

@@ -0,0 +1,4 @@
+driver: docker-wrapper
+indexes:
+    toolchain_index:
+        copy_crates_src: True