diff --git a/src/alire/alire-environment.ads b/src/alire/alire-environment.ads index 778e33d16..9b5d6ed97 100644 --- a/src/alire/alire-environment.ads +++ b/src/alire/alire-environment.ads @@ -17,6 +17,10 @@ package Alire.Environment with Preelaborate is Testsuite : constant String := "ALR_TESTSUITE"; -- If defined, we are running under the testsuite harness + Testsuite_Allow : constant String := "ALR_TESTSUITE_ALLOW"; + -- If defined, we want to allow operations normally disabled forbidden + -- during testsuite runs, like creating a PR in a public server. + Traceback : constant String := "ALR_TRACEBACK_ENABLED"; -- If set to True/1, dump unexpected exceptions to console (same as `-d`) diff --git a/src/alire/alire-publish.adb b/src/alire/alire-publish.adb index 9aa86ca29..84512faf1 100644 --- a/src/alire/alire-publish.adb +++ b/src/alire/alire-publish.adb @@ -560,8 +560,11 @@ package body Alire.Publish is -- more generic message otherwise (when lacking a github login). if not Context.Options.Skip_Submit then - -- Safeguard to avoid tests creating a live pull request - if OS_Lib.Getenv (Environment.Testsuite, "unset") /= "unset" then + -- Safeguard to avoid tests creating a live pull request, unless + -- explicitly desired + if OS_Lib.Getenv (Environment.Testsuite, "unset") /= "unset" + and then OS_Lib.Getenv (Environment.Testsuite_Allow, "unset") = "unset" + then raise Program_Error with "Attempting to go online to create a PR during tests"; end if; diff --git a/testsuite/README.md b/testsuite/README.md index 484ce4c56..d56205ec2 100644 --- a/testsuite/README.md +++ b/testsuite/README.md @@ -57,6 +57,9 @@ no matter their value, or lack of one. - `ALIRE_DISABLE_NETWORK_TESTS`: when defined, tests that require non-local network use will be skipped. +- `ALIRE_ENABLE_LOCAL_TESTS`: when defined, tests that are intended to be run + locally only by the Alire developer team will not be skipped. + Example disabling Docker tests for a single run on Bash: ```Bash $ ALIRE_DISABLE_DOCKER= ./run.sh diff --git a/testsuite/drivers/helpers.py b/testsuite/drivers/helpers.py index b89ff9c1a..6005aca4b 100644 --- a/testsuite/drivers/helpers.py +++ b/testsuite/drivers/helpers.py @@ -177,6 +177,14 @@ def git_blast(path): shutil.rmtree(path) +def git_init_user(): + """ + Initialize git user and email + """ + run(["git", "config", "user.email", "alr@testing.com"]).check_returncode() + run(["git", "config", "user.name", "Alire Testsuite"]).check_returncode() + + def init_git_repo(path): """ Initialize and commit everything inside a folder, returning the HEAD commit @@ -187,10 +195,7 @@ def init_git_repo(path): # You might think to init with --initial-branch=master, but # e.g. Centos's git doesn't support this. assert run(["git", "checkout", "-b", "master"]).returncode == 0 - assert run(["git", "config", "user.email", "alr@testing.com"]) \ - .returncode == 0 - assert run(["git", "config", "user.name", "Alire Testsuite"]) \ - .returncode == 0 + git_init_user() # Workaround for Windows, where somehow we get undeletable files in temps: with open(".gitignore", "wt") as file: diff --git a/testsuite/run-dev.sh b/testsuite/run-dev.sh new file mode 100755 index 000000000..e422fbc90 --- /dev/null +++ b/testsuite/run-dev.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# This script is used to run the testsuite with some extra tests enabled, +# intended only for the main developers in their local machines. + +export ALR_TESTSUITE_ALLOW=1 # So `alr` doesn't raise +export ALIRE_ENABLE_LOCAL_TESTS=1 # So they're actually run + +clear +./run.py -M1 "$@" diff --git a/testsuite/skels/no-index/test.yaml b/testsuite/skels/no-index/test.yaml index a4f7b6b29..97d4f3d5c 100644 --- a/testsuite/skels/no-index/test.yaml +++ b/testsuite/skels/no-index/test.yaml @@ -3,6 +3,7 @@ build_mode: both # one of shared, sandboxed, both (default) control: # Used to disable test depending on one of: - [SKIP, "skip_distro", "Unknown distro testing disabled"] - [SKIP, "skip_docker", "Docker-hosted tests disabled"] + - [SKIP, "skip_local", "Local developer-only tests disabled"] - [SKIP, "skip_network", "Network-requiring tests disabled"] - [SKIP, "skip_linux", "Test is Linux-only"] - [SKIP, "skip_macos", "Test is macOS-only"] diff --git a/testsuite/tests/publish/submit-request-cancel/test.py b/testsuite/tests/publish/submit-request-cancel/test.py new file mode 100644 index 000000000..3154fa367 --- /dev/null +++ b/testsuite/tests/publish/submit-request-cancel/test.py @@ -0,0 +1,106 @@ +""" +A complete online test of submitting a release, requesting a review, and +cancelling the request. +""" + +import os +import random +import re +import subprocess +import sys +import time +from drivers.alr import run_alr +from drivers.helpers import git_init_user + +# This test is a bit special, as it is heavy on networking and complex on +# automation. Not intended to be run on CI, but rather on the developer's local +# machine. We also require having the `gh` command installed and configured, +# GH_TOKEN set to a valid token, and GH_USERNAME set to a valid username. + +# IMPORTANT: if the test fails midway with the PR open, you must manually close +# the PR at https://github.com/alire-project/test-index/pulls + +# Detect gh via gh --version +if subprocess.run(["gh", "--version"]).returncode != 0: + print("SKIP: `gh` not installed") + sys.exit(0) + +# Detect GH_TOKEN +if os.environ.get("GH_TOKEN", "") == "": + print("SKIP: GH_TOKEN not set") + sys.exit(0) + +# Detect GH_USERNAME +if os.environ.get("GH_USERNAME", "") == "": + print("SKIP: GH_USERNAME not set") + sys.exit(0) +else: + run_alr("config", "--global", "--set", + "user.github_login", os.environ["GH_USERNAME"]) + +# Configure the testing remote index +run_alr("config", "--global", "--set", "index.repository_name", "test-index") + +# Clone a simple crate not already in the index with a local remote +subprocess.run(["gh", "repo", "clone", + "alire-project/hello", "hello_upstream", + "--", "--bare"]).check_returncode() +subprocess.run(["git", "clone", "hello_upstream", "hello"]).check_returncode() +os.chdir("hello") + +# To allow eventual concurrent testing, modify the crate version to a random +# one. We need to replace the version in alire.toml only. To avoid troubles +# with line endings, we edit in binary mode. +with open("alire.toml", "rb") as f: + # Prepare the new random version number using random number generation + new_version = f"{random.randint(1, 99999999)}" + # Read the file + lines = f.read() + # Identify the tester for minimum traceability (username and hostname) + tester = os.environ["GH_USERNAME"] + "@" + os.uname()[1] + # Find and replace the line with 'version = "*"', a regex is enough + new_lines = re.sub(rb'version = "[^"]*"', + f'version = "0.1.0-{new_version}+autotest_by_{tester}"'.encode(), + lines) + # Write back the file + with open("alire.toml", "wb") as f: + f.write(new_lines) + +# Commit changes +git_init_user() +subprocess.run(["git", "commit", "-a", "-m 'Unique version'"]).check_returncode() +subprocess.run(["git", "push"]).check_returncode() + +# Publish; we need to force to skip the check that the source is remote +p = run_alr("publish", "--skip-build", quiet=False, force=True) + +# Identify the PR number just created in a message like: +# Visit https://github.com/alire-project/test-index/pull/7 for details +# We use a regex to extract the number: +pr = re.search(r'/pull/(\d+) for details', p.out).group(1) + +# Wait for the checks to complete. In the test index, there is only a mock test +# that always succeeds quickly. This allows us to check the status command too. +waited = 0 +timeout = 30 +while True: + p = run_alr("publish", "--status") + line = [l for l in p.out.splitlines() if pr == l.split()[0]][0].lower() + if "checks_passed" in line: + break + elif "checks_failed" in line: + assert False, f"Checks failed unexpectedly for pr {pr}: {p.out}" + elif waited > timeout: + assert False, f"Checks not completed after {timeout} seconds for pr {pr}" + else: + # Wait a bit and retry, but fail after so many time + time.sleep(1) + waited += 1 + +# Request a review +run_alr("publish", f"--request-review={pr}") + +# Cancel the PR to leave things as they were before the test +run_alr("publish", f"--cancel={pr}", "--reason='automated testing'") + +print("SUCCESS") diff --git a/testsuite/tests/publish/submit-request-cancel/test.yaml b/testsuite/tests/publish/submit-request-cancel/test.yaml new file mode 100644 index 000000000..43ea06963 --- /dev/null +++ b/testsuite/tests/publish/submit-request-cancel/test.yaml @@ -0,0 +1,7 @@ +driver: python-script +build_mode: both +control: + - [SKIP, "skip_local", "Local tests disabled"] + - [SKIP, "skip_network", "Network-requiring tests disabled"] +indexes: + compiler_only_index: {} \ No newline at end of file