Test for automated publishing

This test is dependent on a remote repo (`test-index`) on which a PR is opened
during testing, so we better run this one sporadically and locally, or in a
single selected CI configuration.

src/alire/alire-environment.ads

@@ -17,6 +17,10 @@ package Alire.Environment with Preelaborate is
    Testsuite : constant String := "ALR_TESTSUITE";
    --  If defined, we are running under the testsuite harness
 
+   Testsuite_Allow : constant String := "ALR_TESTSUITE_ALLOW";
+   --  If defined, we want to allow operations normally disabled forbidden
+   --  during testsuite runs, like creating a PR in a public server.
+
    Traceback : constant String := "ALR_TRACEBACK_ENABLED";
    --  If set to True/1, dump unexpected exceptions to console (same as `-d`)
 

src/alire/alire-publish.adb

@@ -560,8 +560,11 @@ package body Alire.Publish is
          --  more generic message otherwise (when lacking a github login).
 
          if not Context.Options.Skip_Submit then
-            --  Safeguard to avoid tests creating a live pull request
-            if OS_Lib.Getenv (Environment.Testsuite, "unset") /= "unset" then
+            --  Safeguard to avoid tests creating a live pull request, unless
+            --  explicitly desired
+            if OS_Lib.Getenv (Environment.Testsuite, "unset") /= "unset"
+              and then OS_Lib.Getenv (Environment.Testsuite_Allow, "unset") = "unset"
+            then
                raise Program_Error
                  with "Attempting to go online to create a PR during tests";
             end if;

testsuite/README.md

@@ -57,6 +57,9 @@ no matter their value, or lack of one.
 - `ALIRE_DISABLE_NETWORK_TESTS`: when defined, tests that
   require non-local network use will be skipped.
 
+- `ALIRE_ENABLE_LOCAL_TESTS`: when defined, tests that are intended to be run
+  locally only by the Alire developer team will not be skipped.
+
 Example disabling Docker tests for a single run on Bash:
 ```Bash
 $ ALIRE_DISABLE_DOCKER= ./run.sh

testsuite/drivers/helpers.py

@@ -177,6 +177,14 @@ def git_blast(path):
     shutil.rmtree(path)
 
 
+def git_init_user():
+    """
+    Initialize git user and email
+    """
+    run(["git", "config", "user.email", "[email protected]"]).check_returncode()
+    run(["git", "config", "user.name", "Alire Testsuite"]).check_returncode()
+
+
 def init_git_repo(path):
     """
     Initialize and commit everything inside a folder, returning the HEAD commit
@@ -187,10 +195,7 @@ def init_git_repo(path):
     # You might think to init with --initial-branch=master, but
     # e.g. Centos's git doesn't support this.
     assert run(["git", "checkout", "-b", "master"]).returncode == 0
-    assert run(["git", "config", "user.email", "[email protected]"]) \
-        .returncode == 0
-    assert run(["git", "config", "user.name", "Alire Testsuite"]) \
-        .returncode == 0
+    git_init_user()
 
     # Workaround for Windows, where somehow we get undeletable files in temps:
     with open(".gitignore", "wt") as file:

testsuite/run-dev.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# This script is used to run the testsuite with some extra tests enabled,
+# intended only for the main developers in their local machines.
+
+export ALR_TESTSUITE_ALLOW=1        # So `alr` doesn't raise
+export ALIRE_ENABLE_LOCAL_TESTS=1   # So they're actually run
+
+clear
+./run.py -M1 "$@"

testsuite/skels/no-index/test.yaml

@@ -3,6 +3,7 @@ build_mode: both  # one of shared, sandboxed, both (default)
 control: # Used to disable test depending on one of:
     - [SKIP, "skip_distro",  "Unknown distro testing disabled"]
     - [SKIP, "skip_docker",  "Docker-hosted tests disabled"]
+    - [SKIP, "skip_local",   "Local developer-only tests disabled"]
     - [SKIP, "skip_network", "Network-requiring tests disabled"]
     - [SKIP, "skip_linux",   "Test is Linux-only"]
     - [SKIP, "skip_macos",   "Test is macOS-only"]

testsuite/tests/publish/submit-request-cancel/test.py

@@ -0,0 +1,106 @@
+"""
+A complete online test of submitting a release, requesting a review, and
+cancelling the request.
+"""
+
+import os
+import random
+import re
+import subprocess
+import sys
+import time
+from drivers.alr import run_alr
+from drivers.helpers import git_init_user
+
+# This test is a bit special, as it is heavy on networking and complex on
+# automation. Not intended to be run on CI, but rather on the developer's local
+# machine. We also require having the `gh` command installed and configured,
+# GH_TOKEN set to a valid token, and GH_USERNAME set to a valid username.
+
+# IMPORTANT: if the test fails midway with the PR open, you must manually close
+# the PR at https://github.com/alire-project/test-index/pulls
+
+# Detect gh via gh --version
+if subprocess.run(["gh", "--version"]).returncode != 0:
+    print("SKIP: `gh` not installed")
+    sys.exit(0)
+
+# Detect GH_TOKEN
+if os.environ.get("GH_TOKEN", "") == "":
+    print("SKIP: GH_TOKEN not set")
+    sys.exit(0)
+
+# Detect GH_USERNAME
+if os.environ.get("GH_USERNAME", "") == "":
+    print("SKIP: GH_USERNAME not set")
+    sys.exit(0)
+else:
+    run_alr("config", "--global", "--set",
+            "user.github_login", os.environ["GH_USERNAME"])
+
+# Configure the testing remote index
+run_alr("config", "--global", "--set", "index.repository_name", "test-index")
+
+# Clone a simple crate not already in the index with a local remote
+subprocess.run(["gh", "repo", "clone",
+                "alire-project/hello", "hello_upstream",
+                "--", "--bare"]).check_returncode()
+subprocess.run(["git", "clone", "hello_upstream", "hello"]).check_returncode()
+os.chdir("hello")
+
+# To allow eventual concurrent testing, modify the crate version to a random
+# one. We need to replace the version in alire.toml only. To avoid troubles
+# with line endings, we edit in binary mode.
+with open("alire.toml", "rb") as f:
+    # Prepare the new random version number using random number generation
+    new_version = f"{random.randint(1, 99999999)}"
+    # Read the file
+    lines = f.read()
+    # Identify the tester for minimum traceability (username and hostname)
+    tester = os.environ["GH_USERNAME"] + "@" + os.uname()[1]
+    # Find and replace the line with 'version = "*"', a regex is enough
+    new_lines = re.sub(rb'version = "[^"]*"',
+                       f'version = "0.1.0-{new_version}+autotest_by_{tester}"'.encode(),
+                       lines)
+    # Write back the file
+    with open("alire.toml", "wb") as f:
+        f.write(new_lines)
+
+# Commit changes
+git_init_user()
+subprocess.run(["git", "commit", "-a", "-m 'Unique version'"]).check_returncode()
+subprocess.run(["git", "push"]).check_returncode()
+
+# Publish; we need to force to skip the check that the source is remote
+p = run_alr("publish", "--skip-build", quiet=False, force=True)
+
+# Identify the PR number just created in a message like:
+# Visit https://github.com/alire-project/test-index/pull/7 for details
+# We use a regex to extract the number:
+pr = re.search(r'/pull/(\d+) for details', p.out).group(1)
+
+# Wait for the checks to complete. In the test index, there is only a mock test
+# that always succeeds quickly. This allows us to check the status command too.
+waited  = 0
+timeout = 30
+while True:
+    p = run_alr("publish", "--status")
+    line = [l for l in p.out.splitlines() if pr == l.split()[0]][0].lower()
+    if "checks_passed" in line:
+        break
+    elif "checks_failed" in line:
+        assert False, f"Checks failed unexpectedly for pr {pr}: {p.out}"
+    elif waited > timeout:
+        assert False, f"Checks not completed after {timeout} seconds for pr {pr}"
+    else:
+        # Wait a bit and retry, but fail after so many time
+        time.sleep(1)
+        waited += 1
+
+# Request a review
+run_alr("publish", f"--request-review={pr}")
+
+# Cancel the PR to leave things as they were before the test
+run_alr("publish", f"--cancel={pr}", "--reason='automated testing'")
+
+print("SUCCESS")

testsuite/tests/publish/submit-request-cancel/test.yaml

@@ -0,0 +1,7 @@
+driver: python-script
+build_mode: both
+control:
+    - [SKIP, "skip_local", "Local tests disabled"]
+    - [SKIP, "skip_network", "Network-requiring tests disabled"]
+indexes:
+    compiler_only_index: {}