From b7ab5377da6c57f4ff64b38d8d7c5a945fb21b33 Mon Sep 17 00:00:00 2001
From: David Pacheco <dap@oxidecomputer.com>
Date: Wed, 28 Aug 2024 17:15:36 -0700
Subject: [PATCH] build end-to-end tests and live tests by default (#6469)

---
 .config/nextest.toml                        | 16 +++++++++++-----
 .github/buildomat/build-and-test.sh         |  2 +-
 Cargo.toml                                  |  6 ++----
 dev-tools/xtask/src/check_workspace_deps.rs |  4 ----
 dev-tools/xtask/src/live_tests.rs           |  5 ++++-
 live-tests/README.adoc                      |  2 +-
 6 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/.config/nextest.toml b/.config/nextest.toml
index 67cabe9f0e..eb62e1c9af 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -3,7 +3,7 @@
 #
 # The required version should be bumped up if we need new features, performance
 # improvements or bugfixes that are present in newer versions of nextest.
-nextest-version = { required = "0.9.64", recommended = "0.9.70" }
+nextest-version = { required = "0.9.77", recommended = "0.9.77" }
 
 experimental = ["setup-scripts"]
 
@@ -35,14 +35,13 @@ clickhouse-cluster = { max-threads = 1 }
 # behaviors that conflict with each other.  They need to be run serially.
 live-tests = { max-threads = 1 }
 
+[profile.default]
+default-filter = 'all() - package(omicron-live-tests) - package(end-to-end-tests)'
+
 [[profile.default.overrides]]
 filter = 'package(oximeter-db) and test(replicated)'
 test-group = 'clickhouse-cluster'
 
-[[profile.default.overrides]]
-filter = 'package(omicron-live-tests)'
-test-group = 'live-tests'
-
 [[profile.default.overrides]]
 # These tests can time out under heavy contention.
 filter = 'binary_id(omicron-nexus::test_all) and test(::schema::)'
@@ -53,3 +52,10 @@ filter = 'binary_id(omicron-nexus::test_all)'
 # As of 2023-01-08, the slowest test in test_all takes 196s on a Ryzen 7950X.
 # 900s is a good upper limit that adds a comfortable buffer.
 slow-timeout = { period = '60s', terminate-after = 15 }
+
+[profile.live-tests]
+default-filter = 'package(omicron-live-tests)'
+
+[[profile.live-tests.overrides]]
+filter = 'package(omicron-live-tests)'
+test-group = 'live-tests'
diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh
index 50bbd8194d..1980012664 100755
--- a/.github/buildomat/build-and-test.sh
+++ b/.github/buildomat/build-and-test.sh
@@ -9,7 +9,7 @@ target_os=$1
 # NOTE: This version should be in sync with the recommended version in
 # .config/nextest.toml. (Maybe build an automated way to pull the recommended
 # version in the future.)
-NEXTEST_VERSION='0.9.70'
+NEXTEST_VERSION='0.9.77'
 
 cargo --version
 rustc --version
diff --git a/Cargo.toml b/Cargo.toml
index f6238a48cb..3d1d19fa65 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -159,8 +159,7 @@ default-members = [
     # See omicron#4392.
     "dns-server",
     "dns-server-api",
-    # Do not include end-to-end-tests in the list of default members, as its
-    # tests only work on a deployed control plane.
+    "end-to-end-tests",
     "gateway",
     "gateway-api",
     "gateway-cli",
@@ -174,8 +173,7 @@ default-members = [
     "internal-dns",
     "ipcc",
     "key-manager",
-    # Do not include live-tests in the list of default members because its tests
-    # only work in a deployed system.  The macros can be here, though.
+    "live-tests",
     "live-tests/macros",
     "nexus",
     "nexus-config",
diff --git a/dev-tools/xtask/src/check_workspace_deps.rs b/dev-tools/xtask/src/check_workspace_deps.rs
index 92e83d8724..a9627569b9 100644
--- a/dev-tools/xtask/src/check_workspace_deps.rs
+++ b/dev-tools/xtask/src/check_workspace_deps.rs
@@ -125,10 +125,6 @@ pub fn run_cmd() -> Result<()> {
                 // Including xtask causes hakari to not work as well and build
                 // times to be longer (omicron#4392).
                 "xtask",
-                // The tests here should not be run by default, as they require
-                // a running control plane.
-                "end-to-end-tests",
-                "omicron-live-tests",
             ]
             .contains(&package.name.as_str())
             .then_some(&package.id)
diff --git a/dev-tools/xtask/src/live_tests.rs b/dev-tools/xtask/src/live_tests.rs
index 35d40343ee..e63881a1fd 100644
--- a/dev-tools/xtask/src/live_tests.rs
+++ b/dev-tools/xtask/src/live_tests.rs
@@ -134,8 +134,11 @@ pub fn run_cmd(_args: Args) -> Result<()> {
     // TMPDIR=/var/tmp puts stuff on disk, cached as needed, rather than the
     // default /tmp which requires that stuff be in-memory.  That can lead to
     // great sadness if the tests wind up writing a lot of data.
+    //
+    // nextest configuration for these tests is specified in the "live-tests"
+    // profile.
     let raw = &[
-        "TMPDIR=/var/tmp ./cargo-nextest nextest run \\",
+        "TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \\",
         &format!(
             "--archive-file {}/{} \\",
             NAME,
diff --git a/live-tests/README.adoc b/live-tests/README.adoc
index 9ea3a93e48..56f9554bb7 100644
--- a/live-tests/README.adoc
+++ b/live-tests/README.adoc
@@ -57,7 +57,7 @@ To use this:
 
 4. On that system, run tests with:
 
-     TMPDIR=/var/tmp ./cargo-nextest nextest run \
+     TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \
          --archive-file live-tests-archive/omicron-live-tests.tar.zst \
          --workspace-remap live-tests-archive
 ```