build end-to-end tests and live tests by default (#6469)

.config/nextest.toml

@@ -3,7 +3,7 @@
 #
 # The required version should be bumped up if we need new features, performance
 # improvements or bugfixes that are present in newer versions of nextest.
-nextest-version = { required = "0.9.64", recommended = "0.9.70" }
+nextest-version = { required = "0.9.77", recommended = "0.9.77" }
 
 experimental = ["setup-scripts"]
 
@@ -35,14 +35,13 @@ clickhouse-cluster = { max-threads = 1 }
 # behaviors that conflict with each other.  They need to be run serially.
 live-tests = { max-threads = 1 }
 
+[profile.default]
+default-filter = 'all() - package(omicron-live-tests) - package(end-to-end-tests)'
+
 [[profile.default.overrides]]
 filter = 'package(oximeter-db) and test(replicated)'
 test-group = 'clickhouse-cluster'
 
-[[profile.default.overrides]]
-filter = 'package(omicron-live-tests)'
-test-group = 'live-tests'
-
 [[profile.default.overrides]]
 # These tests can time out under heavy contention.
 filter = 'binary_id(omicron-nexus::test_all) and test(::schema::)'
@@ -53,3 +52,10 @@ filter = 'binary_id(omicron-nexus::test_all)'
 # As of 2023-01-08, the slowest test in test_all takes 196s on a Ryzen 7950X.
 # 900s is a good upper limit that adds a comfortable buffer.
 slow-timeout = { period = '60s', terminate-after = 15 }
+
+[profile.live-tests]
+default-filter = 'package(omicron-live-tests)'
+
+[[profile.live-tests.overrides]]
+filter = 'package(omicron-live-tests)'
+test-group = 'live-tests'

.github/buildomat/build-and-test.sh

@@ -9,7 +9,7 @@ target_os=$1
 # NOTE: This version should be in sync with the recommended version in
 # .config/nextest.toml. (Maybe build an automated way to pull the recommended
 # version in the future.)
-NEXTEST_VERSION='0.9.70'
+NEXTEST_VERSION='0.9.77'
 
 cargo --version
 rustc --version

Cargo.toml

@@ -159,8 +159,7 @@ default-members = [
     # See omicron#4392.
     "dns-server",
     "dns-server-api",
-    # Do not include end-to-end-tests in the list of default members, as its
-    # tests only work on a deployed control plane.
+    "end-to-end-tests",
     "gateway",
     "gateway-api",
     "gateway-cli",
@@ -174,8 +173,7 @@ default-members = [
     "internal-dns",
     "ipcc",
     "key-manager",
-    # Do not include live-tests in the list of default members because its tests
-    # only work in a deployed system.  The macros can be here, though.
+    "live-tests",
     "live-tests/macros",
     "nexus",
     "nexus-config",

dev-tools/xtask/src/check_workspace_deps.rs

@@ -125,10 +125,6 @@ pub fn run_cmd() -> Result<()> {
                 // Including xtask causes hakari to not work as well and build
                 // times to be longer (omicron#4392).
                 "xtask",
-                // The tests here should not be run by default, as they require
-                // a running control plane.
-                "end-to-end-tests",
-                "omicron-live-tests",
             ]
             .contains(&package.name.as_str())
             .then_some(&package.id)

dev-tools/xtask/src/live_tests.rs

@@ -134,8 +134,11 @@ pub fn run_cmd(_args: Args) -> Result<()> {
     // TMPDIR=/var/tmp puts stuff on disk, cached as needed, rather than the
     // default /tmp which requires that stuff be in-memory.  That can lead to
     // great sadness if the tests wind up writing a lot of data.
+    //
+    // nextest configuration for these tests is specified in the "live-tests"
+    // profile.
     let raw = &[
-        "TMPDIR=/var/tmp ./cargo-nextest nextest run \\",
+        "TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \\",
         &format!(
             "--archive-file {}/{} \\",
             NAME,

live-tests/README.adoc

@@ -57,7 +57,7 @@ To use this:
 
 4. On that system, run tests with:
 
-     TMPDIR=/var/tmp ./cargo-nextest nextest run \
+     TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \
          --archive-file live-tests-archive/omicron-live-tests.tar.zst \
          --workspace-remap live-tests-archive
 ```