From 3cde514a6b987bd8806ce2f9e77dd773ffe60518 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 06:42:12 -0700 Subject: [PATCH 1/7] build end-to-end tests and live tests by default --- .config/nextest.toml | 5 ++++- .github/buildomat/build-and-test.sh | 2 +- Cargo.toml | 6 ++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 67cabe9f0e..28219de9d0 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,7 +3,7 @@ # # The required version should be bumped up if we need new features, performance # improvements or bugfixes that are present in newer versions of nextest. -nextest-version = { required = "0.9.64", recommended = "0.9.70" } +nextest-version = { required = "0.9.76", recommended = "0.9.76" } experimental = ["setup-scripts"] @@ -35,6 +35,9 @@ clickhouse-cluster = { max-threads = 1 } # behaviors that conflict with each other. They need to be run serially. live-tests = { max-threads = 1 } +#[profile.default] +#default-set = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' + [[profile.default.overrides]] filter = 'package(oximeter-db) and test(replicated)' test-group = 'clickhouse-cluster' diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 50bbd8194d..a77a574dd4 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -9,7 +9,7 @@ target_os=$1 # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) -NEXTEST_VERSION='0.9.70' +NEXTEST_VERSION='0.9.76' cargo --version rustc --version diff --git a/Cargo.toml b/Cargo.toml index f44638d1d6..bb8f0c3734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,8 +157,7 @@ default-members = [ # See omicron#4392. "dns-server", "dns-server-api", - # Do not include end-to-end-tests in the list of default members, as its - # tests only work on a deployed control plane. + "end-to-end-tests", "gateway", "gateway-api", "gateway-cli", @@ -172,8 +171,7 @@ default-members = [ "internal-dns", "ipcc", "key-manager", - # Do not include live-tests in the list of default members because its tests - # only work in a deployed system. The macros can be here, though. + "live-tests", "live-tests/macros", "nexus", "nexus-config", From be8f9a840c12d60b4f752a76852b2231d15a5af1 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 07:46:14 -0700 Subject: [PATCH 2/7] actually use the nextest config --- .config/nextest.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 28219de9d0..adc6d51409 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -35,8 +35,8 @@ clickhouse-cluster = { max-threads = 1 } # behaviors that conflict with each other. They need to be run serially. live-tests = { max-threads = 1 } -#[profile.default] -#default-set = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' +[profile.default] +default-set = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' [[profile.default.overrides]] filter = 'package(oximeter-db) and test(replicated)' From 89b7893866397aa00a5397c6ab291b8c9f127c18 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 09:45:37 -0700 Subject: [PATCH 3/7] make live tests runnable again --- dev-tools/xtask/src/live_tests.rs | 5 ++++- live-tests/README.adoc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dev-tools/xtask/src/live_tests.rs b/dev-tools/xtask/src/live_tests.rs index 35d40343ee..4f9a80335d 100644 --- a/dev-tools/xtask/src/live_tests.rs +++ b/dev-tools/xtask/src/live_tests.rs @@ -134,8 +134,11 @@ pub fn run_cmd(_args: Args) -> Result<()> { // TMPDIR=/var/tmp puts stuff on disk, cached as needed, rather than the // default /tmp which requires that stuff be in-memory. That can lead to // great sadness if the tests wind up writing a lot of data. + // + // --bound=all is required to cause nextest to *not* ignore these tests, + // since they're not in the default filter set. let raw = &[ - "TMPDIR=/var/tmp ./cargo-nextest nextest run \\", + "TMPDIR=/var/tmp ./cargo-nextest nextest run --bound=all \\", &format!( "--archive-file {}/{} \\", NAME, diff --git a/live-tests/README.adoc b/live-tests/README.adoc index 9ea3a93e48..5ce9472b05 100644 --- a/live-tests/README.adoc +++ b/live-tests/README.adoc @@ -57,7 +57,7 @@ To use this: 4. On that system, run tests with: - TMPDIR=/var/tmp ./cargo-nextest nextest run \ + TMPDIR=/var/tmp ./cargo-nextest nextest run --bound=all \ --archive-file live-tests-archive/omicron-live-tests.tar.zst \ --workspace-remap live-tests-archive ``` From 8b77172e2d4ae48394beca13997bbcc34d243c3e Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 09:47:41 -0700 Subject: [PATCH 4/7] remove exception from check_workspace_deps --- dev-tools/xtask/src/check_workspace_deps.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev-tools/xtask/src/check_workspace_deps.rs b/dev-tools/xtask/src/check_workspace_deps.rs index 92e83d8724..a9627569b9 100644 --- a/dev-tools/xtask/src/check_workspace_deps.rs +++ b/dev-tools/xtask/src/check_workspace_deps.rs @@ -125,10 +125,6 @@ pub fn run_cmd() -> Result<()> { // Including xtask causes hakari to not work as well and build // times to be longer (omicron#4392). "xtask", - // The tests here should not be run by default, as they require - // a running control plane. - "end-to-end-tests", - "omicron-live-tests", ] .contains(&package.name.as_str()) .then_some(&package.id) From f9bb4ebd5e7db47223774626aca1c08f0ebc5142 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 13:43:45 -0700 Subject: [PATCH 5/7] use profiles instead --- .config/nextest.toml | 11 +++++++---- dev-tools/xtask/src/live_tests.rs | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index adc6d51409..48abb5d3a4 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -42,10 +42,6 @@ default-set = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' filter = 'package(oximeter-db) and test(replicated)' test-group = 'clickhouse-cluster' -[[profile.default.overrides]] -filter = 'package(omicron-live-tests)' -test-group = 'live-tests' - [[profile.default.overrides]] # These tests can time out under heavy contention. filter = 'binary_id(omicron-nexus::test_all) and test(::schema::)' @@ -56,3 +52,10 @@ filter = 'binary_id(omicron-nexus::test_all)' # As of 2023-01-08, the slowest test in test_all takes 196s on a Ryzen 7950X. # 900s is a good upper limit that adds a comfortable buffer. slow-timeout = { period = '60s', terminate-after = 15 } + +[profile.live-tests] +default-set = 'package(omicron-live-tests)' + +[[profile.live-tests.overrides]] +filter = 'package(omicron-live-tests)' +test-group = 'live-tests' diff --git a/dev-tools/xtask/src/live_tests.rs b/dev-tools/xtask/src/live_tests.rs index 4f9a80335d..e63881a1fd 100644 --- a/dev-tools/xtask/src/live_tests.rs +++ b/dev-tools/xtask/src/live_tests.rs @@ -135,10 +135,10 @@ pub fn run_cmd(_args: Args) -> Result<()> { // default /tmp which requires that stuff be in-memory. That can lead to // great sadness if the tests wind up writing a lot of data. // - // --bound=all is required to cause nextest to *not* ignore these tests, - // since they're not in the default filter set. + // nextest configuration for these tests is specified in the "live-tests" + // profile. let raw = &[ - "TMPDIR=/var/tmp ./cargo-nextest nextest run --bound=all \\", + "TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \\", &format!( "--archive-file {}/{} \\", NAME, From 6038bf97f95dc6db78f76230afc98b85223f0e56 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 14:53:40 -0700 Subject: [PATCH 6/7] nextest 0.9.77 --- .config/nextest.toml | 6 +++--- .github/buildomat/build-and-test.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 48abb5d3a4..eb62e1c9af 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -3,7 +3,7 @@ # # The required version should be bumped up if we need new features, performance # improvements or bugfixes that are present in newer versions of nextest. -nextest-version = { required = "0.9.76", recommended = "0.9.76" } +nextest-version = { required = "0.9.77", recommended = "0.9.77" } experimental = ["setup-scripts"] @@ -36,7 +36,7 @@ clickhouse-cluster = { max-threads = 1 } live-tests = { max-threads = 1 } [profile.default] -default-set = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' +default-filter = 'all() - package(omicron-live-tests) - package(end-to-end-tests)' [[profile.default.overrides]] filter = 'package(oximeter-db) and test(replicated)' @@ -54,7 +54,7 @@ filter = 'binary_id(omicron-nexus::test_all)' slow-timeout = { period = '60s', terminate-after = 15 } [profile.live-tests] -default-set = 'package(omicron-live-tests)' +default-filter = 'package(omicron-live-tests)' [[profile.live-tests.overrides]] filter = 'package(omicron-live-tests)' diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index a77a574dd4..1980012664 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -9,7 +9,7 @@ target_os=$1 # NOTE: This version should be in sync with the recommended version in # .config/nextest.toml. (Maybe build an automated way to pull the recommended # version in the future.) -NEXTEST_VERSION='0.9.76' +NEXTEST_VERSION='0.9.77' cargo --version rustc --version From 9189023e044977abc9abfb373f4c1c1bbc795459 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Wed, 28 Aug 2024 15:24:12 -0700 Subject: [PATCH 7/7] fix README --- live-tests/README.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-tests/README.adoc b/live-tests/README.adoc index 5ce9472b05..56f9554bb7 100644 --- a/live-tests/README.adoc +++ b/live-tests/README.adoc @@ -57,7 +57,7 @@ To use this: 4. On that system, run tests with: - TMPDIR=/var/tmp ./cargo-nextest nextest run --bound=all \ + TMPDIR=/var/tmp ./cargo-nextest nextest run --profile=live-tests \ --archive-file live-tests-archive/omicron-live-tests.tar.zst \ --workspace-remap live-tests-archive ```