[crdb-seed] use a tarball, fix omicron-dev run-all

.config/nextest.toml

@@ -8,7 +8,9 @@ nextest-version = { required = "0.9.59", recommended = "0.9.59" }
 experimental = ["setup-scripts"]
 
 [[profile.default.scripts]]
-filter = 'rdeps(nexus-test-utils)'
+# Exclude omicron-dev tests from crdb-seed as we explicitly want to simulate an
+# environment where the seed file doesn't exist.
+filter = 'rdeps(nexus-test-utils) - package(omicron-dev)'
 setup = 'crdb-seed'
 
 [profile.ci]

Cargo.toml

@@ -16,7 +16,7 @@ members = [
     "clients/sled-agent-client",
     "clients/wicketd-client",
     "common",
-    "crdb-seed",
+    "dev-tools/crdb-seed",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/thing-flinger",
@@ -83,6 +83,7 @@ default-members = [
     "clients/sled-agent-client",
     "clients/wicketd-client",
     "common",
+    "dev-tools/crdb-seed",
     "dev-tools/omdb",
     "dev-tools/omicron-dev",
     "dev-tools/thing-flinger",
@@ -137,6 +138,7 @@ assert_matches = "1.5.0"
 assert_cmd = "2.0.12"
 async-bb8-diesel = { git = "https://github.com/oxidecomputer/async-bb8-diesel", rev = "da04c087f835a51e0441addb19c5ef4986e1fcf2" }
 async-trait = "0.1.73"
+atomicwrites = "0.4.1"
 authz-macros = { path = "nexus/authz-macros" }
 backoff = { version = "0.4.0", features = [ "tokio" ] }
 base64 = "0.21.4"

crdb-seed/Cargo.toml → dev-tools/crdb-seed/Cargo.toml

@@ -3,14 +3,12 @@ name = "crdb-seed"
 version = "0.1.0"
 edition = "2021"
 license = "MPL-2.0"
+readme = "README.md"
 
 [dependencies]
-camino.workspace = true
-camino-tempfile.workspace = true
+anyhow.workspace = true
 dropshot.workspace = true
-hex.workspace = true
 omicron-test-utils.workspace = true
-ring.workspace = true
 slog.workspace = true
 tokio.workspace = true
 omicron-workspace-hack.workspace = true

dev-tools/crdb-seed/README.md

@@ -0,0 +1,11 @@
+# crdb-seed
+
+This is a small utility that creates a seed tarball for our CockroachDB instance
+in the temporary directory. It is used as a setup script for nextest (see
+`.config/nextest.rs`).
+
+This utility hashes inputs and attempts to reuse a tarball if it already exists
+(see `digest_unique_to_schema` in `omicron/test-utils/src/dev/seed.rs`).
+
+To invalidate the tarball and cause it to be recreated from scratch, set
+`CRDB_SEED_INVALIDATE=1` in the environment.

dev-tools/crdb-seed/src/main.rs

@@ -0,0 +1,38 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use anyhow::{Context, Result};
+use dropshot::{test_util::LogContext, ConfigLogging, ConfigLoggingLevel};
+use omicron_test_utils::dev::seed::{
+    ensure_seed_tarball_exists, should_invalidate_seed, CRDB_SEED_TAR_ENV,
+};
+use std::io::Write;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // TODO: dropshot is v heavyweight for this, we should be able to pull in a
+    // smaller binary
+    let logctx = LogContext::new(
+        "crdb_seeding",
+        &ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Info },
+    );
+    let (seed_tar, status) =
+        ensure_seed_tarball_exists(&logctx.log, should_invalidate_seed())
+            .await?;
+    status.log(&logctx.log, &seed_tar);
+
+    if let Ok(env_path) = std::env::var("NEXTEST_ENV") {
+        let mut file = std::fs::File::create(&env_path)
+            .context("failed to open NEXTEST_ENV file")?;
+        writeln!(file, "{CRDB_SEED_TAR_ENV}={seed_tar}")
+            .context("failed to write to NEXTEST_ENV file")?;
+    } else {
+        slog::warn!(
+            logctx.log,
+            "NEXTEST_ENV not set (is this script running under nextest?)"
+        );
+    }
+
+    Ok(())
+}

dev-tools/omicron-dev/src/bin/omicron-dev.rs

@@ -14,7 +14,6 @@ use futures::stream::StreamExt;
 use nexus_test_interface::NexusServer;
 use omicron_common::cmd::fatal;
 use omicron_common::cmd::CmdError;
-use omicron_sled_agent::sim;
 use omicron_test_utils::dev;
 use signal_hook::consts::signal::SIGINT;
 use signal_hook_tokio::Signals;
@@ -341,13 +340,12 @@ async fn cmd_run_all(args: &RunAllArgs) -> Result<(), anyhow::Error> {
         config.deployment.dropshot_external.dropshot.bind_address.set_port(p);
     }
 
-    // Start up a ControlPlaneTestContext, which tautologically sets up
-    // everything needed for a simulated control plane.
     println!("omicron-dev: setting up all services ... ");
-    let cptestctx = nexus_test_utils::test_setup_with_config::<
+    let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::<
         omicron_nexus::Server,
-    >("omicron-dev", &mut config, sim::SimMode::Auto, None)
-    .await;
+    >(&mut config)
+    .await
+    .context("error setting up services")?;
     println!("omicron-dev: services are running.");
 
     // Print out basic information about what was started.

dev-tools/omicron-dev/tests/test_omicron_dev.rs

@@ -8,6 +8,7 @@ use anyhow::Context;
 use expectorate::assert_contents;
 use omicron_test_utils::dev::db::has_omicron_schema;
 use omicron_test_utils::dev::process_running;
+use omicron_test_utils::dev::seed::CRDB_SEED_TAR_ENV;
 use omicron_test_utils::dev::test_cmds::assert_exit_code;
 use omicron_test_utils::dev::test_cmds::path_to_executable;
 use omicron_test_utils::dev::test_cmds::run_command;
@@ -389,6 +390,16 @@ async fn test_db_run() {
 // This mirrors the `test_db_run()` test.
 #[tokio::test]
 async fn test_run_all() {
+    // Ensure that the CRDB_SEED_TAR environment variable is not set. We want to
+    // simulate a user running omicron-dev without the test environment.
+    // Check if CRDB_SEED_TAR_ENV is set and panic if it is
+    if let Ok(val) = std::env::var(CRDB_SEED_TAR_ENV) {
+        panic!(
+            "CRDB_SEED_TAR_ENV should not be set here, but is set to {}",
+            val
+        );
+    }
+
     let cmd_path = path_to_omicron_dev();
 
     let cmdstr = format!(

nexus/test-utils/src/db.rs

@@ -8,28 +8,47 @@ use camino::Utf8PathBuf;
 use omicron_test_utils::dev;
 use slog::Logger;
 
-/// Path to the "seed" CockroachDB directory.
+/// Path to the "seed" CockroachDB tarball.
 ///
 /// Populating CockroachDB unfortunately isn't free - creation of
 /// tables, indices, and users takes several seconds to complete.
 ///
 /// By creating a "seed" version of the database, we can cut down
 /// on the time spent performing this operation. Instead, we opt
 /// to copy the database from this seed location.
-fn seed_dir() -> Utf8PathBuf {
+fn seed_tar() -> Utf8PathBuf {
     // The setup script should set this environment variable.
-    let seed_dir = std::env::var("CRDB_SEED_DIR")
-        .expect("CRDB_SEED_DIR missing -- are you running this test with `cargo nextest run`?");
+    let seed_dir =
+        std::env::var(dev::seed::CRDB_SEED_TAR_ENV).unwrap_or_else(|_| {
+            panic!(
+                "{} missing -- are you running this test \
+                 with `cargo nextest run`?",
+                dev::seed::CRDB_SEED_TAR_ENV,
+            )
+        });
     seed_dir.into()
 }
 
-/// Wrapper around [`dev::test_setup_database`] which uses a a
-/// seed directory provided at build-time.
+/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball
+/// provided from the environment.
 pub async fn test_setup_database(log: &Logger) -> dev::db::CockroachInstance {
-    let dir = seed_dir();
+    let input_tar = seed_tar();
     dev::test_setup_database(
         log,
-        dev::StorageSource::CopyFromSeed { input_dir: dir },
+        dev::StorageSource::CopyFromSeed { input_tar },
+    )
+    .await
+}
+
+/// Wrapper around [`dev::test_setup_database`] which uses a seed tarball
+/// provided as an argument.
+pub async fn test_setup_database_from_seed(
+    log: &Logger,
+    input_tar: Utf8PathBuf,
+) -> dev::db::CockroachInstance {
+    dev::test_setup_database(
+        log,
+        dev::StorageSource::CopyFromSeed { input_tar },
     )
     .await
 }