From 492260b6c0584928c4e075da0754ac753905cb39 Mon Sep 17 00:00:00 2001 From: Rain Date: Sun, 28 Jul 2024 12:47:51 -0700 Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20change?= =?UTF-8?q?s=20to=20main=20this=20commit=20is=20based=20on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.6-beta.1 [skip ci] --- Cargo.lock | 1 - dev-tools/omicron-dev/Cargo.toml | 53 +- dev-tools/omicron-dev/src/bin/omicron-dev.rs | 638 +------------------ dev-tools/omicron-dev/src/cert.rs | 64 ++ dev-tools/omicron-dev/src/clickhouse.rs | 214 +++++++ dev-tools/omicron-dev/src/db.rs | 195 ++++++ dev-tools/omicron-dev/src/dispatch.rs | 126 ++++ dev-tools/omicron-dev/src/lib.rs | 19 + dev-tools/omicron-dev/src/mgs.rs | 42 ++ dev-tools/omicron-dev/src/nexus.rs | 126 ++++ dev-tools/xtask/src/check_features.rs | 44 ++ dev-tools/xtask/src/external.rs | 24 +- dev-tools/xtask/src/main.rs | 5 + dev-tools/xtask/src/omicron_dev.rs | 78 +++ docs/how-to-run-simulated.adoc | 22 +- docs/how-to-run.adoc | 2 +- test-utils/src/dev/db.rs | 2 +- wicket/README.md | 18 +- 18 files changed, 993 insertions(+), 680 deletions(-) create mode 100644 dev-tools/omicron-dev/src/cert.rs create mode 100644 dev-tools/omicron-dev/src/clickhouse.rs create mode 100644 dev-tools/omicron-dev/src/db.rs create mode 100644 dev-tools/omicron-dev/src/dispatch.rs create mode 100644 dev-tools/omicron-dev/src/lib.rs create mode 100644 dev-tools/omicron-dev/src/mgs.rs create mode 100644 dev-tools/omicron-dev/src/nexus.rs create mode 100644 dev-tools/xtask/src/omicron_dev.rs diff --git a/Cargo.lock b/Cargo.lock index 41c3bef36f..5d9982d8a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5535,7 +5535,6 @@ dependencies = [ "oxide-client", "pq-sys", "rcgen", - "signal-hook", "signal-hook-tokio", "subprocess", "tokio", diff --git a/dev-tools/omicron-dev/Cargo.toml b/dev-tools/omicron-dev/Cargo.toml index 1dcc4eada7..499c4cfa77 100644 --- a/dev-tools/omicron-dev/Cargo.toml +++ b/dev-tools/omicron-dev/Cargo.toml @@ -14,25 +14,24 @@ omicron-rpaths.workspace = true anyhow.workspace = true camino.workspace = true clap.workspace = true -dropshot.workspace = true -futures.workspace = true -gateway-messages.workspace = true -gateway-test-utils.workspace = true +dropshot = { workspace = true, optional = true } +futures = { workspace = true, optional = true } +gateway-messages = { workspace = true, optional = true } +gateway-test-utils = { workspace = true, optional = true } libc.workspace = true -nexus-config.workspace = true -nexus-test-utils = { workspace = true, features = ["omicron-dev"] } -nexus-test-interface.workspace = true +nexus-config = { workspace = true, optional = true } +nexus-test-interface = { workspace = true, optional = true } +nexus-test-utils = { workspace = true, features = ["omicron-dev"], optional = true } omicron-common.workspace = true -omicron-nexus.workspace = true -omicron-test-utils.workspace = true +omicron-nexus = { workspace = true, optional = true } +omicron-test-utils = { workspace = true, optional = true } # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" -rcgen.workspace = true -signal-hook.workspace = true -signal-hook-tokio.workspace = true +rcgen = { workspace = true, optional = true } +signal-hook-tokio = { workspace = true, optional = true } tokio = { workspace = true, features = [ "full" ] } -tokio-postgres.workspace = true -toml.workspace = true +tokio-postgres = { workspace = true, optional = true } +toml = { workspace = true, optional = true } omicron-workspace-hack.workspace = true [dev-dependencies] @@ -49,3 +48,29 @@ subprocess.workspace = true [[bin]] name = "omicron-dev" doc = false + +[[test]] +name = "test_omicron_dev" +required-features = ["default"] + +[features] +# default-features set to all of the below features for two reasons: +# +# 1. Backwards compatibility with existing users of this crate. +# 2. rust-analyzer will build the crate with all features enabled by default. +# +# The xtask uses --no-default-features to ensure that only the right set of +# features is built. +default = ["include-cert", "include-clickhouse", "include-db", "include-mgs", "include-nexus"] + +# Each feature corresponds to one file in `src`, and is feature-flagged in +# lib.rs. +# +# NOTE: When adding a new feature, also add it to: +# - the `default` feature list above +# - CI_MUTUALLY_EXCLUSIVE_FEATURES in dev-tools/xtask/src/check_features.rs +include-cert = ["dep:rcgen"] +include-clickhouse = ["dep:dropshot", "dep:futures", "dep:omicron-test-utils", "dep:signal-hook-tokio"] +include-db = ["dep:futures", "dep:omicron-test-utils", "dep:signal-hook-tokio", "dep:tokio-postgres"] +include-mgs = ["dep:futures", "dep:gateway-messages", "dep:gateway-test-utils", "dep:signal-hook-tokio"] +include-nexus = ["dep:dropshot", "dep:futures", "dep:nexus-config", "dep:nexus-test-interface", "dep:nexus-test-utils", "dep:omicron-nexus", "dep:signal-hook-tokio", "dep:toml"] diff --git a/dev-tools/omicron-dev/src/bin/omicron-dev.rs b/dev-tools/omicron-dev/src/bin/omicron-dev.rs index 705049bdb1..f8ce35b5bb 100644 --- a/dev-tools/omicron-dev/src/bin/omicron-dev.rs +++ b/dev-tools/omicron-dev/src/bin/omicron-dev.rs @@ -4,646 +4,16 @@ //! Developer tool for easily running bits of Omicron -use anyhow::{bail, Context}; -use camino::Utf8Path; -use camino::Utf8PathBuf; -use clap::Args; use clap::Parser; -use dropshot::test_util::LogContext; -use futures::stream::StreamExt; -use nexus_config::NexusConfig; -use nexus_test_interface::NexusServer; -use omicron_common::cmd::fatal; -use omicron_common::cmd::CmdError; -use omicron_test_utils::dev; -use signal_hook::consts::signal::SIGINT; -use signal_hook_tokio::Signals; -use std::io::Write; -use std::os::unix::prelude::OpenOptionsExt; -use std::path::PathBuf; +use omicron_common::cmd::{fatal, CmdError}; +use omicron_dev::OmicronDevApp; #[tokio::main] async fn main() -> Result<(), anyhow::Error> { - let subcmd = OmicronDb::parse(); - let result = match subcmd { - OmicronDb::DbRun { ref args } => cmd_db_run(args).await, - OmicronDb::DbPopulate { ref args } => cmd_db_populate(args).await, - OmicronDb::DbWipe { ref args } => cmd_db_wipe(args).await, - OmicronDb::ChRun { ref args } => cmd_clickhouse_run(args).await, - OmicronDb::MgsRun { ref args } => cmd_mgs_run(args).await, - OmicronDb::RunAll { ref args } => cmd_run_all(args).await, - OmicronDb::CertCreate { ref args } => cmd_cert_create(args).await, - }; + let app = OmicronDevApp::parse(); + let result = app.exec().await; if let Err(error) = result { fatal(CmdError::Failure(error)); } Ok(()) } - -/// Tools for working with a local Omicron deployment -#[derive(Debug, Parser)] -#[clap(version)] -enum OmicronDb { - /// Start a CockroachDB cluster for development - DbRun { - #[clap(flatten)] - args: DbRunArgs, - }, - - /// Populate an existing CockroachDB cluster with the Omicron schema - DbPopulate { - #[clap(flatten)] - args: DbPopulateArgs, - }, - - /// Wipe the Omicron schema (and all data) from an existing CockroachDB - /// cluster - DbWipe { - #[clap(flatten)] - args: DbWipeArgs, - }, - - /// Run a ClickHouse database server for development - ChRun { - #[clap(flatten)] - args: ChRunArgs, - }, - - /// Run a simulated Management Gateway Service for development - MgsRun { - #[clap(flatten)] - args: MgsRunArgs, - }, - - /// Run a full simulated control plane - RunAll { - #[clap(flatten)] - args: RunAllArgs, - }, - - /// Create a self-signed certificate for use with Omicron - CertCreate { - #[clap(flatten)] - args: CertCreateArgs, - }, -} - -#[derive(Clone, Debug, Args)] -struct DbRunArgs { - /// Path to store database data (default: temp dir cleaned up on exit) - #[clap(long, action)] - store_dir: Option, - - /// Database (SQL) listen port. Use `0` to request any available port. - // We choose an arbitrary default port that's different from the default - // CockroachDB port to avoid conflicting. We don't use 0 because this port - // is specified in a few other places, like the default Nexus config file. - // TODO We could load that file at compile time and use the value there. - #[clap(long, default_value = "32221", action)] - listen_port: u16, - - // This unusual clap configuration makes "populate" default to true, - // allowing a --no-populate override on the CLI. - /// Do not populate the database with any schema - #[clap(long = "no-populate", action(clap::ArgAction::SetFalse))] - populate: bool, -} - -async fn cmd_db_run(args: &DbRunArgs) -> Result<(), anyhow::Error> { - // Set ourselves up to wait for SIGINT. It's important to do this early, - // before we've created resources that we want to have cleaned up on SIGINT - // (e.g., the temporary directory created by the database starter). - let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); - - // Now start CockroachDB. This process looks bureaucratic (create arg - // builder, then create starter, then start it) because we want to be able - // to print what's happening before we do it. - let mut db_arg_builder = - dev::db::CockroachStarterBuilder::new().listen_port(args.listen_port); - - // NOTE: The stdout strings here are not intended to be stable, but they are - // used by the test suite. - - if let Some(store_dir) = &args.store_dir { - println!( - "omicron-dev: using user-provided path for database store: {}", - store_dir.display() - ); - db_arg_builder = db_arg_builder.store_dir(store_dir); - } else { - println!( - "omicron-dev: using temporary directory for database store \ - (cleaned up on clean exit)" - ); - } - - let db_starter = db_arg_builder.build()?; - println!( - "omicron-dev: will run this to start CockroachDB:\n{}", - db_starter.cmdline() - ); - println!("omicron-dev: environment:"); - for (k, v) in db_starter.environment() { - println!(" {}={}", k, v); - } - println!( - "omicron-dev: temporary directory: {}", - db_starter.temp_dir().display() - ); - - let mut db_instance = db_starter.start().await?; - println!("\nomicron-dev: child process: pid {}", db_instance.pid()); - println!( - "omicron-dev: CockroachDB listening at: {}", - db_instance.listen_url() - ); - - if args.populate { - // Populate the database with our schema. - let start = tokio::time::Instant::now(); - println!("omicron-dev: populating database"); - db_instance.populate().await.context("populating database")?; - let end = tokio::time::Instant::now(); - let duration = end.duration_since(start); - println!( - "omicron-dev: populated database in {}.{} seconds", - duration.as_secs(), - duration.subsec_millis() - ); - } - - // Wait for either the child process to shut down on its own or for us to - // receive SIGINT. - tokio::select! { - _ = db_instance.wait_for_shutdown() => { - db_instance.cleanup().await.context("clean up after shutdown")?; - bail!( - "omicron-dev: database shut down unexpectedly \ - (see error output above)" - ); - } - caught_signal = signal_stream.next() => { - assert_eq!(caught_signal.unwrap(), SIGINT); - - /* - * We don't have to do anything to trigger shutdown because the - * shell will have delivered the same SIGINT that we got to the - * cockroach process as well. - */ - eprintln!( - "omicron-dev: caught signal, shutting down and removing \ - temporary directory" - ); - - db_instance - .wait_for_shutdown() - .await - .context("clean up after SIGINT shutdown")?; - } - } - - Ok(()) -} - -#[derive(Debug, Args)] -struct DbPopulateArgs { - /// URL for connecting to the database (postgresql:///...) - #[clap(long, action)] - database_url: String, - - /// Wipe any existing schema (and data!) before populating - #[clap(long, action)] - wipe: bool, -} - -async fn cmd_db_populate(args: &DbPopulateArgs) -> Result<(), anyhow::Error> { - let config = - args.database_url.parse::().with_context( - || format!("parsing database URL {:?}", args.database_url), - )?; - let client = dev::db::Client::connect(&config, tokio_postgres::NoTls) - .await - .with_context(|| format!("connecting to {:?}", args.database_url))?; - - if args.wipe { - println!("omicron-dev: wiping any existing database"); - dev::db::wipe(&client).await?; - } - - println!("omicron-dev: populating database"); - dev::db::populate(&client).await?; - println!("omicron-dev: populated database"); - client.cleanup().await.expect("connection failed"); - Ok(()) -} - -#[derive(Debug, Args)] -struct DbWipeArgs { - /// URL for connecting to the database (postgresql:///...) - #[clap(long, action)] - database_url: String, -} - -async fn cmd_db_wipe(args: &DbWipeArgs) -> Result<(), anyhow::Error> { - let config = - args.database_url.parse::().with_context( - || format!("parsing database URL {:?}", args.database_url), - )?; - let client = dev::db::Client::connect(&config, tokio_postgres::NoTls) - .await - .with_context(|| format!("connecting to {:?}", args.database_url))?; - - println!("omicron-dev: wiping any existing database"); - dev::db::wipe(&client).await?; - println!("omicron-dev: wiped"); - client.cleanup().await.expect("connection failed"); - Ok(()) -} - -#[derive(Clone, Debug, Args)] -struct ChRunArgs { - /// The HTTP port on which the server will listen - #[clap(short, long, default_value = "8123", action)] - port: u16, - /// Starts a ClickHouse replicated cluster of 2 replicas and 3 keeper nodes - #[clap(long, conflicts_with = "port", action)] - replicated: bool, -} - -async fn cmd_clickhouse_run(args: &ChRunArgs) -> Result<(), anyhow::Error> { - let logctx = LogContext::new( - "omicron-dev", - &dropshot::ConfigLogging::StderrTerminal { - level: dropshot::ConfigLoggingLevel::Info, - }, - ); - if args.replicated { - start_replicated_cluster(&logctx).await?; - } else { - start_single_node(&logctx, args.port).await?; - } - Ok(()) -} - -async fn start_single_node( - logctx: &LogContext, - port: u16, -) -> Result<(), anyhow::Error> { - // Start a stream listening for SIGINT - let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); - - // Start the database server process, possibly on a specific port - let mut db_instance = - dev::clickhouse::ClickHouseInstance::new_single_node(logctx, port) - .await?; - println!( - "omicron-dev: running ClickHouse with full command:\n\"clickhouse {}\"", - db_instance.cmdline().join(" ") - ); - println!( - "omicron-dev: ClickHouse is running with PID {}", - db_instance - .pid() - .expect("Failed to get process PID, it may not have started") - ); - println!( - "omicron-dev: ClickHouse HTTP server listening on port {}", - db_instance.port() - ); - println!( - "omicron-dev: using {} for ClickHouse data storage", - db_instance.data_path() - ); - - // Wait for the DB to exit itself (an error), or for SIGINT - tokio::select! { - _ = db_instance.wait_for_shutdown() => { - db_instance.cleanup().await.context("clean up after shutdown")?; - bail!("omicron-dev: ClickHouse shutdown unexpectedly"); - } - caught_signal = signal_stream.next() => { - assert_eq!(caught_signal.unwrap(), SIGINT); - - // As above, we don't need to explicitly kill the DB process, since - // the shell will have delivered the signal to the whole process group. - eprintln!( - "omicron-dev: caught signal, shutting down and removing \ - temporary directory" - ); - - // Remove the data directory. - db_instance - .wait_for_shutdown() - .await - .context("clean up after SIGINT shutdown")?; - } - } - Ok(()) -} - -async fn start_replicated_cluster( - logctx: &LogContext, -) -> Result<(), anyhow::Error> { - // Start a stream listening for SIGINT - let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); - - // Start the database server and keeper processes - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let replica_config = manifest_dir - .as_path() - .join("../../oximeter/db/src/configs/replica_config.xml"); - let keeper_config = manifest_dir - .as_path() - .join("../../oximeter/db/src/configs/keeper_config.xml"); - - let mut cluster = dev::clickhouse::ClickHouseCluster::new( - logctx, - replica_config, - keeper_config, - ) - .await?; - println!( - "omicron-dev: running ClickHouse cluster with configuration files:\n \ - replicas: {}\n keepers: {}", - cluster.replica_config_path().display(), - cluster.keeper_config_path().display() - ); - let pid_error_msg = "Failed to get process PID, it may not have started"; - println!( - "omicron-dev: ClickHouse cluster is running with: server PIDs = [{}, {}] \ - and keeper PIDs = [{}, {}, {}]", - cluster.replica_1 - .pid() - .expect(pid_error_msg), - cluster.replica_2 - .pid() - .expect(pid_error_msg), - cluster.keeper_1 - .pid() - .expect(pid_error_msg), - cluster.keeper_2 - .pid() - .expect(pid_error_msg), - cluster.keeper_3 - .pid() - .expect(pid_error_msg), - ); - println!( - "omicron-dev: ClickHouse HTTP servers listening on ports: {}, {}", - cluster.replica_1.port(), - cluster.replica_2.port() - ); - println!( - "omicron-dev: using {} and {} for ClickHouse data storage", - cluster.replica_1.data_path(), - cluster.replica_2.data_path() - ); - - // Wait for the replicas and keepers to exit themselves (an error), or for SIGINT - tokio::select! { - _ = cluster.replica_1.wait_for_shutdown() => { - cluster.replica_1.cleanup().await.context( - format!("clean up {} after shutdown", cluster.replica_1.data_path()) - )?; - bail!("omicron-dev: ClickHouse replica 1 shutdown unexpectedly"); - } - _ = cluster.replica_2.wait_for_shutdown() => { - cluster.replica_2.cleanup().await.context( - format!("clean up {} after shutdown", cluster.replica_2.data_path()) - )?; - bail!("omicron-dev: ClickHouse replica 2 shutdown unexpectedly"); - } - _ = cluster.keeper_1.wait_for_shutdown() => { - cluster.keeper_1.cleanup().await.context( - format!("clean up {} after shutdown", cluster.keeper_1.data_path()) - )?; - bail!("omicron-dev: ClickHouse keeper 1 shutdown unexpectedly"); - } - _ = cluster.keeper_2.wait_for_shutdown() => { - cluster.keeper_2.cleanup().await.context( - format!("clean up {} after shutdown", cluster.keeper_2.data_path()) - )?; - bail!("omicron-dev: ClickHouse keeper 2 shutdown unexpectedly"); - } - _ = cluster.keeper_3.wait_for_shutdown() => { - cluster.keeper_3.cleanup().await.context( - format!("clean up {} after shutdown", cluster.keeper_3.data_path()) - )?; - bail!("omicron-dev: ClickHouse keeper 3 shutdown unexpectedly"); - } - caught_signal = signal_stream.next() => { - assert_eq!(caught_signal.unwrap(), SIGINT); - eprintln!( - "omicron-dev: caught signal, shutting down and removing \ - temporary directories" - ); - - // Remove the data directories. - let mut instances = vec![ - cluster.replica_1, - cluster.replica_2, - cluster.keeper_1, - cluster.keeper_2, - cluster.keeper_3, - ]; - for instance in instances.iter_mut() { - instance - .wait_for_shutdown() - .await - .context(format!("clean up {} after SIGINT shutdown", instance.data_path()))?; - }; - } - } - Ok(()) -} - -#[derive(Clone, Debug, Args)] -struct RunAllArgs { - /// Nexus external API listen port. Use `0` to request any available port. - #[clap(long, action)] - nexus_listen_port: Option, -} - -async fn cmd_run_all(args: &RunAllArgs) -> Result<(), anyhow::Error> { - // Start a stream listening for SIGINT - let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); - - // Read configuration. - let config_str = include_str!("../../../../nexus/examples/config.toml"); - let mut config: NexusConfig = - toml::from_str(config_str).context("parsing example config")?; - config.pkg.log = dropshot::ConfigLogging::File { - // See LogContext::new(), - path: "UNUSED".to_string().into(), - level: dropshot::ConfigLoggingLevel::Trace, - if_exists: dropshot::ConfigLoggingIfExists::Fail, - }; - - if let Some(p) = args.nexus_listen_port { - config.deployment.dropshot_external.dropshot.bind_address.set_port(p); - } - - println!("omicron-dev: setting up all services ... "); - let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< - omicron_nexus::Server, - >(&mut config) - .await - .context("error setting up services")?; - println!("omicron-dev: services are running."); - - // Print out basic information about what was started. - // NOTE: The stdout strings here are not intended to be stable, but they are - // used by the test suite. - let addr = cptestctx.external_client.bind_address; - println!("omicron-dev: nexus external API: {:?}", addr); - println!( - "omicron-dev: nexus internal API: {:?}", - cptestctx.server.get_http_server_internal_address().await, - ); - println!( - "omicron-dev: cockroachdb pid: {}", - cptestctx.database.pid(), - ); - println!( - "omicron-dev: cockroachdb URL: {}", - cptestctx.database.pg_config() - ); - println!( - "omicron-dev: cockroachdb directory: {}", - cptestctx.database.temp_dir().display() - ); - println!( - "omicron-dev: internal DNS HTTP: http://{}", - cptestctx.internal_dns.dropshot_server.local_addr() - ); - println!( - "omicron-dev: internal DNS: {}", - cptestctx.internal_dns.dns_server.local_address() - ); - println!( - "omicron-dev: external DNS name: {}", - cptestctx.external_dns_zone_name, - ); - println!( - "omicron-dev: external DNS HTTP: http://{}", - cptestctx.external_dns.dropshot_server.local_addr() - ); - println!( - "omicron-dev: external DNS: {}", - cptestctx.external_dns.dns_server.local_address() - ); - println!( - "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", - cptestctx.external_dns.dns_server.local_address().ip(), - cptestctx.external_dns.dns_server.local_address().port(), - cptestctx.silo_name, - cptestctx.external_dns_zone_name, - ); - for (location, gateway) in &cptestctx.gateway { - println!( - "omicron-dev: management gateway: http://{} ({})", - gateway.client.bind_address, location, - ); - } - println!("omicron-dev: silo name: {}", cptestctx.silo_name,); - println!( - "omicron-dev: privileged user name: {}", - cptestctx.user_name.as_ref(), - ); - - // Wait for a signal. - let caught_signal = signal_stream.next().await; - assert_eq!(caught_signal.unwrap(), SIGINT); - eprintln!( - "omicron-dev: caught signal, shutting down and removing \ - temporary directory" - ); - - cptestctx.teardown().await; - Ok(()) -} - -#[derive(Clone, Debug, Args)] -struct CertCreateArgs { - /// path to where the generated certificate and key files should go - /// (e.g., "out/initial-" would cause the files to be called - /// "out/initial-cert.pem" and "out/initial-key.pem") - #[clap(action)] - output_base: Utf8PathBuf, - - /// DNS names that the certificate claims to be valid for (subject - /// alternative names) - #[clap(action, required = true)] - server_names: Vec, -} - -async fn cmd_cert_create(args: &CertCreateArgs) -> Result<(), anyhow::Error> { - let cert = rcgen::generate_simple_self_signed(args.server_names.clone()) - .context("generating certificate")?; - let cert_pem = - cert.serialize_pem().context("serializing certificate as PEM")?; - let key_pem = cert.serialize_private_key_pem(); - - let cert_path = Utf8PathBuf::from(format!("{}cert.pem", args.output_base)); - write_private_file(&cert_path, cert_pem.as_bytes()) - .context("writing certificate file")?; - println!("wrote certificate to {}", cert_path); - - let key_path = Utf8PathBuf::from(format!("{}key.pem", args.output_base)); - write_private_file(&key_path, key_pem.as_bytes()) - .context("writing private key file")?; - println!("wrote private key to {}", key_path); - - Ok(()) -} - -#[cfg_attr(not(mac), allow(clippy::useless_conversion))] -fn write_private_file( - path: &Utf8Path, - contents: &[u8], -) -> Result<(), anyhow::Error> { - // The file should be readable and writable by the user only. - let perms = libc::S_IRUSR | libc::S_IWUSR; - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(perms.into()) // into() needed on mac only - .open(path) - .with_context(|| format!("open {:?} for writing", path))?; - file.write_all(contents).with_context(|| format!("write to {:?}", path)) -} - -#[derive(Clone, Debug, Args)] -struct MgsRunArgs {} - -async fn cmd_mgs_run(_args: &MgsRunArgs) -> Result<(), anyhow::Error> { - // Start a stream listening for SIGINT - let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); - let mut signal_stream = signals.fuse(); - - println!("omicron-dev: setting up MGS ... "); - let gwtestctx = gateway_test_utils::setup::test_setup( - "omicron-dev", - gateway_messages::SpPort::One, - ) - .await; - println!("omicron-dev: MGS is running."); - - let addr = gwtestctx.client.bind_address; - println!("omicron-dev: MGS API: http://{:?}", addr); - - // Wait for a signal. - let caught_signal = signal_stream.next().await; - assert_eq!(caught_signal.unwrap(), SIGINT); - eprintln!( - "omicron-dev: caught signal, shutting down and removing \ - temporary directory" - ); - - gwtestctx.teardown().await; - Ok(()) -} diff --git a/dev-tools/omicron-dev/src/cert.rs b/dev-tools/omicron-dev/src/cert.rs new file mode 100644 index 0000000000..e9d9371d20 --- /dev/null +++ b/dev-tools/omicron-dev/src/cert.rs @@ -0,0 +1,64 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::{io::Write, os::unix::fs::OpenOptionsExt}; + +use anyhow::Context; +use camino::{Utf8Path, Utf8PathBuf}; +use clap::Args; + +#[derive(Clone, Debug, Args)] +pub(crate) struct CertCreateArgs { + /// path to where the generated certificate and key files should go + /// (e.g., "out/initial-" would cause the files to be called + /// "out/initial-cert.pem" and "out/initial-key.pem") + #[clap(action)] + output_base: Utf8PathBuf, + + /// DNS names that the certificate claims to be valid for (subject + /// alternative names) + #[clap(action, required = true)] + server_names: Vec, +} + +impl CertCreateArgs { + pub(crate) async fn exec(&self) -> Result<(), anyhow::Error> { + let cert = + rcgen::generate_simple_self_signed(self.server_names.clone()) + .context("generating certificate")?; + let cert_pem = + cert.serialize_pem().context("serializing certificate as PEM")?; + let key_pem = cert.serialize_private_key_pem(); + + let cert_path = + Utf8PathBuf::from(format!("{}cert.pem", self.output_base)); + write_private_file(&cert_path, cert_pem.as_bytes()) + .context("writing certificate file")?; + println!("wrote certificate to {}", cert_path); + + let key_path = + Utf8PathBuf::from(format!("{}key.pem", self.output_base)); + write_private_file(&key_path, key_pem.as_bytes()) + .context("writing private key file")?; + println!("wrote private key to {}", key_path); + + Ok(()) + } +} + +#[cfg_attr(not(mac), allow(clippy::useless_conversion))] +fn write_private_file( + path: &Utf8Path, + contents: &[u8], +) -> Result<(), anyhow::Error> { + // The file should be readable and writable by the user only. + let perms = libc::S_IRUSR | libc::S_IWUSR; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(perms.into()) // into() needed on mac only + .open(path) + .with_context(|| format!("open {:?} for writing", path))?; + file.write_all(contents).with_context(|| format!("write to {:?}", path)) +} diff --git a/dev-tools/omicron-dev/src/clickhouse.rs b/dev-tools/omicron-dev/src/clickhouse.rs new file mode 100644 index 0000000000..93008afd1d --- /dev/null +++ b/dev-tools/omicron-dev/src/clickhouse.rs @@ -0,0 +1,214 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::path::PathBuf; + +use anyhow::{bail, Context}; +use clap::Args; +use dropshot::test_util::LogContext; +use futures::StreamExt; +use libc::SIGINT; +use omicron_test_utils::dev; +use signal_hook_tokio::Signals; + +#[derive(Clone, Debug, Args)] +pub(crate) struct ChRunArgs { + /// The HTTP port on which the server will listen + #[clap(short, long, default_value = "8123", action)] + port: u16, + /// Starts a ClickHouse replicated cluster of 2 replicas and 3 keeper nodes + #[clap(long, conflicts_with = "port", action)] + replicated: bool, +} + +impl ChRunArgs { + pub(crate) async fn exec(&self) -> Result<(), anyhow::Error> { + let logctx = LogContext::new( + "omicron-dev", + &dropshot::ConfigLogging::StderrTerminal { + level: dropshot::ConfigLoggingLevel::Info, + }, + ); + if self.replicated { + start_replicated_cluster(&logctx).await?; + } else { + start_single_node(&logctx, self.port).await?; + } + Ok(()) + } +} + +async fn start_single_node( + logctx: &LogContext, + port: u16, +) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + // Start the database server process, possibly on a specific port + let mut db_instance = + dev::clickhouse::ClickHouseInstance::new_single_node(logctx, port) + .await?; + println!( + "omicron-dev: running ClickHouse with full command:\n\"clickhouse {}\"", + db_instance.cmdline().join(" ") + ); + println!( + "omicron-dev: ClickHouse is running with PID {}", + db_instance + .pid() + .expect("Failed to get process PID, it may not have started") + ); + println!( + "omicron-dev: ClickHouse HTTP server listening on port {}", + db_instance.port() + ); + println!( + "omicron-dev: using {} for ClickHouse data storage", + db_instance.data_path() + ); + + // Wait for the DB to exit itself (an error), or for SIGINT + tokio::select! { + _ = db_instance.wait_for_shutdown() => { + db_instance.cleanup().await.context("clean up after shutdown")?; + bail!("omicron-dev: ClickHouse shutdown unexpectedly"); + } + caught_signal = signal_stream.next() => { + assert_eq!(caught_signal.unwrap(), SIGINT); + + // As above, we don't need to explicitly kill the DB process, since + // the shell will have delivered the signal to the whole process group. + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + // Remove the data directory. + db_instance + .wait_for_shutdown() + .await + .context("clean up after SIGINT shutdown")?; + } + } + Ok(()) +} + +async fn start_replicated_cluster( + logctx: &LogContext, +) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + // Start the database server and keeper processes + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let replica_config = manifest_dir + .as_path() + .join("../../oximeter/db/src/configs/replica_config.xml"); + let keeper_config = manifest_dir + .as_path() + .join("../../oximeter/db/src/configs/keeper_config.xml"); + + let mut cluster = dev::clickhouse::ClickHouseCluster::new( + logctx, + replica_config, + keeper_config, + ) + .await?; + println!( + "omicron-dev: running ClickHouse cluster with configuration files:\n \ + replicas: {}\n keepers: {}", + cluster.replica_config_path().display(), + cluster.keeper_config_path().display() + ); + let pid_error_msg = "Failed to get process PID, it may not have started"; + println!( + "omicron-dev: ClickHouse cluster is running with: server PIDs = [{}, {}] \ + and keeper PIDs = [{}, {}, {}]", + cluster.replica_1 + .pid() + .expect(pid_error_msg), + cluster.replica_2 + .pid() + .expect(pid_error_msg), + cluster.keeper_1 + .pid() + .expect(pid_error_msg), + cluster.keeper_2 + .pid() + .expect(pid_error_msg), + cluster.keeper_3 + .pid() + .expect(pid_error_msg), + ); + println!( + "omicron-dev: ClickHouse HTTP servers listening on ports: {}, {}", + cluster.replica_1.port(), + cluster.replica_2.port() + ); + println!( + "omicron-dev: using {} and {} for ClickHouse data storage", + cluster.replica_1.data_path(), + cluster.replica_2.data_path() + ); + + // Wait for the replicas and keepers to exit themselves (an error), or for SIGINT + tokio::select! { + _ = cluster.replica_1.wait_for_shutdown() => { + cluster.replica_1.cleanup().await.context( + format!("clean up {} after shutdown", cluster.replica_1.data_path()) + )?; + bail!("omicron-dev: ClickHouse replica 1 shutdown unexpectedly"); + } + _ = cluster.replica_2.wait_for_shutdown() => { + cluster.replica_2.cleanup().await.context( + format!("clean up {} after shutdown", cluster.replica_2.data_path()) + )?; + bail!("omicron-dev: ClickHouse replica 2 shutdown unexpectedly"); + } + _ = cluster.keeper_1.wait_for_shutdown() => { + cluster.keeper_1.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_1.data_path()) + )?; + bail!("omicron-dev: ClickHouse keeper 1 shutdown unexpectedly"); + } + _ = cluster.keeper_2.wait_for_shutdown() => { + cluster.keeper_2.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_2.data_path()) + )?; + bail!("omicron-dev: ClickHouse keeper 2 shutdown unexpectedly"); + } + _ = cluster.keeper_3.wait_for_shutdown() => { + cluster.keeper_3.cleanup().await.context( + format!("clean up {} after shutdown", cluster.keeper_3.data_path()) + )?; + bail!("omicron-dev: ClickHouse keeper 3 shutdown unexpectedly"); + } + caught_signal = signal_stream.next() => { + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directories" + ); + + // Remove the data directories. + let mut instances = vec![ + cluster.replica_1, + cluster.replica_2, + cluster.keeper_1, + cluster.keeper_2, + cluster.keeper_3, + ]; + for instance in instances.iter_mut() { + instance + .wait_for_shutdown() + .await + .context(format!("clean up {} after SIGINT shutdown", instance.data_path()))?; + }; + } + } + Ok(()) +} diff --git a/dev-tools/omicron-dev/src/db.rs b/dev-tools/omicron-dev/src/db.rs new file mode 100644 index 0000000000..4a6bfee866 --- /dev/null +++ b/dev-tools/omicron-dev/src/db.rs @@ -0,0 +1,195 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::{bail, Context, Result}; +use camino::Utf8PathBuf; +use clap::Args; +use futures::stream::StreamExt; +use libc::SIGINT; +use omicron_test_utils::dev; +use signal_hook_tokio::Signals; + +#[derive(Clone, Debug, Args)] +pub(crate) struct DbRunArgs { + /// Path to store database data (default: temp dir cleaned up on exit) + #[clap(long, action)] + store_dir: Option, + + /// Database (SQL) listen port. Use `0` to request any available port. + // We choose an arbitrary default port that's different from the default + // CockroachDB port to avoid conflicting. We don't use 0 because this port + // is specified in a few other places, like the default Nexus config file. + // TODO We could load that file at compile time and use the value there. + #[clap(long, default_value = "32221", action)] + listen_port: u16, + + // This unusual clap configuration makes "populate" default to true, + // allowing a --no-populate override on the CLI. + /// Do not populate the database with any schema + #[clap(long = "no-populate", action(clap::ArgAction::SetFalse))] + populate: bool, +} + +impl DbRunArgs { + pub(crate) async fn exec(&self) -> Result<()> { + // Set ourselves up to wait for SIGINT. It's important to do this early, + // before we've created resources that we want to have cleaned up on SIGINT + // (e.g., the temporary directory created by the database starter). + let signals = + Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + // Now start CockroachDB. This process looks bureaucratic (create arg + // builder, then create starter, then start it) because we want to be able + // to print what's happening before we do it. + let mut db_arg_builder = dev::db::CockroachStarterBuilder::new() + .listen_port(self.listen_port); + + // NOTE: The stdout strings here are not intended to be stable, but they are + // used by the test suite. + + if let Some(store_dir) = &self.store_dir { + println!( + "omicron-dev: using user-provided path for database store: {}", + store_dir, + ); + db_arg_builder = db_arg_builder.store_dir(store_dir); + } else { + println!( + "omicron-dev: using temporary directory for database store \ + (cleaned up on clean exit)" + ); + } + + let db_starter = db_arg_builder.build()?; + println!( + "omicron-dev: will run this to start CockroachDB:\n{}", + db_starter.cmdline() + ); + println!("omicron-dev: environment:"); + for (k, v) in db_starter.environment() { + println!(" {}={}", k, v); + } + println!( + "omicron-dev: temporary directory: {}", + db_starter.temp_dir().display() + ); + + let mut db_instance = db_starter.start().await?; + println!("\nomicron-dev: child process: pid {}", db_instance.pid()); + println!( + "omicron-dev: CockroachDB listening at: {}", + db_instance.listen_url() + ); + + if self.populate { + // Populate the database with our schema. + let start = tokio::time::Instant::now(); + println!("omicron-dev: populating database"); + db_instance.populate().await.context("populating database")?; + let end = tokio::time::Instant::now(); + let duration = end.duration_since(start); + println!( + "omicron-dev: populated database in {}.{} seconds", + duration.as_secs(), + duration.subsec_millis() + ); + } + + // Wait for either the child process to shut down on its own or for us to + // receive SIGINT. + tokio::select! { + _ = db_instance.wait_for_shutdown() => { + db_instance.cleanup().await.context("clean up after shutdown")?; + bail!( + "omicron-dev: database shut down unexpectedly \ + (see error output above)" + ); + } + caught_signal = signal_stream.next() => { + assert_eq!(caught_signal.unwrap(), SIGINT); + + /* + * We don't have to do anything to trigger shutdown because the + * shell will have delivered the same SIGINT that we got to the + * cockroach process as well. + */ + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + db_instance + .wait_for_shutdown() + .await + .context("clean up after SIGINT shutdown")?; + } + } + + Ok(()) + } +} + +#[derive(Debug, Args)] +pub(crate) struct DbPopulateArgs { + /// URL for connecting to the database (postgresql:///...) + #[clap(long, action)] + database_url: String, + + /// Wipe any existing schema (and data!) before populating + #[clap(long, action)] + wipe: bool, +} + +impl DbPopulateArgs { + pub(crate) async fn exec(&self) -> Result<()> { + let config = + self.database_url.parse::().with_context( + || format!("parsing database URL {:?}", self.database_url), + )?; + let client = dev::db::Client::connect(&config, tokio_postgres::NoTls) + .await + .with_context(|| { + format!("connecting to {:?}", self.database_url) + })?; + + if self.wipe { + println!("omicron-dev: wiping any existing database"); + dev::db::wipe(&client).await?; + } + + println!("omicron-dev: populating database"); + dev::db::populate(&client).await?; + println!("omicron-dev: populated database"); + client.cleanup().await.expect("connection failed"); + Ok(()) + } +} + +#[derive(Debug, Args)] +pub(crate) struct DbWipeArgs { + /// URL for connecting to the database (postgresql:///...) + #[clap(long, action)] + database_url: String, +} + +impl DbWipeArgs { + pub(crate) async fn exec(&self) -> Result<()> { + let config = + self.database_url.parse::().with_context( + || format!("parsing database URL {:?}", self.database_url), + )?; + let client = dev::db::Client::connect(&config, tokio_postgres::NoTls) + .await + .with_context(|| { + format!("connecting to {:?}", self.database_url) + })?; + + println!("omicron-dev: wiping any existing database"); + dev::db::wipe(&client).await?; + println!("omicron-dev: wiped"); + client.cleanup().await.expect("connection failed"); + Ok(()) + } +} diff --git a/dev-tools/omicron-dev/src/dispatch.rs b/dev-tools/omicron-dev/src/dispatch.rs new file mode 100644 index 0000000000..87b249d22a --- /dev/null +++ b/dev-tools/omicron-dev/src/dispatch.rs @@ -0,0 +1,126 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Main entry point for the Omicron developer tool. + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +/// Tools for working with a local Omicron deployment +#[derive(Debug, Parser)] +#[clap(version)] +pub struct OmicronDevApp { + #[clap(subcommand)] + command: OmicronDevCmd, +} + +impl OmicronDevApp { + pub async fn exec(self) -> Result<()> { + match self.command { + #[cfg(feature = "include-db")] + OmicronDevCmd::DbRun { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + #[cfg(feature = "include-db")] + OmicronDevCmd::DbPopulate { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + #[cfg(feature = "include-db")] + OmicronDevCmd::DbWipe { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + #[cfg(feature = "include-clickhouse")] + OmicronDevCmd::ChRun { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + #[cfg(feature = "include-mgs")] + OmicronDevCmd::MgsRun { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + #[cfg(feature = "include-nexus")] + OmicronDevCmd::RunAll { args } => { + // This does build Nexus anyway, so there's no point in + // printing the tip. + args.exec().await + } + #[cfg(feature = "include-cert")] + OmicronDevCmd::CertCreate { args } => { + print_xtask_nexus_tip(); + args.exec().await + } + } + } +} + +// A default build implies that omicron-dev was run via `cargo run -p +// omicron-dev`, which can be slow if it has to build all of Nexus. In that +// case, suggest using `cargo xtask omicron-dev` to avoid building Nexus. +#[cfg(feature = "default")] +#[allow(dead_code)] +fn print_xtask_nexus_tip() { + eprintln!("omicron-dev: tip: use `cargo xtask omicron-dev` to avoid building Nexus"); +} + +#[cfg(not(feature = "default"))] +#[allow(dead_code)] +fn print_xtask_nexus_tip() {} + +// NOTE: This enum should stay in sync with dev-tools/xtask/src/omicron_dev.rs. +#[derive(Debug, Subcommand)] +pub(crate) enum OmicronDevCmd { + #[cfg(feature = "include-db")] + /// Start a CockroachDB cluster for development + DbRun { + #[clap(flatten)] + args: crate::db::DbRunArgs, + }, + + #[cfg(feature = "include-db")] + /// Populate an existing CockroachDB cluster with the Omicron schema + DbPopulate { + #[clap(flatten)] + args: crate::db::DbPopulateArgs, + }, + + #[cfg(feature = "include-db")] + /// Wipe the Omicron schema (and all data) from an existing CockroachDB + /// cluster + DbWipe { + #[clap(flatten)] + args: crate::db::DbWipeArgs, + }, + + #[cfg(feature = "include-clickhouse")] + /// Run a ClickHouse database server for development + ChRun { + #[clap(flatten)] + args: crate::clickhouse::ChRunArgs, + }, + + #[cfg(feature = "include-mgs")] + /// Run a simulated Management Gateway Service for development + MgsRun { + #[clap(flatten)] + args: crate::mgs::MgsRunArgs, + }, + + #[cfg(feature = "include-nexus")] + /// Run a full simulated control plane + RunAll { + #[clap(flatten)] + args: crate::nexus::RunAllArgs, + }, + + #[cfg(feature = "include-cert")] + /// Create a self-signed certificate for use with Omicron + CertCreate { + #[clap(flatten)] + args: crate::cert::CertCreateArgs, + }, +} diff --git a/dev-tools/omicron-dev/src/lib.rs b/dev-tools/omicron-dev/src/lib.rs new file mode 100644 index 0000000000..28a0e6cee7 --- /dev/null +++ b/dev-tools/omicron-dev/src/lib.rs @@ -0,0 +1,19 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Developer tool for easily running bits of Omicron. + +#[cfg(feature = "include-cert")] +mod cert; +#[cfg(feature = "include-clickhouse")] +mod clickhouse; +#[cfg(feature = "include-db")] +mod db; +mod dispatch; +#[cfg(feature = "include-mgs")] +mod mgs; +#[cfg(feature = "include-nexus")] +mod nexus; + +pub use dispatch::*; diff --git a/dev-tools/omicron-dev/src/mgs.rs b/dev-tools/omicron-dev/src/mgs.rs new file mode 100644 index 0000000000..4a76fc64b9 --- /dev/null +++ b/dev-tools/omicron-dev/src/mgs.rs @@ -0,0 +1,42 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use clap::Args; +use futures::StreamExt; +use libc::SIGINT; +use signal_hook_tokio::Signals; + +#[derive(Clone, Debug, Args)] +pub(crate) struct MgsRunArgs {} + +impl MgsRunArgs { + pub(crate) async fn exec(&self) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = + Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + println!("omicron-dev: setting up MGS ... "); + let gwtestctx = gateway_test_utils::setup::test_setup( + "omicron-dev", + gateway_messages::SpPort::One, + ) + .await; + println!("omicron-dev: MGS is running."); + + let addr = gwtestctx.client.bind_address; + println!("omicron-dev: MGS API: http://{:?}", addr); + + // Wait for a signal. + let caught_signal = signal_stream.next().await; + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + gwtestctx.teardown().await; + Ok(()) + } +} diff --git a/dev-tools/omicron-dev/src/nexus.rs b/dev-tools/omicron-dev/src/nexus.rs new file mode 100644 index 0000000000..e840adf826 --- /dev/null +++ b/dev-tools/omicron-dev/src/nexus.rs @@ -0,0 +1,126 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::Context; +use clap::Args; +use futures::StreamExt; +use libc::SIGINT; +use nexus_config::NexusConfig; +use nexus_test_interface::NexusServer; +use signal_hook_tokio::Signals; + +#[derive(Clone, Debug, Args)] +pub(crate) struct RunAllArgs { + /// Nexus external API listen port. Use `0` to request any available port. + #[clap(long, action)] + nexus_listen_port: Option, +} + +impl RunAllArgs { + pub(crate) async fn exec(&self) -> Result<(), anyhow::Error> { + // Start a stream listening for SIGINT + let signals = + Signals::new(&[SIGINT]).expect("failed to wait for SIGINT"); + let mut signal_stream = signals.fuse(); + + // Read configuration. + let config_str = include_str!("../../../nexus/examples/config.toml"); + let mut config: NexusConfig = + toml::from_str(config_str).context("parsing example config")?; + config.pkg.log = dropshot::ConfigLogging::File { + // See LogContext::new(), + path: "UNUSED".to_string().into(), + level: dropshot::ConfigLoggingLevel::Trace, + if_exists: dropshot::ConfigLoggingIfExists::Fail, + }; + + if let Some(p) = self.nexus_listen_port { + config + .deployment + .dropshot_external + .dropshot + .bind_address + .set_port(p); + } + + println!("omicron-dev: setting up all services ... "); + let cptestctx = nexus_test_utils::omicron_dev_setup_with_config::< + omicron_nexus::Server, + >(&mut config) + .await + .context("error setting up services")?; + println!("omicron-dev: services are running."); + + // Print out basic information about what was started. + // NOTE: The stdout strings here are not intended to be stable, but they are + // used by the test suite. + let addr = cptestctx.external_client.bind_address; + println!("omicron-dev: nexus external API: {:?}", addr); + println!( + "omicron-dev: nexus internal API: {:?}", + cptestctx.server.get_http_server_internal_address().await, + ); + println!( + "omicron-dev: cockroachdb pid: {}", + cptestctx.database.pid(), + ); + println!( + "omicron-dev: cockroachdb URL: {}", + cptestctx.database.pg_config() + ); + println!( + "omicron-dev: cockroachdb directory: {}", + cptestctx.database.temp_dir().display() + ); + println!( + "omicron-dev: internal DNS HTTP: http://{}", + cptestctx.internal_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: internal DNS: {}", + cptestctx.internal_dns.dns_server.local_address() + ); + println!( + "omicron-dev: external DNS name: {}", + cptestctx.external_dns_zone_name, + ); + println!( + "omicron-dev: external DNS HTTP: http://{}", + cptestctx.external_dns.dropshot_server.local_addr() + ); + println!( + "omicron-dev: external DNS: {}", + cptestctx.external_dns.dns_server.local_address() + ); + println!( + "omicron-dev: e.g. `dig @{} -p {} {}.sys.{}`", + cptestctx.external_dns.dns_server.local_address().ip(), + cptestctx.external_dns.dns_server.local_address().port(), + cptestctx.silo_name, + cptestctx.external_dns_zone_name, + ); + for (location, gateway) in &cptestctx.gateway { + println!( + "omicron-dev: management gateway: http://{} ({})", + gateway.client.bind_address, location, + ); + } + println!("omicron-dev: silo name: {}", cptestctx.silo_name,); + println!( + "omicron-dev: privileged user name: {}", + cptestctx.user_name.as_ref(), + ); + + // Wait for a signal. + let caught_signal = signal_stream.next().await; + assert_eq!(caught_signal.unwrap(), SIGINT); + eprintln!( + "omicron-dev: caught signal, shutting down and removing \ + temporary directory" + ); + + cptestctx.teardown().await; + Ok(()) + } +} diff --git a/dev-tools/xtask/src/check_features.rs b/dev-tools/xtask/src/check_features.rs index a9dbc2bff7..7f0f9865c5 100644 --- a/dev-tools/xtask/src/check_features.rs +++ b/dev-tools/xtask/src/check_features.rs @@ -11,6 +11,20 @@ use std::{collections::HashSet, process::Command}; const SUPPORTED_ARCHITECTURES: [&str; 1] = ["x86_64"]; const CI_EXCLUDED_FEATURES: [&str; 2] = ["image-trampoline", "image-standard"]; +const CI_MUTUALLY_EXCLUSIVE_FEATURES: &[&str] = &[ + // The "include-" features are from omicron-dev. They aren't actually + // mutually exclusive, but we only really care about one feature being + // enabled at a time as well as all at once (that's covered by the default + // feature that already comes with omicron-dev). + // + // These should stay in sync with the list of "include-*" features in + // dev-tools/omicron-dev/Cargo.toml. + "include-cert", + "include-clickhouse", + "include-db", + "include-mgs", + "include-nexus", +]; #[derive(Parser)] pub struct Args { @@ -20,6 +34,9 @@ pub struct Args { /// Features to exclude from the check. #[clap(long, value_name = "FEATURES")] exclude_features: Option>, + /// Features to mark as mutually exclusive. + #[clap(long, value_name = "FEATURES")] + mutually_exclusive_features: Option>, /// Depth of the feature powerset to check. #[clap(long, value_name = "NUM")] depth: Option, @@ -67,12 +84,39 @@ pub fn run_cmd(args: Args) -> Result<()> { // Add the `--exclude-features` flag if we are running in CI mode. command.args(["--exclude-features", &ex]); + + let mutually_exclusive = + if let Some(mut features) = args.mutually_exclusive_features { + // Extend the list of mutually exclusive features with the CI + // defaults. + features.extend( + CI_MUTUALLY_EXCLUSIVE_FEATURES + .into_iter() + .map(|s| s.to_string()), + ); + + // Remove duplicates. + let mutually_exclusive = + features.into_iter().collect::>(); + + mutually_exclusive.into_iter().collect::>().join(",") + } else { + CI_MUTUALLY_EXCLUSIVE_FEATURES.join(",") + }; + + // Add the `--mutually-exclusive-features` flag if it was provided. + command.args(["--mutually-exclusive-features", &mutually_exclusive]); } else { install_cargo_hack(&cargo, args.install_version)?; // Add "only" the `--exclude-features` flag if it was provided. if let Some(features) = args.exclude_features { command.args(["--exclude-features", &features.join(",")]); } + // Add "only" the `--mutually-exclusive-features` flag if it was provided. + if let Some(features) = args.mutually_exclusive_features { + command + .args(["--mutually-exclusive-features", &features.join(",")]); + } } if let Some(depth) = args.depth { diff --git a/dev-tools/xtask/src/external.rs b/dev-tools/xtask/src/external.rs index 05e668297d..998c4291f1 100644 --- a/dev-tools/xtask/src/external.rs +++ b/dev-tools/xtask/src/external.rs @@ -31,6 +31,9 @@ use clap::Parser; disable_version_flag(true) )] pub struct External { + #[clap(skip)] + external_args: Vec, + #[clap(trailing_var_arg(true), allow_hyphen_values(true))] args: Vec, @@ -52,6 +55,17 @@ impl External { self } + /// Add additional arguments for the underlying command. + /// + /// These arguments go after `--` and before `self.args`. + pub fn external_args( + mut self, + args: impl IntoIterator>, + ) -> External { + self.external_args.extend(args.into_iter().map(|s| s.into())); + self + } + pub fn exec_example(self, example_target: impl AsRef) -> Result<()> { self.exec_common("--example", example_target.as_ref()) } @@ -61,8 +75,14 @@ impl External { } fn exec_common(mut self, kind: &'static str, target: &OsStr) -> Result<()> { - let error = - self.command.arg(kind).arg(target).arg("--").args(self.args).exec(); + let error = self + .command + .arg(kind) + .arg(target) + .arg("--") + .args(self.external_args) + .args(self.args) + .exec(); Err(error).context("failed to exec `cargo run`") } } diff --git a/dev-tools/xtask/src/main.rs b/dev-tools/xtask/src/main.rs index 0ea2332c31..c1af1ce809 100644 --- a/dev-tools/xtask/src/main.rs +++ b/dev-tools/xtask/src/main.rs @@ -16,6 +16,7 @@ mod clippy; mod download; #[cfg_attr(not(target_os = "illumos"), allow(dead_code))] mod external; +mod omicron_dev; mod usdt; #[cfg(target_os = "illumos")] @@ -54,6 +55,9 @@ enum Cmds { /// For more information, see dev-tools/openapi-manager/README.adoc. Openapi(external::External), + /// Run Omicron development tasks + OmicronDev(omicron_dev::OmicronDevArgs), + #[cfg(target_os = "illumos")] /// Build a TUF repo Releng(external::External), @@ -98,6 +102,7 @@ async fn main() -> Result<()> { Cmds::CheckWorkspaceDeps => check_workspace_deps::run_cmd(), Cmds::Download(args) => download::run_cmd(args).await, Cmds::Openapi(external) => external.exec_bin("openapi-manager"), + Cmds::OmicronDev(args) => omicron_dev::run_cmd(args).await, #[cfg(target_os = "illumos")] Cmds::Releng(external) => { external.cargo_args(["--release"]).exec_bin("omicron-releng") diff --git a/dev-tools/xtask/src/omicron_dev.rs b/dev-tools/xtask/src/omicron_dev.rs new file mode 100644 index 0000000000..73a60e5058 --- /dev/null +++ b/dev-tools/xtask/src/omicron_dev.rs @@ -0,0 +1,78 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Developer tool for easily running bits of Omicron. + +use anyhow::Result; +use clap::{Args, Subcommand}; + +use crate::external; + +#[derive(Args)] +pub(crate) struct OmicronDevArgs { + /// Run the Omicron dev tool. + #[clap(subcommand)] + cmd: OmicronDevCmd, +} + +#[derive(Subcommand)] +enum OmicronDevCmd { + /// Start a CockroachDB cluster for development + DbRun(external::External), + + /// Populate an existing CockroachDB cluster with the Omicron schema + DbPopulate(external::External), + + /// Wipe the Omicron schema (and all data) from an existing CockroachDB + /// cluster + DbWipe(external::External), + + /// Run a ClickHouse database server for development + ChRun(external::External), + + /// Run a simulated Management Gateway Service for development + MgsRun(external::External), + + /// Run a full simulated control plane + RunAll(external::External), + + /// Create a self-signed certificate for use with Omicron + CertCreate(external::External), +} + +pub(crate) async fn run_cmd(args: OmicronDevArgs) -> Result<()> { + match args.cmd { + OmicronDevCmd::DbRun(args) => { + run_omicron_dev(args, "include-db", "db-run") + } + OmicronDevCmd::DbPopulate(args) => { + run_omicron_dev(args, "include-db", "db-populate") + } + OmicronDevCmd::DbWipe(args) => { + run_omicron_dev(args, "include-db", "db-wipe") + } + OmicronDevCmd::ChRun(args) => { + run_omicron_dev(args, "include-clickhouse", "ch-run") + } + OmicronDevCmd::MgsRun(args) => { + run_omicron_dev(args, "include-mgs", "mgs-run") + } + OmicronDevCmd::RunAll(args) => { + run_omicron_dev(args, "include-nexus", "run-all") + } + OmicronDevCmd::CertCreate(args) => { + run_omicron_dev(args, "include-cert", "cert-create") + } + } +} + +fn run_omicron_dev( + args: external::External, + feature: impl AsRef, + command: impl AsRef, +) -> Result<()> { + args.cargo_args(["--no-default-features", "--features", feature.as_ref()]) + .external_args([command.as_ref()]) + .exec_bin("omicron-dev") +} diff --git a/docs/how-to-run-simulated.adoc b/docs/how-to-run-simulated.adoc index 86f7a0915b..d97880f1de 100644 --- a/docs/how-to-run-simulated.adoc +++ b/docs/how-to-run-simulated.adoc @@ -52,8 +52,8 @@ You don't need to do this again if you just did it. But you'll need to do it ea To **run Omicron** you need to run several programs: -* a CockroachDB cluster. For development, you can use the `omicron-dev` tool in this repository to start a single-node CockroachDB cluster **that will delete the database when you shut it down.** -* a ClickHouse server. You should use the `omicron-dev` tool for this as well, see below, and as with CockroachDB, +* a CockroachDB cluster. For development, you can use the `cargo xtask omicron-dev` tool in this repository to start a single-node CockroachDB cluster **that will delete the database when you shut it down.** +* a ClickHouse server. You should use the `cargo xtask omicron-dev` tool for this as well, see below, and as with CockroachDB, the database files will be deleted when you stop the program. * `nexus`: the guts of the control plane * `sled-agent-sim`: a simulator for the component that manages a single sled @@ -64,11 +64,11 @@ You can run these by hand, but it's easier to use `omicron-dev run-all`. See be === Quick start -. Run `omicron-dev run-all`. This will run all of these components with a default configuration that should work in a typical development environment. The database will be stored in a temporary directory. Logs for all services will go to a single, unified log file. The tool will print information about reaching Nexus as well as CockroachDB: +. Run `cargo xtask omicron-dev run-all`. This will run all of these components with a default configuration that should work in a typical development environment. The database will be stored in a temporary directory. Logs for all services will go to a single, unified log file. The tool will print information about reaching Nexus as well as CockroachDB: + [source,text] ---- -$ omicron-dev run-all +$ cargo xtask omicron-dev run-all omicron-dev: setting up all services ... log file: /dangerzone/omicron_tmp/omicron-dev-omicron-dev.4647.0.log note: configured to log to "/dangerzone/omicron_tmp/omicron-dev-omicron-dev.4647.0.log" @@ -98,11 +98,11 @@ There are many reasons it's useful to run the pieces of the stack by hand, espec CAUTION: This process does not currently work. See https://github.com/oxidecomputer/omicron/issues/4421[omicron#4421] for details. The pieces here may still be useful for reference. -. Start CockroachDB using `omicron-dev db-run`: +. Start CockroachDB using `cargo xtask omicron-dev db-run`: + [source,text] ---- -$ cargo run --bin=omicron-dev -- db-run +$ cargo xtask omicron-dev db-run Finished dev [unoptimized + debuginfo] target(s) in 0.15s Running `target/debug/omicron-dev db-run` omicron-dev: using temporary directory for database store (cleaned up on clean exit) @@ -157,7 +157,7 @@ Note that as the output indicates, this cluster will be available to anybody tha + [source,text] ---- -$ cargo run --bin omicron-dev -- ch-run +$ cargo xtask omicron-dev ch-run Finished dev [unoptimized + debuginfo] target(s) in 0.47s Running `target/debug/omicron-dev ch-run` omicron-dev: running ClickHouse (PID: 2463), full command is "clickhouse server --log-file /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot/clickhouse-server.log --errorlog-file /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot/clickhouse-server.errlog -- --http_port 8123 --path /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot" @@ -167,7 +167,7 @@ omicron-dev: using /var/folders/67/2tlym22x1r3d2kwbh84j298w0000gn/T/.tmpJ5nhot f If you wish to start a ClickHouse replicated cluster instead of a single node, run the following instead: [source,text] --- -$ cargo run --bin omicron-dev -- ch-run --replicated +$ cargo xtask omicron-dev ch-run --replicated Finished dev [unoptimized + debuginfo] target(s) in 0.31s Running `target/debug/omicron-dev ch-run --replicated` omicron-dev: running ClickHouse cluster with configuration files: @@ -217,11 +217,11 @@ Dec 02 18:00:01.093 DEBG registered endpoint, path: /producers, method: POST, lo While it's often useful to run _some_ part of the stack by hand (see above), if you only want to run your own Nexus, one option is to run `omicron-dev run-all` first to get a whole simulated stack up, then run a second Nexus by hand with a custom config file. -To do this, first run `omicron-dev run-all`: +To do this, first run `cargo xtask omicron-dev run-all`: [source,text] ---- -$ cargo run --bin=omicron-dev -- run-all +$ cargo xtask omicron-dev run-all Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.95s Running `target/debug/omicron-dev run-all` omicron-dev: setting up all services ... @@ -316,7 +316,7 @@ When you run the above, you will wind up with Nexus listening on HTTP (with no T + [source,text] ---- -$ cargo run --bin=omicron-dev -- cert-create demo- '*.sys.oxide-dev.test' +$ cargo xtask omicron-dev cert-create demo- '*.sys.oxide-dev.test' wrote certificate to demo-cert.pem wrote private key to demo-key.pem ---- diff --git a/docs/how-to-run.adoc b/docs/how-to-run.adoc index 9bd99c23d3..087dc2e2c7 100644 --- a/docs/how-to-run.adoc +++ b/docs/how-to-run.adoc @@ -244,7 +244,7 @@ You can skip this step. In that case, the externally-facing services (API and c You can generate a self-signed TLS certificate chain with: ---- -$ cargo run --bin=omicron-dev -- cert-create ./smf/sled-agent/$MACHINE/initial-tls- '*.sys.oxide.test' +$ cargo xtask omicron-dev cert-create ./smf/sled-agent/$MACHINE/initial-tls- '*.sys.oxide.test' ---- === Rack setup configuration diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index fcb14a4f15..c1fb3b4a3b 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -653,7 +653,7 @@ impl Drop for CockroachInstance { "WARN: temporary directory leaked: {path:?}\n\ \tIf you would like to access the database for debugging, run the following:\n\n\ \t# Run the database\n\ - \tcargo run --bin omicron-dev db-run --no-populate --store-dir {data_path:?}\n\ + \tcargo xtask omicron-dev db-run --no-populate --store-dir {data_path:?}\n\ \t# Access the database. Note the port may change if you run multiple databases.\n\ \tcockroach sql --host=localhost:32221 --insecure", data_path = path.join("data"), diff --git a/wicket/README.md b/wicket/README.md index fc1c93fe83..b7ee49b5ca 100644 --- a/wicket/README.md +++ b/wicket/README.md @@ -127,29 +127,15 @@ Making this simpler is tracked in The easiest way to do this is to run: ``` -cargo run -p omicron-dev mgs-run +cargo xtask omicron-dev mgs-run ``` This will print out a line similar to `omicron-dev: MGS API: http://[::1]:12225`. Note the address for use below. -Another option, which may lead to quicker iteration cycles if you're modifying -MGS or sp-sim, is to run the services by hand from the root of omicron: - -``` -cargo run --bin sp-sim -- sp-sim/examples/config.toml -cargo run --bin mgs run --id c19a698f-c6f9-4a17-ae30-20d711b8f7dc --address '[::1]:12225' gateway/examples/config.toml -``` - -The port number in `--address` is arbitrary. - -**Note:** If you're adding new functionality to wicket, it is quite possible -that sp-sim is missing support for it! Generally, sp-sim has features added to -it on an as-needed basis. - ### Using a real SP The easiest way is to change the mgs config to point to a running SP instead -of a simulated SP +of a simulated SP. ``` [[switch.port]]