Merge branch 'main' into clickhouse-server-smf

Cargo.toml

@@ -362,6 +362,10 @@ headers = "0.3.9"
 heck = "0.5"
 hex = "0.4.3"
 hex-literal = "0.4.1"
+hickory-client = "0.24.1"
+hickory-proto = "0.24.1"
+hickory-resolver = "0.24.1"
+hickory-server = "0.24.1"
 highway = "1.2.0"
 hkdf = "0.12.4"
 http = "0.2.12"
@@ -373,7 +377,7 @@ hyper-rustls = "0.26.0"
 hyper-staticfile = "0.9.5"
 illumos-utils = { path = "illumos-utils" }
 indent_write = "2.2.0"
-indexmap = "2.3.0"
+indexmap = "2.4.0"
 indicatif = { version = "0.17.8", features = ["rayon"] }
 installinator = { path = "installinator" }
 installinator-api = { path = "installinator-api" }
@@ -507,7 +511,7 @@ secrecy = "0.8.0"
 semver = { version = "1.0.23", features = ["std", "serde"] }
 serde = { version = "1.0", default-features = false, features = [ "derive", "rc" ] }
 serde_human_bytes = { git = "https://github.com/oxidecomputer/serde_human_bytes", branch = "main" }
-serde_json = "1.0.124"
+serde_json = "1.0.125"
 serde_path_to_error = "0.1.16"
 serde_tokenstream = "0.2"
 serde_urlencoded = "0.7.1"
@@ -572,10 +576,6 @@ tokio-util = { version = "0.7.11", features = ["io", "io-util"] }
 toml = "0.8.19"
 toml_edit = "0.22.20"
 tough = { version = "0.17.1", features = [ "http" ] }
-trust-dns-client = "0.22"
-trust-dns-proto = "0.22"
-trust-dns-resolver = "0.22"
-trust-dns-server = "0.22"
 trybuild = "1.0.99"
 tufaceous = { path = "tufaceous" }
 tufaceous-lib = { path = "tufaceous-lib" }
@@ -734,8 +734,6 @@ opt-level = 3
 opt-level = 3
 [profile.dev.package.rand_core]
 opt-level = 3
-[profile.dev.package.rand_hc]
-opt-level = 3
 [profile.dev.package.rand_xorshift]
 opt-level = 3
 [profile.dev.package.rsa]

clients/oxide-client/Cargo.toml

@@ -12,6 +12,7 @@ anyhow.workspace = true
 base64.workspace = true
 chrono.workspace = true
 futures.workspace = true
+hickory-resolver.workspace = true
 http.workspace = true
 hyper.workspace = true
 progenitor.workspace = true
@@ -22,6 +23,5 @@ serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
 tokio = { workspace = true, features = [ "net" ] }
-trust-dns-resolver.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true

clients/oxide-client/src/lib.rs

@@ -7,13 +7,13 @@
 use anyhow::anyhow;
 use anyhow::Context;
 use futures::FutureExt;
+use hickory_resolver::config::{
+    NameServerConfig, Protocol, ResolverConfig, ResolverOpts,
+};
+use hickory_resolver::TokioAsyncResolver;
 use std::net::SocketAddr;
 use std::sync::Arc;
 use thiserror::Error;
-use trust_dns_resolver::config::{
-    NameServerConfig, Protocol, ResolverConfig, ResolverOpts,
-};
-use trust_dns_resolver::TokioAsyncResolver;
 
 progenitor::generate_api!(
     spec = "../../openapi/nexus.json",
@@ -46,14 +46,15 @@ impl CustomDnsResolver {
             socket_addr: dns_addr,
             protocol: Protocol::Udp,
             tls_dns_name: None,
-            trust_nx_responses: false,
+            trust_negative_responses: false,
             bind_addr: None,
         });
+        let mut resolver_opts = ResolverOpts::default();
+        // Enable edns for potentially larger records
+        resolver_opts.edns0 = true;
 
-        let resolver = Arc::new(
-            TokioAsyncResolver::tokio(resolver_config, ResolverOpts::default())
-                .context("failed to create resolver")?,
-        );
+        let resolver =
+            Arc::new(TokioAsyncResolver::tokio(resolver_config, resolver_opts));
         Ok(CustomDnsResolver { dns_addr, resolver })
     }
 

common/src/address.rs

@@ -8,6 +8,7 @@
 //! and Nexus, who need to agree upon addressing schemes.
 
 use crate::api::external::{self, Error};
+use crate::policy::{DNS_REDUNDANCY, MAX_DNS_REDUNDANCY};
 use ipnetwork::Ipv6Network;
 use once_cell::sync::Lazy;
 use oxnet::{Ipv4Net, Ipv6Net};
@@ -25,31 +26,6 @@ pub const MAX_PORT: u16 = u16::MAX;
 /// minimum possible value for a tcp or udp port
 pub const MIN_PORT: u16 = u16::MIN;
 
-/// The amount of redundancy for boundary NTP servers.
-pub const BOUNDARY_NTP_REDUNDANCY: usize = 2;
-
-/// The amount of redundancy for Nexus services.
-///
-/// This is used by both RSS (to distribute the initial set of services) and the
-/// Reconfigurator (to know whether to add new Nexus zones)
-pub const NEXUS_REDUNDANCY: usize = 3;
-
-/// The amount of redundancy for CockroachDb services.
-///
-/// This is used by both RSS (to distribute the initial set of services) and the
-/// Reconfigurator (to know whether to add new crdb zones)
-pub const COCKROACHDB_REDUNDANCY: usize = 5;
-
-/// The amount of redundancy for internal DNS servers.
-///
-/// Must be less than or equal to MAX_DNS_REDUNDANCY.
-pub const DNS_REDUNDANCY: usize = 3;
-
-/// The maximum amount of redundancy for DNS servers.
-///
-/// This determines the number of addresses which are reserved for DNS servers.
-pub const MAX_DNS_REDUNDANCY: usize = 5;
-
 pub const DNS_PORT: u16 = 53;
 pub const DNS_HTTP_PORT: u16 = 5353;
 pub const SLED_AGENT_PORT: u16 = 12345;

common/src/lib.rs

@@ -26,6 +26,7 @@ pub mod backoff;
 pub mod cmd;
 pub mod disk;
 pub mod ledger;
+pub mod policy;
 pub mod progenitor_operation_retry;
 pub mod update;
 pub mod vlan;

common/src/policy.rs

@@ -0,0 +1,40 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Fleet policy related functionality used by both Reconfigurator and RSS.
+
+/// The amount of redundancy for boundary NTP servers.
+pub const BOUNDARY_NTP_REDUNDANCY: usize = 2;
+
+/// The amount of redundancy for Nexus services.
+///
+/// This is used by both RSS (to distribute the initial set of services) and the
+/// Reconfigurator (to know whether to add new Nexus zones)
+pub const NEXUS_REDUNDANCY: usize = 3;
+
+/// The amount of redundancy for CockroachDb services.
+///
+/// This is used by both RSS (to distribute the initial set of services) and the
+/// Reconfigurator (to know whether to add new crdb zones)
+pub const COCKROACHDB_REDUNDANCY: usize = 5;
+
+/// The amount of redundancy for internal DNS servers.
+///
+/// Must be less than or equal to MAX_DNS_REDUNDANCY.
+pub const DNS_REDUNDANCY: usize = 3;
+
+/// The maximum amount of redundancy for DNS servers.
+///
+/// This determines the number of addresses which are reserved for DNS servers.
+pub const MAX_DNS_REDUNDANCY: usize = 5;
+
+/// The amount of redundancy for clickhouse servers
+///
+/// Clickhouse servers contain lazily replicated data
+pub const CLICKHOUSE_SERVER_REDUNDANCY: usize = 3;
+
+/// The amount of redundancy for clickhouse keepers
+///
+/// Keepers maintain strongly consistent metadata about data replication
+pub const CLICKHOUSE_KEEPER_REDUNDANCY: usize = 5;

dev-tools/omdb/Cargo.toml

@@ -28,6 +28,7 @@ gateway-messages.workspace = true
 gateway-test-utils.workspace = true
 humantime.workspace = true
 internal-dns.workspace = true
+itertools.workspace = true
 nexus-client.workspace = true
 nexus-config.workspace = true
 nexus-db-model.workspace = true

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -19,6 +19,7 @@ use clap::Subcommand;
 use clap::ValueEnum;
 use futures::future::try_join;
 use futures::TryStreamExt;
+use itertools::Itertools;
 use nexus_client::types::ActivationReason;
 use nexus_client::types::BackgroundTask;
 use nexus_client::types::BackgroundTasksActivateRequest;
@@ -33,6 +34,7 @@ use nexus_saga_recovery::LastPass;
 use nexus_types::deployment::Blueprint;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
+use nexus_types::internal_api::background::RegionSnapshotReplacementGarbageCollectStatus;
 use nexus_types::internal_api::background::RegionSnapshotReplacementStartStatus;
 use nexus_types::inventory::BaseboardId;
 use omicron_uuid_kinds::CollectionUuid;
@@ -46,6 +48,7 @@ use reedline::Reedline;
 use serde::Deserialize;
 use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::str::FromStr;
 use tabled::Tabled;
 use uuid::Uuid;
@@ -93,11 +96,21 @@ enum BackgroundTasksCommands {
     /// Print a summary of the status of all background tasks
     List,
     /// Print human-readable summary of the status of each background task
-    Show,
+    Show(BackgroundTasksShowArgs),
     /// Activate one or more background tasks
     Activate(BackgroundTasksActivateArgs),
 }
 
+#[derive(Debug, Args)]
+struct BackgroundTasksShowArgs {
+    /// Names of background tasks to show (default: all)
+    ///
+    /// You can use any background task name here or one of the special strings
+    /// "all", "dns_external", or "dns_internal".
+    #[clap(value_name = "TASK_NAME")]
+    tasks: Vec<String>,
+}
+
 #[derive(Debug, Args)]
 struct BackgroundTasksActivateArgs {
     /// Name of the background tasks to activate
@@ -361,8 +374,8 @@ impl NexusArgs {
                 command: BackgroundTasksCommands::List,
             }) => cmd_nexus_background_tasks_list(&client).await,
             NexusCommands::BackgroundTasks(BackgroundTasksArgs {
-                command: BackgroundTasksCommands::Show,
-            }) => cmd_nexus_background_tasks_show(&client).await,
+                command: BackgroundTasksCommands::Show(args),
+            }) => cmd_nexus_background_tasks_show(&client, args).await,
             NexusCommands::BackgroundTasks(BackgroundTasksArgs {
                 command: BackgroundTasksCommands::Activate(args),
             }) => {
@@ -523,7 +536,9 @@ async fn cmd_nexus_background_tasks_list(
 ) -> Result<(), anyhow::Error> {
     let response =
         client.bgtask_list().await.context("listing background tasks")?;
-    let tasks = response.into_inner();
+    // Convert the HashMap to a BTreeMap because we want the keys in sorted
+    // order.
+    let tasks = response.into_inner().into_iter().collect::<BTreeMap<_, _>>();
     let table_rows = tasks.values().map(BackgroundTaskStatusRow::from);
     let table = tabled::Table::new(table_rows)
         .with(tabled::settings::Style::empty())
@@ -536,6 +551,7 @@ async fn cmd_nexus_background_tasks_list(
 /// Runs `omdb nexus background-tasks show`
 async fn cmd_nexus_background_tasks_show(
     client: &nexus_client::Client,
+    args: &BackgroundTasksShowArgs,
 ) -> Result<(), anyhow::Error> {
     let response =
         client.bgtask_list().await.context("listing background tasks")?;
@@ -544,8 +560,50 @@ async fn cmd_nexus_background_tasks_show(
     let mut tasks =
         response.into_inner().into_iter().collect::<BTreeMap<_, _>>();
 
-    // We want to pick the order that we print some tasks intentionally.  Then
-    // we want to print anything else that we find.
+    // Now, pick out the tasks that the user selected.
+    //
+    // The set of user tasks may include:
+    //
+    // - nothing at all, in which case we include all tasks
+    // - individual task names
+    // - certain groups that we recognize, like "dns_external" for all the tasks
+    //   related to external DNS propagation.  "all" means "all tasks".
+    let selected_set: BTreeSet<_> =
+        args.tasks.iter().map(AsRef::as_ref).collect();
+    let selected_all = selected_set.is_empty() || selected_set.contains("all");
+    if !selected_all {
+        for s in &selected_set {
+            if !tasks.contains_key(*s)
+                && *s != "all"
+                && *s != "dns_external"
+                && *s != "dns_internal"
+            {
+                bail!(
+                    "unknown task name: {:?} (known task names: all, \
+                    dns_external, dns_internal, {})",
+                    s,
+                    tasks.keys().join(", ")
+                );
+            }
+        }
+
+        tasks.retain(|k, _| {
+            selected_set.contains(k.as_str())
+                || selected_set.contains("all")
+                || (selected_set.contains("dns_external")
+                    && k.starts_with("dns_")
+                    && k.ends_with("_external"))
+                || (selected_set.contains("dns_internal")
+                    && k.starts_with("dns_")
+                    && k.ends_with("_internal"))
+        });
+    }
+
+    // Some tasks should be grouped and printed together in a certain order,
+    // even though their names aren't alphabetical.  Notably, the DNS tasks
+    // logically go from config -> servers -> propagation, so we want to print
+    // them in that order.  So we pick these out first and then print anything
+    // else that we find in alphabetical order.
     for name in [
         "dns_config_internal",
         "dns_servers_internal",
@@ -559,7 +617,7 @@ async fn cmd_nexus_background_tasks_show(
     ] {
         if let Some(bgtask) = tasks.remove(name) {
             print_task(&bgtask);
-        } else {
+        } else if selected_all {
             eprintln!("warning: expected to find background task {:?}", name);
         }
     }
@@ -1395,7 +1453,7 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                 }
             }
         };
-    } else if name == "region_snapshot_replacement" {
+    } else if name == "region_snapshot_replacement_start" {
         match serde_json::from_value::<RegionSnapshotReplacementStartStatus>(
             details.clone(),
         ) {
@@ -1421,6 +1479,31 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
                     println!("    > {line}");
                 }
 
+                println!("    errors: {}", status.errors.len());
+                for line in &status.errors {
+                    println!("    > {line}");
+                }
+            }
+        }
+    } else if name == "region_snapshot_replacement_garbage_collection" {
+        match serde_json::from_value::<
+            RegionSnapshotReplacementGarbageCollectStatus,
+        >(details.clone())
+        {
+            Err(error) => eprintln!(
+                "warning: failed to interpret task details: {:?}: {:?}",
+                error, details
+            ),
+
+            Ok(status) => {
+                println!(
+                    "    total garbage collections requested: {}",
+                    status.garbage_collect_requested.len(),
+                );
+                for line in &status.garbage_collect_requested {
+                    println!("    > {line}");
+                }
+
                 println!("    errors: {}", status.errors.len());
                 for line in &status.errors {
                     println!("    > {line}");