crdb v22.2 prep (R9)

.github/buildomat/build-and-test.sh

@@ -83,6 +83,23 @@ ptime -m cargo build -Z unstable-options --timings=json --workspace --tests --lo
 banner test
 ptime -m timeout 2h cargo nextest run --profile ci --locked --verbose
 
+#
+# If upgraded CockroachDB clusters are running on a different cluster version
+# than newly-initialized ones, run database-related tests from the older cluster
+# version.
+#
+if cmp tools/cockroachdb_version tools/prev_cockroachdb_version; then
+    # files are the same
+    true
+else
+    # if return code is > 1, cmp failed
+    [[ $? -gt 1 ]] && exit 2
+    # otherwise, files differ
+    cargo xtask download cockroach-prev
+    CRDB_SEED_USE_PREV=yes ptime -m timeout 1h cargo nextest run --profile ci --locked --verbose \
+        --filter-expr 'rdeps(nexus-test-utils) - package(omicron-dev)'
+fi
+
 #
 # https://github.com/nextest-rs/nextest/issues/16
 #

dev-tools/omdb/Cargo.toml

@@ -58,13 +58,14 @@ multimap.workspace = true
 indicatif.workspace = true
 
 [dev-dependencies]
+camino-tempfile.workspace = true
 expectorate.workspace = true
+http.workspace = true
 nexus-test-utils.workspace = true
 nexus-test-utils-macros.workspace = true
 omicron-nexus.workspace = true
 omicron-test-utils.workspace = true
 subprocess.workspace = true
-camino-tempfile.workspace = true
 
 # Disable doc builds by default for our binaries to work around issue
 # rust-lang/cargo#8373.  These docs would not be very useful anyway.

dev-tools/omdb/tests/successes.out

@@ -602,8 +602,8 @@ WARNING: Zones exist without physical disks!
 
 
  COCKROACHDB SETTINGS:
-    state fingerprint:::::::::::::::::   d4d87aa2ad877a4cc2fddd0573952362739110de
-    cluster.preserve_downgrade_option:   "22.1"
+    state fingerprint:::::::::::::::::   <COCKROACHDB_FINGERPRINT_REDACTED>
+    cluster.preserve_downgrade_option:   <COCKROACHDB_VERSION_REDACTED>
 
  METADATA:
     created by:::::::::::   nexus-test-utils
@@ -640,8 +640,8 @@ WARNING: Zones exist without physical disks!
 
 
  COCKROACHDB SETTINGS:
-    state fingerprint:::::::::::::::::   d4d87aa2ad877a4cc2fddd0573952362739110de
-    cluster.preserve_downgrade_option:   "22.1"
+    state fingerprint:::::::::::::::::   <COCKROACHDB_FINGERPRINT_REDACTED>
+    cluster.preserve_downgrade_option:   <COCKROACHDB_VERSION_REDACTED>
 
  METADATA:
     created by:::::::::::   nexus-test-utils
@@ -681,8 +681,8 @@ to:   blueprint ......<REDACTED_BLUEPRINT_ID>.......
 
 
  COCKROACHDB SETTINGS:
-    state fingerprint:::::::::::::::::   d4d87aa2ad877a4cc2fddd0573952362739110de (unchanged)
-    cluster.preserve_downgrade_option:   "22.1" (unchanged)
+    state fingerprint:::::::::::::::::   <COCKROACHDB_FINGERPRINT_REDACTED> (unchanged)
+    cluster.preserve_downgrade_option:   <COCKROACHDB_VERSION_REDACTED> (unchanged)
 
  METADATA:
     internal DNS version:   1 (unchanged)

dev-tools/omdb/tests/test_all_output.rs

@@ -7,8 +7,11 @@
 //! Feel free to change the tool's output.  This test just makes it easy to make
 //! sure you're only breaking what you intend.
 
+use dropshot::Method;
 use expectorate::assert_contents;
+use http::StatusCode;
 use nexus_test_utils_macros::nexus_test;
+use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::SledFilter;
 use nexus_types::deployment::UnstableReconfiguratorState;
 use omicron_test_utils::dev::test_cmds::path_to_executable;
@@ -83,6 +86,20 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
     let tmppath = tmpdir.path().join("reconfigurator-save.out");
     let initial_blueprint_id = cptestctx.initial_blueprint_id.to_string();
 
+    // Get the CockroachDB metadata from the blueprint so we can redact it
+    let initial_blueprint: Blueprint = dropshot::test_util::read_json(
+        &mut cptestctx
+            .internal_client
+            .make_request_no_body(
+                Method::GET,
+                &format!("/deployment/blueprints/all/{initial_blueprint_id}"),
+                StatusCode::OK,
+            )
+            .await
+            .unwrap(),
+    )
+    .await;
+
     let mut output = String::new();
 
     let invocations: &[&[&str]] = &[
@@ -119,6 +136,19 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
         // ControlPlaneTestContext.
     ];
 
+    let mut redactions = ExtraRedactions::new();
+    redactions
+        .variable_length("tmp_path", tmppath.as_str())
+        .fixed_length("blueprint_id", &initial_blueprint_id)
+        .variable_length(
+            "cockroachdb_fingerprint",
+            &initial_blueprint.cockroachdb_fingerprint,
+        );
+    let crdb_version =
+        initial_blueprint.cockroachdb_setting_preserve_downgrade.to_string();
+    if initial_blueprint.cockroachdb_setting_preserve_downgrade.is_set() {
+        redactions.variable_length("cockroachdb_version", &crdb_version);
+    }
     for args in invocations {
         println!("running commands with args: {:?}", args);
         let p = postgres_url.to_string();
@@ -133,9 +163,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
             },
             &cmd_path,
             args,
-            ExtraRedactions::new()
-                .variable_length("tmp_path", tmppath.as_str())
-                .fixed_length("blueprint_id", &initial_blueprint_id),
+            &redactions,
         )
         .await;
     }

dev-tools/xtask/src/download.rs

@@ -22,6 +22,7 @@ use strum::IntoEnumIterator;
 use tar::Archive;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::process::Command;
+use tokio::sync::Mutex;
 
 const BUILDOMAT_URL: &'static str =
     "https://buildomat.eng.oxide.computer/public/file";
@@ -50,6 +51,9 @@ enum Target {
     /// CockroachDB binary
     Cockroach,
 
+    /// CockroachDB binary (previous major version)
+    CockroachPrev,
+
     /// Web console assets
     Console,
 
@@ -125,7 +129,8 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> {
                         bail!("We should have already filtered this 'All' target out?");
                     }
                     Target::Clickhouse => downloader.download_clickhouse().await,
-                    Target::Cockroach => downloader.download_cockroach().await,
+                    Target::Cockroach => downloader.download_cockroach("").await,
+                    Target::CockroachPrev => downloader.download_cockroach("prev_").await,
                     Target::Console => downloader.download_console().await,
                     Target::DendriteOpenapi => {
                         downloader.download_dendrite_openapi().await
@@ -197,43 +202,52 @@ impl<'a> Downloader<'a> {
     }
 }
 
-/// Parses a file of the format:
-///
-/// ```ignore
-/// KEY1="value1"
-/// KEY2="value2"
-/// ```
-///
-/// And returns an array of the values in the same order as keys.
-async fn get_values_from_file<const N: usize>(
-    keys: [&str; N],
-    path: &Utf8Path,
-) -> Result<[String; N]> {
-    // Map of "key" => "Position in output".
-    let mut keys: HashMap<&str, usize> =
-        keys.into_iter().enumerate().map(|(i, s)| (s, i)).collect();
-
-    const EMPTY_STRING: String = String::new();
-    let mut values = [EMPTY_STRING; N];
-
-    let content = tokio::fs::read_to_string(&path)
-        .await
-        .context("Failed to read {path}")?;
-    for line in content.lines() {
-        let line = line.trim();
-        let Some((key, value)) = line.split_once('=') else {
-            continue;
-        };
-        let value = value.trim_matches('"');
-        if let Some(i) = keys.remove(key) {
-            values[i] = value.to_string();
+macro_rules! get_values_fn {
+    ($name:ident, $ty:ty, $def:expr, $check:expr) => {
+        /// Parses a file of the format:
+        ///
+        /// ```ignore
+        /// KEY1="value1"
+        /// KEY2="value2"
+        /// ```
+        ///
+        /// And returns an array of the values in the same order as keys.
+        async fn $name<const N: usize>(
+            keys: [&str; N],
+            path: &Utf8Path,
+        ) -> Result<[$ty; N]> {
+            // Map of "key" => "Position in output".
+            let mut keys: HashMap<&str, usize> =
+                keys.into_iter().enumerate().map(|(i, s)| (s, i)).collect();
+
+            const DEFAULT: $ty = $def;
+            let mut values = [DEFAULT; N];
+
+            let content = tokio::fs::read_to_string(&path)
+                .await
+                .context("Failed to read {path}")?;
+            for line in content.lines() {
+                let line = line.trim();
+                let Some((key, value)) = line.split_once('=') else {
+                    continue;
+                };
+                let value = value.trim_matches('"');
+                if let Some(i) = keys.remove(key) {
+                    values[i] = value.to_string().into();
+                }
+            }
+            if $check && !keys.is_empty() {
+                bail!(
+                    "Could not find keys: {:?}",
+                    keys.keys().collect::<Vec<_>>(),
+                );
+            }
+            Ok(values)
         }
-    }
-    if !keys.is_empty() {
-        bail!("Could not find keys: {:?}", keys.keys().collect::<Vec<_>>(),);
-    }
-    Ok(values)
+    };
 }
+get_values_fn!(get_values_from_file, String, String::new(), true);
+get_values_fn!(try_get_values_from_file, Option<String>, None, false);
 
 /// Send a GET request to `url`, downloading the contents to `path`.
 ///
@@ -486,34 +500,52 @@ impl<'a> Downloader<'a> {
         Ok(())
     }
 
-    async fn download_cockroach(&self) -> Result<()> {
+    async fn download_cockroach(&self, prefix: &str) -> Result<()> {
         let os = os_name()?;
 
         let download_dir = self.output_dir.join("downloads");
-        let destination_dir = self.output_dir.join("cockroachdb");
+        let destination_dir =
+            self.output_dir.join(format!("{prefix}cockroachdb"));
 
-        let checksums_path = self.versions_dir.join("cockroachdb_checksums");
-        let [checksum] = get_values_from_file(
+        let checksums_path =
+            self.versions_dir.join(format!("{prefix}cockroachdb_checksums"));
+        let [mut checksum] = get_values_from_file(
             [&format!("CIDL_SHA256_{}", os.env_name())],
             &checksums_path,
         )
         .await?;
+        let mut build = match os {
+            Os::Illumos => "illumos",
+            Os::Linux => "linux-amd64",
+            Os::Mac => "darwin-10.9-amd64",
+        };
+
+        if matches!(os, Os::Mac) && std::env::consts::ARCH == "aarch64" {
+            if let [Some(s)] = try_get_values_from_file(
+                ["CIDL_SHA256_DARWIN_AMD64"],
+                &checksums_path,
+            )
+            .await?
+            {
+                checksum = s;
+                build = "darwin-11.0-amd64";
+            }
+        }
 
-        let versions_path = self.versions_dir.join("cockroachdb_version");
+        let versions_path =
+            self.versions_dir.join(format!("{prefix}cockroachdb_version"));
         let version = tokio::fs::read_to_string(&versions_path)
             .await
             .context("Failed to read version from {versions_path}")?;
         let version = version.trim();
 
         let (url_base, suffix) = match os {
-            Os::Illumos => ("https://illumos.org/downloads", "tar.gz"),
+            Os::Illumos => (
+                "https://oxide-cockroachdb-build.s3.us-west-2.amazonaws.com",
+                "tar.gz",
+            ),
             Os::Linux | Os::Mac => ("https://binaries.cockroachdb.com", "tgz"),
         };
-        let build = match os {
-            Os::Illumos => "illumos",
-            Os::Linux => "linux-amd64",
-            Os::Mac => "darwin-10.9-amd64",
-        };
 
         let version_directory = format!("cockroach-{version}");
         let tarball_name = format!("{version_directory}.{build}");
@@ -522,6 +554,11 @@ impl<'a> Downloader<'a> {
 
         let tarball_path = download_dir.join(tarball_filename);
 
+        // Ensure that the download and unpack steps, which might write to the
+        // same paths, only run one at a time.
+        static MUTEX: Mutex<()> = Mutex::const_new(());
+        let mutex_lock = MUTEX.lock().await;
+
         tokio::fs::create_dir_all(&download_dir).await?;
         tokio::fs::create_dir_all(&destination_dir).await?;
 
@@ -539,6 +576,9 @@ impl<'a> Downloader<'a> {
         info!(self.log, "tarball path: {tarball_path}");
         unpack_tarball(&self.log, &tarball_path, &download_dir).await?;
 
+        // We are done writing to potentially shared files
+        drop(mutex_lock);
+
         // This is where the binary will end up eventually
         let cockroach_binary = destination_dir.join("bin/cockroach");
 

docs/crdb-upgrades.adoc

@@ -60,13 +60,15 @@ a tick, but they must occur in that order.)
 . Add an enum variant for the new version to `CockroachDbClusterVersion`
   in `nexus/types/src/deployment/planning_input.rs`, and change the
   associated constant `NEWLY_INITIALIZED` to that value.
-. Run the test suite, which should catch any unexpected SQL
+. Regenerate the Nexus internal OpenAPI document, which contains an enum
+  of CockroachDB versions:
++
+....
+EXPECTORATE=overwrite cargo nextest run -p omicron-nexus -- integration_tests::commands::test_nexus_openapi_internal
+....
+. Run the full test suite, which should catch any unexpected SQL
   compatibility issues between releases and help validate that your
   build works.
-  * You will need to run the `test_omdb_success_cases` test from
-    omicron-omdb with `EXPECTORATE=overwrite`; this file contains the
-    expected output of various omdb commands, including a fingerprint of
-    CockroachDB's cluster state.
 . Submit a PR for your changes to garbage-compactor; when merged,
   publish the final build to the `oxide-cockroachdb-build` S3 bucket.
 . Update `tools/cockroachdb_checksums`. For non-illumos checksums, use