Skip to content

Commit

Permalink
crdb v22.2 prep (R9)
Browse files Browse the repository at this point in the history
  • Loading branch information
iliana committed Jul 3, 2024
1 parent 599fe72 commit 1c773e3
Show file tree
Hide file tree
Showing 16 changed files with 375 additions and 150 deletions.
17 changes: 17 additions & 0 deletions .github/buildomat/build-and-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,23 @@ ptime -m cargo build -Z unstable-options --timings=json --workspace --tests --lo
banner test
ptime -m timeout 2h cargo nextest run --profile ci --locked --verbose

#
# If upgraded CockroachDB clusters are running on a different cluster version
# than newly-initialized ones, run database-related tests from the older cluster
# version.
#
if cmp tools/cockroachdb_version tools/prev_cockroachdb_version; then
# files are the same
true
else
# if return code is > 1, cmp failed
[[ $? -gt 1 ]] && exit 2
# otherwise, files differ
cargo xtask download cockroach-prev
CRDB_SEED_USE_PREV=yes ptime -m timeout 1h cargo nextest run --profile ci --locked --verbose \
--filter-expr 'rdeps(nexus-test-utils) - package(omicron-dev)'
fi

#
# https://github.com/nextest-rs/nextest/issues/16
#
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion dev-tools/omdb/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -58,13 +58,14 @@ multimap.workspace = true
indicatif.workspace = true

[dev-dependencies]
camino-tempfile.workspace = true
expectorate.workspace = true
http.workspace = true
nexus-test-utils.workspace = true
nexus-test-utils-macros.workspace = true
omicron-nexus.workspace = true
omicron-test-utils.workspace = true
subprocess.workspace = true
camino-tempfile.workspace = true

# Disable doc builds by default for our binaries to work around issue
# rust-lang/cargo#8373. These docs would not be very useful anyway.
Expand Down
12 changes: 6 additions & 6 deletions dev-tools/omdb/tests/successes.out
Original file line number Diff line number Diff line change
Expand Up @@ -602,8 +602,8 @@ WARNING: Zones exist without physical disks!


COCKROACHDB SETTINGS:
state fingerprint::::::::::::::::: d4d87aa2ad877a4cc2fddd0573952362739110de
cluster.preserve_downgrade_option: "22.1"
state fingerprint::::::::::::::::: <COCKROACHDB_FINGERPRINT_REDACTED>
cluster.preserve_downgrade_option: <COCKROACHDB_VERSION_REDACTED>

METADATA:
created by::::::::::: nexus-test-utils
Expand Down Expand Up @@ -640,8 +640,8 @@ WARNING: Zones exist without physical disks!


COCKROACHDB SETTINGS:
state fingerprint::::::::::::::::: d4d87aa2ad877a4cc2fddd0573952362739110de
cluster.preserve_downgrade_option: "22.1"
state fingerprint::::::::::::::::: <COCKROACHDB_FINGERPRINT_REDACTED>
cluster.preserve_downgrade_option: <COCKROACHDB_VERSION_REDACTED>

METADATA:
created by::::::::::: nexus-test-utils
Expand Down Expand Up @@ -681,8 +681,8 @@ to: blueprint ......<REDACTED_BLUEPRINT_ID>.......


COCKROACHDB SETTINGS:
state fingerprint::::::::::::::::: d4d87aa2ad877a4cc2fddd0573952362739110de (unchanged)
cluster.preserve_downgrade_option: "22.1" (unchanged)
state fingerprint::::::::::::::::: <COCKROACHDB_FINGERPRINT_REDACTED> (unchanged)
cluster.preserve_downgrade_option: <COCKROACHDB_VERSION_REDACTED> (unchanged)

METADATA:
internal DNS version: 1 (unchanged)
Expand Down
34 changes: 31 additions & 3 deletions dev-tools/omdb/tests/test_all_output.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@
//! Feel free to change the tool's output. This test just makes it easy to make
//! sure you're only breaking what you intend.
use dropshot::Method;
use expectorate::assert_contents;
use http::StatusCode;
use nexus_test_utils_macros::nexus_test;
use nexus_types::deployment::Blueprint;
use nexus_types::deployment::SledFilter;
use nexus_types::deployment::UnstableReconfiguratorState;
use omicron_test_utils::dev::test_cmds::path_to_executable;
Expand Down Expand Up @@ -83,6 +86,20 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
let tmppath = tmpdir.path().join("reconfigurator-save.out");
let initial_blueprint_id = cptestctx.initial_blueprint_id.to_string();

// Get the CockroachDB metadata from the blueprint so we can redact it
let initial_blueprint: Blueprint = dropshot::test_util::read_json(
&mut cptestctx
.internal_client
.make_request_no_body(
Method::GET,
&format!("/deployment/blueprints/all/{initial_blueprint_id}"),
StatusCode::OK,
)
.await
.unwrap(),
)
.await;

let mut output = String::new();

let invocations: &[&[&str]] = &[
Expand Down Expand Up @@ -119,6 +136,19 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
// ControlPlaneTestContext.
];

let mut redactions = ExtraRedactions::new();
redactions
.variable_length("tmp_path", tmppath.as_str())
.fixed_length("blueprint_id", &initial_blueprint_id)
.variable_length(
"cockroachdb_fingerprint",
&initial_blueprint.cockroachdb_fingerprint,
);
let crdb_version =
initial_blueprint.cockroachdb_setting_preserve_downgrade.to_string();
if initial_blueprint.cockroachdb_setting_preserve_downgrade.is_set() {
redactions.variable_length("cockroachdb_version", &crdb_version);
}
for args in invocations {
println!("running commands with args: {:?}", args);
let p = postgres_url.to_string();
Expand All @@ -133,9 +163,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) {
},
&cmd_path,
args,
ExtraRedactions::new()
.variable_length("tmp_path", tmppath.as_str())
.fixed_length("blueprint_id", &initial_blueprint_id),
&redactions,
)
.await;
}
Expand Down
134 changes: 87 additions & 47 deletions dev-tools/xtask/src/download.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ use strum::IntoEnumIterator;
use tar::Archive;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::process::Command;
use tokio::sync::Mutex;

const BUILDOMAT_URL: &'static str =
"https://buildomat.eng.oxide.computer/public/file";
Expand Down Expand Up @@ -50,6 +51,9 @@ enum Target {
/// CockroachDB binary
Cockroach,

/// CockroachDB binary (previous major version)
CockroachPrev,

/// Web console assets
Console,

Expand Down Expand Up @@ -125,7 +129,8 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> {
bail!("We should have already filtered this 'All' target out?");
}
Target::Clickhouse => downloader.download_clickhouse().await,
Target::Cockroach => downloader.download_cockroach().await,
Target::Cockroach => downloader.download_cockroach("").await,
Target::CockroachPrev => downloader.download_cockroach("prev_").await,
Target::Console => downloader.download_console().await,
Target::DendriteOpenapi => {
downloader.download_dendrite_openapi().await
Expand Down Expand Up @@ -197,43 +202,52 @@ impl<'a> Downloader<'a> {
}
}

/// Parses a file of the format:
///
/// ```ignore
/// KEY1="value1"
/// KEY2="value2"
/// ```
///
/// And returns an array of the values in the same order as keys.
async fn get_values_from_file<const N: usize>(
keys: [&str; N],
path: &Utf8Path,
) -> Result<[String; N]> {
// Map of "key" => "Position in output".
let mut keys: HashMap<&str, usize> =
keys.into_iter().enumerate().map(|(i, s)| (s, i)).collect();

const EMPTY_STRING: String = String::new();
let mut values = [EMPTY_STRING; N];

let content = tokio::fs::read_to_string(&path)
.await
.context("Failed to read {path}")?;
for line in content.lines() {
let line = line.trim();
let Some((key, value)) = line.split_once('=') else {
continue;
};
let value = value.trim_matches('"');
if let Some(i) = keys.remove(key) {
values[i] = value.to_string();
macro_rules! get_values_fn {
($name:ident, $ty:ty, $def:expr, $check:expr) => {
/// Parses a file of the format:
///
/// ```ignore
/// KEY1="value1"
/// KEY2="value2"
/// ```
///
/// And returns an array of the values in the same order as keys.
async fn $name<const N: usize>(
keys: [&str; N],
path: &Utf8Path,
) -> Result<[$ty; N]> {
// Map of "key" => "Position in output".
let mut keys: HashMap<&str, usize> =
keys.into_iter().enumerate().map(|(i, s)| (s, i)).collect();

const DEFAULT: $ty = $def;
let mut values = [DEFAULT; N];

let content = tokio::fs::read_to_string(&path)
.await
.context("Failed to read {path}")?;
for line in content.lines() {
let line = line.trim();
let Some((key, value)) = line.split_once('=') else {
continue;
};
let value = value.trim_matches('"');
if let Some(i) = keys.remove(key) {
values[i] = value.to_string().into();
}
}
if $check && !keys.is_empty() {
bail!(
"Could not find keys: {:?}",
keys.keys().collect::<Vec<_>>(),
);
}
Ok(values)
}
}
if !keys.is_empty() {
bail!("Could not find keys: {:?}", keys.keys().collect::<Vec<_>>(),);
}
Ok(values)
};
}
get_values_fn!(get_values_from_file, String, String::new(), true);
get_values_fn!(try_get_values_from_file, Option<String>, None, false);

/// Send a GET request to `url`, downloading the contents to `path`.
///
Expand Down Expand Up @@ -486,34 +500,52 @@ impl<'a> Downloader<'a> {
Ok(())
}

async fn download_cockroach(&self) -> Result<()> {
async fn download_cockroach(&self, prefix: &str) -> Result<()> {
let os = os_name()?;

let download_dir = self.output_dir.join("downloads");
let destination_dir = self.output_dir.join("cockroachdb");
let destination_dir =
self.output_dir.join(format!("{prefix}cockroachdb"));

let checksums_path = self.versions_dir.join("cockroachdb_checksums");
let [checksum] = get_values_from_file(
let checksums_path =
self.versions_dir.join(format!("{prefix}cockroachdb_checksums"));
let [mut checksum] = get_values_from_file(
[&format!("CIDL_SHA256_{}", os.env_name())],
&checksums_path,
)
.await?;
let mut build = match os {
Os::Illumos => "illumos",
Os::Linux => "linux-amd64",
Os::Mac => "darwin-10.9-amd64",
};

if matches!(os, Os::Mac) && std::env::consts::ARCH == "aarch64" {
if let [Some(s)] = try_get_values_from_file(
["CIDL_SHA256_DARWIN_ARM64"],
&checksums_path,
)
.await?
{
checksum = s;
build = "darwin-11.0-arm64";
}
}

let versions_path = self.versions_dir.join("cockroachdb_version");
let versions_path =
self.versions_dir.join(format!("{prefix}cockroachdb_version"));
let version = tokio::fs::read_to_string(&versions_path)
.await
.context("Failed to read version from {versions_path}")?;
let version = version.trim();

let (url_base, suffix) = match os {
Os::Illumos => ("https://illumos.org/downloads", "tar.gz"),
Os::Illumos => (
"https://oxide-cockroachdb-build.s3.us-west-2.amazonaws.com",
"tar.gz",
),
Os::Linux | Os::Mac => ("https://binaries.cockroachdb.com", "tgz"),
};
let build = match os {
Os::Illumos => "illumos",
Os::Linux => "linux-amd64",
Os::Mac => "darwin-10.9-amd64",
};

let version_directory = format!("cockroach-{version}");
let tarball_name = format!("{version_directory}.{build}");
Expand All @@ -522,6 +554,11 @@ impl<'a> Downloader<'a> {

let tarball_path = download_dir.join(tarball_filename);

// Ensure that the download and unpack steps, which might write to the
// same paths, only run one at a time.
static MUTEX: Mutex<()> = Mutex::const_new(());
let mutex_lock = MUTEX.lock().await;

tokio::fs::create_dir_all(&download_dir).await?;
tokio::fs::create_dir_all(&destination_dir).await?;

Expand All @@ -539,6 +576,9 @@ impl<'a> Downloader<'a> {
info!(self.log, "tarball path: {tarball_path}");
unpack_tarball(&self.log, &tarball_path, &download_dir).await?;

// We are done writing to potentially shared files
drop(mutex_lock);

// This is where the binary will end up eventually
let cockroach_binary = destination_dir.join("bin/cockroach");

Expand Down
12 changes: 7 additions & 5 deletions docs/crdb-upgrades.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,15 @@ a tick, but they must occur in that order.)
. Add an enum variant for the new version to `CockroachDbClusterVersion`
in `nexus/types/src/deployment/planning_input.rs`, and change the
associated constant `NEWLY_INITIALIZED` to that value.
. Run the test suite, which should catch any unexpected SQL
. Regenerate the Nexus internal OpenAPI document, which contains an enum
of CockroachDB versions:
+
....
EXPECTORATE=overwrite cargo nextest run -p omicron-nexus -- integration_tests::commands::test_nexus_openapi_internal
....
. Run the full test suite, which should catch any unexpected SQL
compatibility issues between releases and help validate that your
build works.
* You will need to run the `test_omdb_success_cases` test from
omicron-omdb with `EXPECTORATE=overwrite`; this file contains the
expected output of various omdb commands, including a fingerprint of
CockroachDB's cluster state.
. Submit a PR for your changes to garbage-compactor; when merged,
publish the final build to the `oxide-cockroachdb-build` S3 bucket.
. Update `tools/cockroachdb_checksums`. For non-illumos checksums, use
Expand Down
Loading

0 comments on commit 1c773e3

Please sign in to comment.