From 0bde27c79b07b663646ac494116bda7fd4dbd2a7 Mon Sep 17 00:00:00 2001 From: David Pacheco Date: Fri, 23 Feb 2024 15:22:23 -0800 Subject: [PATCH] add REPL --- Cargo.lock | 135 ++++++++++- Cargo.toml | 3 + dev-tools/reconfigurator/Cargo.toml | 29 +++ dev-tools/reconfigurator/src/main.rs | 345 +++++++++++++++++++++++++++ 4 files changed, 499 insertions(+), 13 deletions(-) create mode 100644 dev-tools/reconfigurator/Cargo.toml create mode 100644 dev-tools/reconfigurator/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index a3853f710d..10fac23e7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -219,6 +219,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "arrayvec" version = "0.7.4" @@ -566,7 +572,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c2f0dc9a68c6317d884f97cc36cf5a3d20ba14ce404227df55e1af708ab04bc" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.7.4", "constant_time_eq 0.2.6", ] @@ -577,7 +583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.7.4", "cc", "cfg-if", "constant_time_eq 0.3.0", @@ -1279,6 +1285,23 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossterm" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a84cda67535339806297f1b331d6dd6320470d2a0fe65381e79ee9e156dd3d13" +dependencies = [ + "bitflags 1.3.2", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.1", + "serde", + "signal-hook", + "signal-hook-mio", + "winapi", +] + [[package]] name = "crossterm" version = "0.27.0" @@ -4644,6 +4667,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "nu-ansi-term" +version = "0.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c073d3c1930d0751774acf49e66653acecb416c3a54c6ec095a9b11caddb5a68" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "nu-ansi-term" version = "0.50.0" @@ -5112,7 +5144,7 @@ dependencies = [ "async-bb8-diesel", "chrono", "clap 4.5.0", - "crossterm", + "crossterm 0.27.0", "crucible-agent-client", "csv", "diesel", @@ -5371,7 +5403,7 @@ dependencies = [ "const-oid", "crossbeam-epoch", "crossbeam-utils", - "crossterm", + "crossterm 0.27.0", "crypto-common", "der", "diesel", @@ -5796,7 +5828,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "oximeter", - "reedline", + "reedline 0.29.0", "regex", "reqwest", "rustyline", @@ -6915,7 +6947,7 @@ dependencies = [ "bitflags 2.4.0", "cassowary", "compact_str", - "crossterm", + "crossterm 0.27.0", "indoc 2.0.3", "itertools 0.12.1", "lru", @@ -6967,6 +6999,28 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "reconfigurator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.0", + "dropshot", + "nexus-deployment", + "nexus-types", + "omicron-common", + "omicron-workspace-hack", + "reedline-repl-rs", + "serde", + "serde_json", + "slog", + "slog-error-chain", + "swrite", + "tabled", + "textwrap 0.16.1", + "uuid", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -7005,6 +7059,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "reedline" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fde955d11817fdcb79d703932fb6b473192cb36b6a92ba21f7f4ac0513374e" +dependencies = [ + "chrono", + "crossterm 0.26.1", + "fd-lock 3.0.13", + "itertools 0.10.5", + "nu-ansi-term 0.49.0", + "serde", + "strip-ansi-escapes 0.1.1", + "strum 0.25.0", + "strum_macros 0.25.2", + "thiserror", + "unicode-segmentation", + "unicode-width", +] + [[package]] name = "reedline" version = "0.29.0" @@ -7012,12 +7086,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e01ebfbdb1a88963121d3c928c97be7f10fec7795bec8b918c8cda1db7c29e6" dependencies = [ "chrono", - "crossterm", + "crossterm 0.27.0", "fd-lock 3.0.13", "itertools 0.12.1", - "nu-ansi-term", + "nu-ansi-term 0.50.0", "serde", - "strip-ansi-escapes", + "strip-ansi-escapes 0.2.0", "strum 0.25.0", "strum_macros 0.25.2", "thiserror", @@ -7025,6 +7099,21 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "reedline-repl-rs" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda59601d3b0c3ed9de96d342ca40f95cca2409e8497249139a6c723b211ee33" +dependencies = [ + "clap 4.5.0", + "crossterm 0.26.1", + "nu-ansi-term 0.49.0", + "reedline 0.22.0", + "regex", + "winapi-util", + "yansi", +] + [[package]] name = "ref-cast" version = "1.0.20" @@ -8628,13 +8717,22 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "strip-ansi-escapes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" +dependencies = [ + "vte 0.10.1", +] + [[package]] name = "strip-ansi-escapes" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" dependencies = [ - "vte", + "vte 0.11.1", ] [[package]] @@ -10128,6 +10226,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "vte" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" +dependencies = [ + "arrayvec 0.5.2", + "utf8parse", + "vte_generate_state_changes", +] + [[package]] name = "vte" version = "0.11.1" @@ -10323,7 +10432,7 @@ dependencies = [ "camino", "ciborium", "clap 4.5.0", - "crossterm", + "crossterm 0.27.0", "futures", "humantime", "indexmap 2.2.3", @@ -10384,9 +10493,9 @@ dependencies = [ "camino", "ciborium", "clap 4.5.0", - "crossterm", + "crossterm 0.27.0", "omicron-workspace-hack", - "reedline", + "reedline 0.29.0", "serde", "slog", "slog-async", diff --git a/Cargo.toml b/Cargo.toml index db37547ea0..ff3ad05463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/oxlog", + "dev-tools/reconfigurator", "dev-tools/xtask", "dns-server", "end-to-end-tests", @@ -97,6 +98,7 @@ default-members = [ "dev-tools/omdb", "dev-tools/omicron-dev", "dev-tools/oxlog", + "dev-tools/reconfigurator", # Do not include xtask in the list of default members, because this causes # hakari to not work as well and build times to be longer. # See omicron#4392. @@ -324,6 +326,7 @@ ratatui = "0.26.1" rayon = "1.8" rcgen = "0.12.1" reedline = "0.29.0" +reedline-repl-rs = "1.0.7" ref-cast = "1.0" regex = "1.10.3" regress = "0.8.0" diff --git a/dev-tools/reconfigurator/Cargo.toml b/dev-tools/reconfigurator/Cargo.toml new file mode 100644 index 0000000000..9147089c3b --- /dev/null +++ b/dev-tools/reconfigurator/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "reconfigurator" +version = "0.1.0" +edition = "2021" +license = "MPL-2.0" + +[dependencies] +anyhow.workspace = true +clap.workspace = true +dropshot.workspace = true +nexus-types.workspace = true +nexus-deployment.workspace = true +omicron-common.workspace = true +serde.workspace = true +serde_json.workspace = true +reedline-repl-rs.workspace = true +slog.workspace = true +slog-error-chain.workspace = true +swrite.workspace = true +tabled.workspace = true +textwrap.workspace = true +uuid.workspace = true +omicron-workspace-hack.workspace = true + +# Disable doc builds by default for our binaries to work around issue +# rust-lang/cargo#8373. These docs would not be very useful anyway. +[[bin]] +name = "reconfigurator" +doc = false diff --git a/dev-tools/reconfigurator/src/main.rs b/dev-tools/reconfigurator/src/main.rs new file mode 100644 index 0000000000..82adab4b8a --- /dev/null +++ b/dev-tools/reconfigurator/src/main.rs @@ -0,0 +1,345 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use anyhow::{anyhow, Context}; +use nexus_deployment::blueprint_builder::BlueprintBuilder; +use nexus_deployment::planner::Planner; +use nexus_deployment::synthetic::{SledBuilder, SyntheticSystemBuilder}; +use nexus_types::deployment::Blueprint; +use nexus_types::inventory::Collection; +use omicron_common::api::external::Generation; +use reedline_repl_rs::clap::{Arg, ArgMatches, Command}; +use reedline_repl_rs::Repl; +use swrite::{swriteln, SWrite}; +use tabled::Tabled; +use uuid::Uuid; + +// Current status: +// - starting to flesh out the basic CRUD stuff. it kind of works. it's hard +// to really see it because I haven't fleshed out "show". +// - I think it's finally time to switch to using the derive version of clap + +// reedline directly. This is in my way right now because I'm getting back +// errors that I cannot see. See item below. +// - in a bit of a dead end in the demo flow because SyntheticSystemBuilder does +// not put any zones onto anything, and build-blueprint-from-inventory +// requires that there are zones. options here include: +// - create a function to create an initial Blueprint from a *blank* inventory +// (or maybe a SystemBuilder? since RSS won't have an inventory) that does +// what RSS would do (maybe even have RSS use this?) +// - add support to SyntheticSystemBuilder for putting zones onto things +// (not sure this is worth doing) +// - build support for loading *saved* stuff from existing systems and only +// support that (nope) +// - flesh out: +// - "inventory show" +// - "blueprint show" +// - "inventory-diff-zones" +// - "blueprint-diff-zones" +// - "blueprint-diff-zones-from-inventory" +// - after I've done that, I should be able to walk through a pretty nice demo +// of the form: +// - configure some sleds +// - generate inventory +// - generate a "rack setup" blueprint over those sleds +// - show it +// - diff inventory -> blueprint +// +// At this point I'll want to be able to walk through multiple steps. To do +// this, I need a way to generate an inventory from the result of _executing_ a +// blueprint. I could: +// +// - call the real deployment code against simulated stuff +// (sled agents + database) +// This is appealing but in the limit may require quite a lot of Nexus to +// work. Can you even create a fake datastore outside the nexus-db-queries +// crate? If so, how much does this stuff assume stuff has been set up by a +// real Nexus? +// - implement dry-run deployment code +// - we have a basic idea how this might work +// - but then we also need a way to fake up an inventory from the result +// - seems like this would require the alternate approach of implementing +// methods on SystemBuilder to specify zones +// +// With this in place, though, the demo could proceed: +// +// - generate inventory after initial setup +// - add sled +// - generate "add sled" blueprint +// - execute +// - regenerate inventory +// - repeat for multi-step process +// +// We could have a similar flow where we start not by generating a "rack setup" +// blueprint but by starting with an inventory that has zones in it. (We might +// _get_ that through the above simulation.) +// +// XXX-dap reedline-repl-rs nits +// - cannot turn off the date banner +// - cannot use structopt version of command definitions +// - when it prints errors, it doesn't print messages from causes, which is +// terrible when it comes to anyhow +// - commands in help are not listed in alphabetical order nor the order in +// which I added them + +#[derive(Debug)] +struct ReconfiguratorSim { + system: SyntheticSystemBuilder, + collections: Vec, + blueprints: Vec, + log: slog::Logger, +} + +fn main() -> anyhow::Result<()> { + let log = dropshot::ConfigLogging::StderrTerminal { + level: dropshot::ConfigLoggingLevel::Debug, + } + .to_logger("reconfigurator-sim") + .context("creating logger")?; + let sim = ReconfiguratorSim { + system: SyntheticSystemBuilder::new(), + collections: vec![], + blueprints: vec![], + log, + }; + let mut repl = Repl::new(sim) + .with_name("reconfigurator-sim") + .with_description("simulate blueprint planning and execution") + .with_partial_completions(false) + .with_command( + Command::new("sled-list").about("list sleds"), + cmd_sled_list, + ) + .with_command( + Command::new("sled-add").about("add a new sled"), + cmd_sled_add, + ) + .with_command( + Command::new("sled-show") + .about("show details about a sled") + .arg(Arg::new("sled_id").required(true)), + cmd_sled_show, + ) + .with_command( + Command::new("inventory-list") + .about("lists all inventory collections"), + cmd_inventory_list, + ) + .with_command( + Command::new("inventory-generate").about( + "generates an inventory collection from the configured \ + sleds", + ), + cmd_inventory_generate, + ) + .with_command( + Command::new("blueprint-list").about("lists all blueprints"), + cmd_blueprint_list, + ) + .with_command( + Command::new("blueprint-from-inventory") + .about("generate an initial blueprint from inventory") + .arg(Arg::new("collection_id").required(true)), + cmd_blueprint_from_inventory, + ) + .with_command( + Command::new("blueprint-plan") + .about("run planner to generate a new blueprint") + .arg(Arg::new("parent_blueprint_id").required(true)) + .arg(Arg::new("collection_id").required(true)), + cmd_blueprint_plan, + ); + + repl.run().context("unexpected failure") +} + +fn cmd_sled_list( + _args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct Sled { + id: Uuid, + nzpools: usize, + subnet: String, + } + + let policy = sim.system.to_policy().context("failed to generate policy")?; + let rows = policy.sleds.iter().map(|(sled_id, sled_resources)| Sled { + id: *sled_id, + subnet: sled_resources.subnet.net().to_string(), + nzpools: sled_resources.zpools.len(), + }); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + Ok(Some(table)) +} + +fn cmd_sled_add( + _args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + let mut sled = SledBuilder::new(); + let _ = sim.system.sled(sled).context("adding sled")?; + Ok(Some(format!("added sled"))) +} + +fn cmd_sled_show( + args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + let policy = sim.system.to_policy().context("failed to generate policy")?; + let sled_id: Uuid = args + .get_one::("sled_id") + .ok_or_else(|| anyhow!("missing sled_id"))? + .parse() + .context("sled_id")?; + let sled_resources = policy + .sleds + .get(&sled_id) + .ok_or_else(|| anyhow!("no sled with id {:?}", sled_id))?; + let mut s = String::new(); + swriteln!(s, "sled {}", sled_id); + swriteln!(s, "subnet {}", sled_resources.subnet.net()); + swriteln!(s, "zpools ({}):", sled_resources.zpools.len()); + for z in &sled_resources.zpools { + swriteln!(s, " {:?}", z); + } + Ok(Some(s)) +} + +fn cmd_inventory_list( + _args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct InventoryRow { + id: Uuid, + } + + let rows = sim + .collections + .iter() + .map(|collection| InventoryRow { id: collection.id }); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + Ok(Some(table)) +} + +fn cmd_inventory_generate( + _args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + let inventory = + sim.system.to_collection().context("generating inventory")?; + let rv = format!( + "generated inventory collection {} from configured sleds", + inventory.id + ); + sim.collections.push(inventory); + Ok(Some(rv)) +} + +fn cmd_blueprint_list( + _args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + #[derive(Tabled)] + #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] + struct BlueprintRow { + id: Uuid, + } + + let rows = sim + .blueprints + .iter() + .map(|blueprint| BlueprintRow { id: blueprint.id }); + let table = tabled::Table::new(rows) + .with(tabled::settings::Style::empty()) + .with(tabled::settings::Padding::new(0, 1, 0, 0)) + .to_string(); + Ok(Some(table)) +} + +fn cmd_blueprint_from_inventory( + args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + let collection_id: Uuid = args + .get_one::("collection_id") + .ok_or_else(|| anyhow!("missing collection_id"))? + .parse() + .context("collection_id")?; + let collection = sim + .collections + .iter() + .find(|c| c.id == collection_id) + .ok_or_else(|| anyhow!("no such collection: {}", collection_id))?; + let dns_version = Generation::new(); + let policy = sim.system.to_policy().context("generating policy")?; + let creator = "reconfigurator-sim"; + let blueprint = BlueprintBuilder::build_initial_from_collection( + collection, + dns_version, + &policy, + creator, + ) + .context("building collection")?; + let rv = format!( + "generated blueprint {} from inventory collection {}", + blueprint.id, collection_id + ); + sim.blueprints.push(blueprint); + Ok(Some(rv)) +} + +fn cmd_blueprint_plan( + args: ArgMatches, + sim: &mut ReconfiguratorSim, +) -> anyhow::Result> { + let parent_blueprint_id: Uuid = args + .get_one::("parent_blueprint_id") + .ok_or_else(|| anyhow!("missing parent_blueprint_id"))? + .parse() + .context("parent_blueprint_id")?; + let collection_id: Uuid = args + .get_one::("collection_id") + .ok_or_else(|| anyhow!("missing collection_id"))? + .parse() + .context("collection_id")?; + let parent_blueprint = sim + .blueprints + .iter() + .find(|b| b.id == parent_blueprint_id) + .ok_or_else(|| anyhow!("no such blueprint: {}", parent_blueprint_id))?; + let collection = sim + .collections + .iter() + .find(|c| c.id == collection_id) + .ok_or_else(|| anyhow!("no such collection: {}", collection_id))?; + let dns_version = Generation::new(); + let policy = sim.system.to_policy().context("generating policy")?; + let creator = "reconfigurator-sim"; + let planner = Planner::new_based_on( + sim.log.clone(), + parent_blueprint, + dns_version, + &policy, + creator, + collection, + ) + .context("creating planner")?; + let blueprint = planner.plan().context("generating blueprint")?; + let rv = format!( + "generated blueprint {} based on parent blueprint {}", + blueprint.id, parent_blueprint_id, + ); + sim.blueprints.push(blueprint); + Ok(Some(rv)) +}