Skip to content

Commit

Permalink
WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@davepacheco
davepacheco committed Feb 23, 2024
1 parent b525098 commit 0fd3e27
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 176 deletions.
211 changes: 52 additions & 159 deletions nexus/deployment/src/blueprint_builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -724,162 +724,63 @@ impl<'a> BlueprintZones<'a> {
#[cfg(test)]
pub mod test {
use super::*;
use crate::synthetic::SledBuilder;
use crate::synthetic::SyntheticSystemBuilder;
use nexus_types::external_api::views::SledProvisionState;
use omicron_common::address::IpRange;
use omicron_common::address::Ipv4Range;
use omicron_common::address::Ipv6Subnet;
use omicron_common::address::SLED_PREFIX;
use omicron_common::api::external::ByteCount;
use sled_agent_client::types::{
Baseboard, Inventory, OmicronZoneConfig, OmicronZoneDataset,
OmicronZoneType, OmicronZonesConfig, SledRole,
};
use sled_agent_client::types::{OmicronZoneConfig, OmicronZoneType};
use std::str::FromStr;

/// Returns a collection and policy describing a pretty simple system
pub fn example() -> (Collection, Policy) {
let mut builder = nexus_inventory::CollectionBuilder::new("test-suite");

let sled_ids = [
"72443b6c-b8bb-4ffa-ab3a-aeaa428ed79b",
"a5f3db3a-61aa-4f90-ad3e-02833c253bf5",
"0d168386-2551-44e8-98dd-ae7a7570f8a0",
];
let mut policy = Policy {
sleds: BTreeMap::new(),
// IPs from TEST-NET-1 (RFC 5737)
service_ip_pool_ranges: vec![Ipv4Range::new(
"192.0.2.2".parse().unwrap(),
"192.0.2.20".parse().unwrap(),
)
.unwrap()
.into()],
target_nexus_zone_count: 3,
};
let mut service_ip_pool_range = policy.service_ip_pool_ranges[0].iter();
let mut nexus_nic_ips = NEXUS_OPTE_IPV4_SUBNET
.iter()
.skip(NUM_INITIAL_RESERVED_IP_ADDRESSES);
let mut nexus_nic_macs = {
let mut used = HashSet::new();
std::iter::from_fn(move || {
let mut mac = MacAddr::random_system();
while !used.insert(mac) {
mac = MacAddr::random_system();
}
Some(mac)
})
};

for sled_id_str in sled_ids.iter() {
let sled_id: Uuid = sled_id_str.parse().unwrap();
let sled_ip = policy_add_sled(&mut policy, sled_id);
let serial_number = format!("s{}", policy.sleds.len());
builder
.found_sled_inventory(
"test-suite",
Inventory {
baseboard: Baseboard::Gimlet {
identifier: serial_number,
model: String::from("model1"),
revision: 0,
},
reservoir_size: ByteCount::from(1024),
sled_role: SledRole::Gimlet,
sled_agent_address: SocketAddrV6::new(
sled_ip, 12345, 0, 0,
)
.to_string(),
sled_id,
usable_hardware_threads: 10,
usable_physical_ram: ByteCount::from(1024 * 1024),
},
)
.unwrap();
/// Returns a collection and policy describing a small but non-trivial
/// system
pub fn example_system() -> SyntheticSystemBuilder {
// First, build a system that just has some sleds in it.
let mut system_builder = SyntheticSystemBuilder::new();
for _ in 0..3 {
let _ = system_builder.sled(SledBuilder::new()).unwrap();
}
let policy = system_builder.to_policy().unwrap();

let zpools = &policy.sleds.get(&sled_id).unwrap().zpools;
let mut sled_ips =
std::iter::successors(Some(sled_ip.saturating_add(1)), |ip| {
println!("sled_ips iterator: currently {ip:?}");
Some(ip.saturating_add(1))
});
let zones: Vec<_> = std::iter::once({
let ip = sled_ips.next().unwrap();
OmicronZoneConfig {
id: Uuid::new_v4(),
underlay_address: ip,
zone_type: OmicronZoneType::InternalNtp {
address: SocketAddrV6::new(ip, 12345, 0, 0).to_string(),
dns_servers: vec![],
domain: None,
ntp_servers: vec![],
},
}
})
.chain(std::iter::once({
let id = Uuid::new_v4();
let ip = sled_ips.next().unwrap();
let external_ip =
service_ip_pool_range.next().expect("no service IPs left");
let nic_ip =
nexus_nic_ips.next().expect("no nexus nic IPs left");
OmicronZoneConfig {
id,
underlay_address: ip,
zone_type: OmicronZoneType::Nexus {
internal_address: SocketAddrV6::new(ip, 12346, 0, 0)
.to_string(),
external_ip,
nic: NetworkInterface {
id: Uuid::new_v4(),
kind: NetworkInterfaceKind::Service(id),
name: format!("nexus-{id}").parse().unwrap(),
ip: nic_ip.into(),
mac: nexus_nic_macs
.next()
.expect("no nexus nic MACs left"),
subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(),
vni: Vni::SERVICES_VNI,
primary: true,
slot: 0,
},
external_tls: false,
external_dns_servers: Vec::new(),
},
}
}))
.chain(zpools.iter().map(|zpool_name| {
let ip = sled_ips.next().unwrap();
OmicronZoneConfig {
id: Uuid::new_v4(),
underlay_address: ip,
zone_type: OmicronZoneType::Crucible {
address: String::from("[::1]:12345"),
dataset: OmicronZoneDataset {
pool_name: zpool_name.clone(),
},
},
}
}))
.collect();
// Now use the BlueprintBuilder itself to add some stuff.
// XXX-dap is it... okay that we use the BlueprintBuilder here, when
// we're largely testing the BlueprintBuilder?
let initial_blueprint =
BlueprintBuilder::build_initial_from_collection(
&system_builder.to_collection().unwrap(),
&policy,
"test suite",
)
.unwrap();
let mut builder = BlueprintBuilder::new_based_on(
&initial_blueprint,
&policy,
"test suite",
)
.unwrap();

builder
.found_sled_omicron_zones(
"test-suite",
sled_id,
OmicronZonesConfig {
generation: Generation::new().next(),
zones,
},
)
.unwrap();
for (sled_id, sled_resources) in &policy.sleds {
let _ = builder.sled_ensure_zone_ntp(*sled_id).unwrap();
let _ =
builder.sled_ensure_zone_multiple_nexus(*sled_id, 1).unwrap();
for pool_name in &sled_resources.zpools {
let _ = builder
.sled_ensure_zone_crucible(*sled_id, pool_name.clone())
.unwrap();
}
}

let collection = builder.build();
system_builder
}

(collection, policy)
// XXX-dap remove me
pub fn example() -> (Collection, Policy) {
let system = example_system();
(system.to_collection().unwrap(), system.to_policy().unwrap())
}

// XXX-dap remove me
pub fn policy_add_sled(policy: &mut Policy, sled_id: Uuid) -> Ipv6Addr {
let i = policy.sleds.len() + 1;
let sled_ip: Ipv6Addr =
Expand Down Expand Up @@ -977,14 +878,9 @@ pub mod test {

#[test]
fn test_basic() {
let (collection, mut policy) = example();
let blueprint1 = BlueprintBuilder::build_initial_from_collection(
&collection,
Generation::new(),
&policy,
"the_test",
)
.expect("failed to create initial blueprint");
let mut system = example_system();
let policy = system.to_policy().unwrap();
let blueprint1 = system.to_blueprint("test suite").unwrap();
verify_blueprint(&blueprint1);

let mut builder = BlueprintBuilder::new_based_on(
Expand Down Expand Up @@ -1020,14 +916,11 @@ pub mod test {

// The next step is adding these zones to a new sled.
let new_sled_id = Uuid::new_v4();
let _ = policy_add_sled(&mut policy, new_sled_id);
let mut builder = BlueprintBuilder::new_based_on(
&blueprint2,
Generation::new(),
&policy,
"test_basic",
)
.expect("failed to create builder");
let _ = system.sled(SledBuilder::new().id(new_sled_id)).unwrap();
let policy = system.to_policy().unwrap();
let mut builder =
BlueprintBuilder::new_based_on(&blueprint2, &policy, "test_basic")
.expect("failed to create builder");
builder.sled_ensure_zone_ntp(new_sled_id).unwrap();
let new_sled_resources = policy.sleds.get(&new_sled_id).unwrap();
for pool_name in &new_sled_resources.zpools {
Expand Down
54 changes: 37 additions & 17 deletions nexus/deployment/src/synthetic.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use nexus_types::inventory::ZpoolName;
use omicron_common::address::get_sled_address;
use omicron_common::address::IpRange;
use omicron_common::address::Ipv6Subnet;
use omicron_common::address::NEXUS_REDUNDANCY;
use omicron_common::address::RACK_PREFIX;
use omicron_common::address::SLED_PREFIX;
use omicron_common::api::external::ByteCount;
Expand All @@ -45,6 +46,8 @@ pub struct SyntheticSystemBuilder {
sled_subnets: Box<dyn SubnetIterator>,
available_non_scrimlet_slots: BTreeSet<u32>,
available_scrimlet_slots: BTreeSet<u32>,
target_nexus_zone_count: usize,
service_ip_pool_ranges: Vec<IpRange>,
}

impl SyntheticSystemBuilder {
Expand Down Expand Up @@ -87,12 +90,24 @@ impl SyntheticSystemBuilder {
.skip(1)
.map(|s| Ipv6Subnet::new(s.network())),
);

// Policy defaults
let target_nexus_zone_count = NEXUS_REDUNDANCY;
// IPs from TEST-NET-1 (RFC 5737)
let service_ip_pool_ranges = vec![IpRange::try_from((
"192.0.2.2".parse::<Ipv4Addr>().unwrap(),
"192.0.2.20".parse::<Ipv4Addr>().unwrap(),
))
.unwrap()];

SyntheticSystemBuilder {
sleds: Vec::new(),
collector: None,
sled_subnets,
available_non_scrimlet_slots,
available_scrimlet_slots,
target_nexus_zone_count,
service_ip_pool_ranges,
}
}

Expand All @@ -110,7 +125,7 @@ impl SyntheticSystemBuilder {
for slot_number in 1..32 {
let mut sled = SledBuilder::new();
if slot_number == 14 || slot_number == 16 {
sled.sled_role(SledRole::Scrimlet);
sled = sled.sled_role(SledRole::Scrimlet);
}
builder.sled(sled)?;
}
Expand All @@ -125,6 +140,19 @@ impl SyntheticSystemBuilder {
self
}

pub fn target_nexus_zone_count(&mut self, count: usize) -> &mut Self {
self.target_nexus_zone_count = count;
self
}

pub fn service_ip_pool_ranges(
&mut self,
ranges: Vec<IpRange>,
) -> &mut Self {
self.service_ip_pool_ranges = ranges;
self
}

pub fn sled(&mut self, sled: SledBuilder) -> anyhow::Result<&mut Self> {
let sled_subnet = self
.sled_subnets
Expand Down Expand Up @@ -196,14 +224,6 @@ impl SyntheticSystemBuilder {
}

pub fn to_policy(&self) -> anyhow::Result<Policy> {
// XXX-dap configurable?
let service_ip_pool_ranges = vec![IpRange::try_from((
"192.168.1.20".parse::<Ipv4Addr>().unwrap(),
"192.168.1.29".parse::<Ipv4Addr>().unwrap(),
))
.unwrap()];
// XXX-dap configurable?
let target_nexus_zone_count = 3;
let sleds = self
.sleds
.iter()
Expand All @@ -219,8 +239,8 @@ impl SyntheticSystemBuilder {

Ok(Policy {
sleds,
service_ip_pool_ranges: service_ip_pool_ranges,
target_nexus_zone_count: target_nexus_zone_count,
service_ip_pool_ranges: self.service_ip_pool_ranges.clone(),
target_nexus_zone_count: self.target_nexus_zone_count,
})
}

Expand Down Expand Up @@ -264,35 +284,35 @@ impl SledBuilder {
}
}

pub fn id(&mut self, id: Uuid) -> &mut Self {
pub fn id(mut self, id: Uuid) -> Self {
self.id = Some(id);
self
}

pub fn unique<S>(&mut self, unique: S) -> &mut Self
pub fn unique<S>(mut self, unique: S) -> Self
where
String: From<S>,
{
self.unique = Some(String::from(unique));
self
}

pub fn npools(&mut self, npools: u8) -> &mut Self {
pub fn npools(mut self, npools: u8) -> Self {
self.npools = npools;
self
}

pub fn hardware(&mut self, hardware: SledHardware) -> &mut Self {
pub fn hardware(mut self, hardware: SledHardware) -> Self {
self.hardware = hardware;
self
}

pub fn hardware_slot(&mut self, hardware_slot: u32) -> &mut Self {
pub fn hardware_slot(mut self, hardware_slot: u32) -> Self {
self.hardware_slot = Some(hardware_slot);
self
}

pub fn sled_role(&mut self, sled_role: SledRole) -> &mut Self {
pub fn sled_role(mut self, sled_role: SledRole) -> Self {
self.sled_role = sled_role;
self
}
Expand Down

0 comments on commit 0fd3e27

Please sign in to comment.