diff --git a/Cargo.lock b/Cargo.lock index c3fc5d86cf..5391891463 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4262,6 +4262,23 @@ dependencies = [ "rustc_version 0.1.7", ] +[[package]] +name = "nexus-capabilities" +version = "0.1.0" +dependencies = [ + "async-trait", + "futures", + "ipnetwork", + "nexus-db-queries", + "omicron-common", + "omicron-workspace-hack", + "reqwest", + "sled-agent-client", + "slog", + "slog-error-chain", + "uuid", +] + [[package]] name = "nexus-client" version = "0.1.0" @@ -4460,6 +4477,7 @@ dependencies = [ "illumos-utils", "internal-dns", "ipnet", + "nexus-capabilities", "nexus-db-model", "nexus-db-queries", "nexus-inventory", @@ -5031,6 +5049,7 @@ dependencies = [ "macaddr", "mg-admin-client", "mime_guess", + "nexus-capabilities", "nexus-db-model", "nexus-db-queries", "nexus-defaults", diff --git a/Cargo.toml b/Cargo.toml index e19ed35c32..80df45bd40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ members = [ "key-manager", "nexus", "nexus/authz-macros", + "nexus/capabilities", "nexus/db-macros", "nexus/db-model", "nexus/db-queries", @@ -114,6 +115,7 @@ default-members = [ "key-manager", "nexus", "nexus/authz-macros", + "nexus/capabilities", "nexus/macros-common", "nexus/db-macros", "nexus/db-model", @@ -253,6 +255,7 @@ mockall = "0.12" newtype_derive = "0.1.6" mg-admin-client = { path = "clients/mg-admin-client" } multimap = "0.10.0" +nexus-capabilities = { path = "nexus/capabilities" } nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } nexus-db-queries = { path = "nexus/db-queries" } diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 581e16e84d..32263f8ece 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -77,6 +77,7 @@ tough.workspace = true trust-dns-resolver.workspace = true uuid.workspace = true +nexus-capabilities.workspace = true nexus-defaults.workspace = true nexus-db-model.workspace = true nexus-db-queries.workspace = true diff --git a/nexus/capabilities/Cargo.toml b/nexus/capabilities/Cargo.toml new file mode 100644 index 0000000000..799c049724 --- /dev/null +++ b/nexus/capabilities/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "nexus-capabilities" +version = "0.1.0" +edition = "2021" + +[dependencies] +async-trait.workspace = true +futures.workspace = true +ipnetwork.workspace = true +nexus-db-queries.workspace = true +omicron-common.workspace = true +reqwest.workspace = true +sled-agent-client.workspace = true +slog.workspace = true +slog-error-chain.workspace = true +uuid.workspace = true + +omicron-workspace-hack.workspace = true diff --git a/nexus/capabilities/src/firewall_rules.rs b/nexus/capabilities/src/firewall_rules.rs new file mode 100644 index 0000000000..0c812116fc --- /dev/null +++ b/nexus/capabilities/src/firewall_rules.rs @@ -0,0 +1,461 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Nexus capabilities for updating firewall rules + +use crate::SledAgent; +use futures::future::join_all; +use ipnetwork::IpNetwork; +use nexus_db_queries::authz; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db; +use nexus_db_queries::db::identity::Asset; +use nexus_db_queries::db::identity::Resource; +use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::model::Name; +use omicron_common::api::external; +use omicron_common::api::external::Error; +use omicron_common::api::external::IpNet; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::internal::nexus::HostIdentifier; +use sled_agent_client::types::NetworkInterface; +use slog::debug; +use slog::info; +use slog::warn; +use std::collections::HashMap; +use std::collections::HashSet; +use std::net::IpAddr; +use uuid::Uuid; + +#[async_trait::async_trait] +pub trait FirewallRules: SledAgent { + async fn vpc_list_firewall_rules( + &self, + opctx: &OpContext, + vpc_lookup: &lookup::Vpc<'_>, + ) -> ListResultVec { + let (.., authz_vpc) = + vpc_lookup.lookup_for(authz::Action::Read).await?; + let rules = + self.datastore().vpc_list_firewall_rules(opctx, &authz_vpc).await?; + Ok(rules) + } + + /// Ensure firewall rules for internal services get reflected on all the + /// relevant sleds. + async fn plumb_service_firewall_rules( + &self, + opctx: &OpContext, + sleds_filter: &[Uuid], + sled_lookup_opctx: &OpContext, + ) -> Result<(), Error> { + let svcs_vpc = LookupPath::new(opctx, self.datastore()) + .vpc_id(*db::fixed_data::vpc::SERVICES_VPC_ID); + let svcs_fw_rules = + self.vpc_list_firewall_rules(opctx, &svcs_vpc).await?; + let (_, _, _, svcs_vpc) = svcs_vpc.fetch().await?; + self.send_sled_agents_firewall_rules( + opctx, + &svcs_vpc, + &svcs_fw_rules, + sleds_filter, + sled_lookup_opctx, + ) + .await?; + Ok(()) + } + + async fn send_sled_agents_firewall_rules( + &self, + opctx: &OpContext, + vpc: &db::model::Vpc, + rules: &[db::model::VpcFirewallRule], + sleds_filter: &[Uuid], + sled_lookup_opctx: &OpContext, + ) -> Result<(), Error> { + let rules_for_sled = self + .resolve_firewall_rules_for_sled_agent(opctx, vpc, rules) + .await?; + debug!(self.log(), "resolved {} rules for sleds", rules_for_sled.len()); + let sled_rules_request = + sled_agent_client::types::VpcFirewallRulesEnsureBody { + vni: vpc.vni.0, + rules: rules_for_sled, + }; + + let vpc_to_sleds = self + .datastore() + .vpc_resolve_to_sleds(vpc.id(), sleds_filter) + .await?; + debug!( + self.log(), "resolved sleds for vpc {}", vpc.name(); + "vpc_to_sled" => ?vpc_to_sleds, + ); + + let mut sled_requests = Vec::with_capacity(vpc_to_sleds.len()); + for sled in &vpc_to_sleds { + let sled_id = sled.id(); + let vpc_id = vpc.id(); + let sled_rules_request = sled_rules_request.clone(); + sled_requests.push(async move { + self.sled_client_by_id(sled_lookup_opctx, sled_id) + .await? + .vpc_firewall_rules_put(&vpc_id, &sled_rules_request) + .await + .map_err(|e| Error::internal_error(&e.to_string())) + }); + } + + debug!(self.log(), "sending firewall rules to sled agents"); + let results = join_all(sled_requests).await; + // TODO-correctness: handle more than one failure in the sled-agent requests + // https://github.com/oxidecomputer/omicron/issues/1791 + for (sled, result) in vpc_to_sleds.iter().zip(results) { + if let Err(e) = result { + warn!(self.log(), "failed to update firewall rules on sled agent"; + "sled_id" => %sled.id(), + "vpc_id" => %vpc.id(), + "error" => %e); + return Err(e); + } + } + info!( + self.log(), + "updated firewall rules on {} sleds", + vpc_to_sleds.len() + ); + + Ok(()) + } + + async fn resolve_firewall_rules_for_sled_agent( + &self, + opctx: &OpContext, + vpc: &db::model::Vpc, + rules: &[db::model::VpcFirewallRule], + ) -> Result, Error> { + // Collect the names of instances, subnets, and VPCs that are either + // targets or host filters. We have to find the sleds for all the + // targets, and we'll need information about the IP addresses or + // subnets for things that are specified as host filters as well. + let mut instances: HashSet = HashSet::new(); + let mut subnets: HashSet = HashSet::new(); + let mut vpcs: HashSet = HashSet::new(); + for rule in rules { + for target in &rule.targets { + match &target.0 { + external::VpcFirewallRuleTarget::Instance(name) => { + instances.insert(name.clone().into()); + } + external::VpcFirewallRuleTarget::Subnet(name) => { + subnets.insert(name.clone().into()); + } + external::VpcFirewallRuleTarget::Vpc(name) => { + if name != vpc.name() { + return Err(Error::invalid_request( + "cross-VPC firewall target unsupported", + )); + } + vpcs.insert(name.clone().into()); + } + external::VpcFirewallRuleTarget::Ip(_) + | external::VpcFirewallRuleTarget::IpNet(_) => { + vpcs.insert(vpc.name().clone().into()); + } + } + } + + for host in rule.filter_hosts.iter().flatten() { + match &host.0 { + external::VpcFirewallRuleHostFilter::Instance(name) => { + instances.insert(name.clone().into()); + } + external::VpcFirewallRuleHostFilter::Subnet(name) => { + subnets.insert(name.clone().into()); + } + external::VpcFirewallRuleHostFilter::Vpc(name) => { + if name != vpc.name() { + return Err(Error::invalid_request( + "cross-VPC firewall host filter unsupported", + )); + } + vpcs.insert(name.clone().into()); + } + // We don't need to resolve anything for Ip(Net)s. + external::VpcFirewallRuleHostFilter::Ip(_) => (), + external::VpcFirewallRuleHostFilter::IpNet(_) => (), + } + } + } + + // Resolve named instances, VPCs, and subnets. + // TODO-correctness: It's possible the resolving queries produce + // inconsistent results due to concurrent changes. They should be + // transactional. + type NetMap = HashMap>; + type NicMap = HashMap>; + let no_networks: Vec = Vec::new(); + let no_interfaces: Vec = Vec::new(); + + let mut instance_interfaces: NicMap = HashMap::new(); + for instance_name in &instances { + if let Ok((.., authz_instance)) = + LookupPath::new(opctx, self.datastore()) + .project_id(vpc.project_id) + .instance_name(instance_name) + .lookup_for(authz::Action::ListChildren) + .await + { + for iface in self + .datastore() + .derive_guest_network_interface_info(opctx, &authz_instance) + .await? + { + instance_interfaces + .entry(instance_name.0.clone()) + .or_insert_with(Vec::new) + .push(iface); + } + } + } + + let mut vpc_interfaces: NicMap = HashMap::new(); + for vpc_name in &vpcs { + if let Ok((.., authz_vpc)) = + LookupPath::new(opctx, self.datastore()) + .project_id(vpc.project_id) + .vpc_name(vpc_name) + .lookup_for(authz::Action::ListChildren) + .await + { + for iface in self + .datastore() + .derive_vpc_network_interface_info(opctx, &authz_vpc) + .await? + { + vpc_interfaces + .entry(vpc_name.0.clone()) + .or_insert_with(Vec::new) + .push(iface); + } + } + } + + let mut subnet_interfaces: NicMap = HashMap::new(); + for subnet_name in &subnets { + if let Ok((.., authz_subnet)) = + LookupPath::new(opctx, self.datastore()) + .project_id(vpc.project_id) + .vpc_name(&Name::from(vpc.name().clone())) + .vpc_subnet_name(subnet_name) + .lookup_for(authz::Action::ListChildren) + .await + { + for iface in self + .datastore() + .derive_subnet_network_interface_info(opctx, &authz_subnet) + .await? + { + subnet_interfaces + .entry(subnet_name.0.clone()) + .or_insert_with(Vec::new) + .push(iface); + } + } + } + + let subnet_networks: NetMap = self + .datastore() + .resolve_vpc_subnets_to_ip_networks(vpc, subnets) + .await? + .into_iter() + .map(|(name, v)| (name.0, v)) + .collect(); + + debug!( + self.log(), + "resolved names for firewall rules"; + "instance_interfaces" => ?instance_interfaces, + "vpc_interfaces" => ?vpc_interfaces, + "subnet_interfaces" => ?subnet_interfaces, + "subnet_networks" => ?subnet_networks, + ); + + // Compile resolved rules for the sled agents. + let mut sled_agent_rules = Vec::with_capacity(rules.len()); + for rule in rules { + // TODO: what is the correct behavior when a name is not found? + // Options: + // (1) Fail update request (though note this can still arise + // from things like instance deletion) + // (2) Allow update request, ignore this rule (but store it + // in case it becomes valid later). This is consistent + // with the semantics of the rules. Rules with bad + // references should likely at least be flagged to users. + // We currently adopt option (2), as this allows users to add + // firewall rules (including default rules) before instances + // and their interfaces are instantiated. + + // Collect unique network interface targets. + // This would be easier if `NetworkInterface` were `Hash`, + // but that's not easy because it's a generated type. We + // use the pair (VNI, MAC) as a unique interface identifier. + let mut nics = HashSet::new(); + let mut targets = Vec::with_capacity(rule.targets.len()); + let mut push_target_nic = |nic: &NetworkInterface| { + if nics.insert((nic.vni, *nic.mac)) { + targets.push(nic.clone()); + } + }; + for target in &rule.targets { + match &target.0 { + external::VpcFirewallRuleTarget::Vpc(name) => { + vpc_interfaces + .get(&name) + .unwrap_or(&no_interfaces) + .iter() + .for_each(&mut push_target_nic); + } + external::VpcFirewallRuleTarget::Subnet(name) => { + subnet_interfaces + .get(&name) + .unwrap_or(&no_interfaces) + .iter() + .for_each(&mut push_target_nic); + } + external::VpcFirewallRuleTarget::Instance(name) => { + instance_interfaces + .get(&name) + .unwrap_or(&no_interfaces) + .iter() + .for_each(&mut push_target_nic); + } + external::VpcFirewallRuleTarget::Ip(addr) => { + vpc_interfaces + .get(vpc.name()) + .unwrap_or(&no_interfaces) + .iter() + .filter(|nic| nic.ip == *addr) + .for_each(&mut push_target_nic); + } + external::VpcFirewallRuleTarget::IpNet(net) => { + vpc_interfaces + .get(vpc.name()) + .unwrap_or(&no_interfaces) + .iter() + .filter(|nic| match (net, nic.ip) { + (IpNet::V4(net), IpAddr::V4(ip)) => { + net.contains(ip) + } + (IpNet::V6(net), IpAddr::V6(ip)) => { + net.contains(ip) + } + (_, _) => false, + }) + .for_each(&mut push_target_nic); + } + } + } + if !rule.targets.is_empty() && targets.is_empty() { + // Target not found; skip this rule. + continue; + } + + let filter_hosts = match &rule.filter_hosts { + None => None, + Some(hosts) => { + let mut host_addrs = Vec::with_capacity(hosts.len()); + for host in hosts { + match &host.0 { + external::VpcFirewallRuleHostFilter::Instance( + name, + ) => { + for interface in instance_interfaces + .get(&name) + .unwrap_or(&no_interfaces) + { + host_addrs.push( + HostIdentifier::Ip(IpNet::from( + interface.ip, + )) + .into(), + ) + } + } + external::VpcFirewallRuleHostFilter::Subnet( + name, + ) => { + for subnet in subnet_networks + .get(&name) + .unwrap_or(&no_networks) + { + host_addrs.push( + HostIdentifier::Ip(IpNet::from( + *subnet, + )) + .into(), + ); + } + } + external::VpcFirewallRuleHostFilter::Ip(addr) => { + host_addrs.push( + HostIdentifier::Ip(IpNet::from(*addr)) + .into(), + ) + } + external::VpcFirewallRuleHostFilter::IpNet(net) => { + host_addrs.push(HostIdentifier::Ip(*net).into()) + } + external::VpcFirewallRuleHostFilter::Vpc(name) => { + for interface in vpc_interfaces + .get(&name) + .unwrap_or(&no_interfaces) + { + host_addrs.push( + HostIdentifier::Vpc(interface.vni) + .into(), + ) + } + } + } + } + if !hosts.is_empty() && host_addrs.is_empty() { + // Filter host not found; skip this rule. + continue; + } + Some(host_addrs) + } + }; + + let filter_ports = rule + .filter_ports + .as_ref() + .map(|ports| ports.iter().map(|v| v.0.into()).collect()); + + let filter_protocols = + rule.filter_protocols.as_ref().map(|protocols| { + protocols.iter().map(|v| v.0.into()).collect() + }); + + sled_agent_rules.push(sled_agent_client::types::VpcFirewallRule { + status: rule.status.0.into(), + direction: rule.direction.0.into(), + targets, + filter_hosts, + filter_ports, + filter_protocols, + action: rule.action.0.into(), + priority: rule.priority.0 .0, + }); + } + debug!( + self.log(), + "resolved firewall rules for sled agents"; + "sled_agent_rules" => ?sled_agent_rules, + ); + + Ok(sled_agent_rules) + } +} diff --git a/nexus/capabilities/src/lib.rs b/nexus/capabilities/src/lib.rs new file mode 100644 index 0000000000..bc504f2b98 --- /dev/null +++ b/nexus/capabilities/src/lib.rs @@ -0,0 +1,19 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Nexus functionality, divided into capability groups + +use nexus_db_queries::db; +use slog::Logger; + +mod firewall_rules; +mod sled_agent; + +pub use firewall_rules::FirewallRules; +pub use sled_agent::SledAgent; + +pub trait Base { + fn log(&self) -> &Logger; + fn datastore(&self) -> &db::DataStore; +} diff --git a/nexus/capabilities/src/sled_agent.rs b/nexus/capabilities/src/sled_agent.rs new file mode 100644 index 0000000000..f05141d741 --- /dev/null +++ b/nexus/capabilities/src/sled_agent.rs @@ -0,0 +1,54 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Nexus capabilities for creating sled-agent clients + +use crate::Base; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; +use sled_agent_client::Client; +use slog::o; +use std::net::SocketAddrV6; +use std::sync::Arc; +use uuid::Uuid; + +#[async_trait::async_trait] +pub trait SledAgent: Base { + fn sled_lookup<'a>( + &'a self, + opctx: &'a OpContext, + sled_id: Uuid, + ) -> lookup::Sled<'a> { + LookupPath::new(opctx, self.datastore()).sled_id(sled_id) + } + + fn sled_client(&self, id: Uuid, address: SocketAddrV6) -> Client { + let log = self.log().new(o!("SledAgent" => id.to_string())); + let dur = std::time::Duration::from_secs(60); + let client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build() + .unwrap(); + Client::new_with_client(&format!("http://{address}"), client, log) + } + + async fn sled_client_by_id( + &self, + lookup_opctx: &OpContext, + id: Uuid, + ) -> Result, omicron_common::api::external::Error> { + // TODO: We should consider injecting connection pooling here, + // but for now, connections to sled agents are constructed + // on an "as requested" basis. + // + // Frankly, returning an "Arc" here without a connection pool is a + // little silly; it's not actually used if each client connection exists + // as a one-shot. + let (.., sled) = self.sled_lookup(lookup_opctx, id).fetch().await?; + + Ok(Arc::new(self.sled_client(id, sled.address()))) + } +} diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 55d3e9b43f..c55143eae8 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(38, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(39, 0, 0); table! { disk (id) { @@ -1549,6 +1549,8 @@ allow_tables_to_appear_in_same_query!( allow_tables_to_appear_in_same_query!(hw_baseboard_id, inv_sled_agent,); allow_tables_to_appear_in_same_query!( + bp_omicron_zone, + bp_target, dataset, disk, image, diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 020916928d..c28edfaf75 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -1210,7 +1210,7 @@ mod tests { #[tokio::test] async fn test_empty_blueprint() { // Setup - let logctx = dev::test_setup_log("inventory_insert"); + let logctx = dev::test_setup_log("test_empty_blueprint"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; @@ -1273,7 +1273,7 @@ mod tests { #[tokio::test] async fn test_representative_blueprint() { // Setup - let logctx = dev::test_setup_log("inventory_insert"); + let logctx = dev::test_setup_log("test_representative_blueprint"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; @@ -1456,7 +1456,7 @@ mod tests { #[tokio::test] async fn test_set_target() { // Setup - let logctx = dev::test_setup_log("inventory_insert"); + let logctx = dev::test_setup_log("test_set_target"); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = datastore_test(&logctx, &db).await; diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 4626301f76..f0c563fe53 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -650,8 +650,8 @@ impl DataStore { // Resolve each VNIC in the VPC to the Sled it's on, so we know which // Sleds to notify when firewall rules change. use db::schema::{ - instance, instance_network_interface, service, - service_network_interface, sled, vmm, + bp_omicron_zone, bp_target, instance, instance_network_interface, + service, service_network_interface, sled, vmm, }; let instance_query = instance_network_interface::table @@ -671,7 +671,12 @@ impl DataStore { .filter(vmm::time_deleted.is_null()) .select(Sled::as_select()); - let service_query = service_network_interface::table + // When Nexus accepts the rack initialization handoff from RSS, it + // populates the `service` table. We eventually want to retire it + // (https://github.com/oxidecomputer/omicron/issues/4947), and the + // Reconfigurator does not add new entries to it. We still need to query + // it for systems that are not yet under Reconfigurator control... + let rss_service_query = service_network_interface::table .inner_join( service::table .on(service::id.eq(service_network_interface::service_id)), @@ -681,6 +686,41 @@ impl DataStore { .filter(service_network_interface::time_deleted.is_null()) .select(Sled::as_select()); + // ... and we also need to query for the current target blueprint to + // support systems that _are_ under Reconfigurator control. This query + // needs a join with a subquery, which isn't currently supported by + // diesel's DSL, so we fall back to a raw SQL string and rely on tests + // to ensure that this is valid. + // + // Diesel requires us to use aliases in order to refer to the + // `inv_collection` table twice in the same query. + let (bp_target1, bp_target2) = diesel::alias!( + db::schema::bp_target as bp_target1, + db::schema::bp_target as bp_target2 + ); + let reconfig_service_query = service_network_interface::table + .inner_join(bp_omicron_zone::table.on( + bp_omicron_zone::id.eq(service_network_interface::service_id), + )) + .inner_join( + bp_target1.on(bp_omicron_zone::blueprint_id + .eq(bp_target1.field(bp_target::blueprint_id))), + ) + .inner_join(sled::table.on(sled::id.eq(bp_omicron_zone::sled_id))) + .filter( + // This filters us down to the one current target blueprint (if + // it exists); i.e., the target with the maximal version. + bp_target1.field(bp_target::blueprint_id).eq_any( + bp_target2 + .select(bp_target2.field(bp_target::blueprint_id)) + .order_by(bp_target2.field(bp_target::version).desc()) + .limit(1), + ), + ) + .filter(service_network_interface::vpc_id.eq(vpc_id)) + .filter(service_network_interface::time_deleted.is_null()) + .select(Sled::as_select()); + let mut sleds = sled::table .select(Sled::as_select()) .filter(sled::time_deleted.is_null()) @@ -691,7 +731,11 @@ impl DataStore { let conn = self.pool_connection_unauthorized().await?; sleds - .intersect(instance_query.union(service_query)) + .intersect( + instance_query + .union(rss_service_query) + .union(reconfig_service_query), + ) .get_results_async(&*conn) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) @@ -1172,13 +1216,34 @@ impl DataStore { mod tests { use super::*; use crate::db::datastore::test_utils::datastore_test; + use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use crate::db::model::Project; use crate::db::queries::vpc::MAX_VNI_SEARCH_RANGE_SIZE; + use nexus_db_model::SledBaseboard; + use nexus_db_model::SledSystemHardware; + use nexus_db_model::SledUpdate; use nexus_test_utils::db::test_setup_database; + use nexus_types::deployment::Blueprint; + use nexus_types::deployment::BlueprintTarget; + use nexus_types::deployment::NetworkInterface; + use nexus_types::deployment::NetworkInterfaceKind; + use nexus_types::deployment::OmicronZoneConfig; + use nexus_types::deployment::OmicronZoneType; + use nexus_types::deployment::OmicronZonesConfig; use nexus_types::external_api::params; + use nexus_types::identity::Asset; + use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::api::external; + use omicron_common::api::external::Generation; + use omicron_common::api::external::IpNet; + use omicron_common::api::external::MacAddr; + use omicron_common::api::external::Vni; + use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use omicron_test_utils::dev; use slog::info; + use std::collections::BTreeMap; + use std::collections::BTreeSet; + use std::net::IpAddr; // Test that we detect the right error condition and return None when we // fail to insert a VPC due to VNI exhaustion. @@ -1397,4 +1462,384 @@ mod tests { db.cleanup().await.unwrap(); logctx.cleanup_successful(); } + + #[derive(Debug)] + struct Harness { + rack_id: Uuid, + sled_ids: Vec, + nexuses: Vec, + } + + #[derive(Debug)] + struct HarnessNexus { + id: Uuid, + ip: IpAddr, + mac: MacAddr, + nic_id: Uuid, + } + + impl Harness { + fn new(num_sleds: usize) -> Self { + let mut sled_ids = + (0..num_sleds).map(|_| Uuid::new_v4()).collect::>(); + sled_ids.sort(); + + let mut nexus_ips = NEXUS_OPTE_IPV4_SUBNET + .iter() + .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .map(IpAddr::from); + let mut nexus_macs = MacAddr::iter_system(); + let nexuses = (0..num_sleds) + .map(|_| HarnessNexus { + id: Uuid::new_v4(), + ip: nexus_ips.next().unwrap(), + mac: nexus_macs.next().unwrap(), + nic_id: Uuid::new_v4(), + }) + .collect::>(); + Self { rack_id: Uuid::new_v4(), sled_ids, nexuses } + } + + fn db_sleds(&self) -> impl Iterator + '_ { + self.sled_ids.iter().enumerate().map(|(index, sled_id)| { + SledUpdate::new( + *sled_id, + "[::1]:0".parse().unwrap(), + SledBaseboard { + serial_number: format!("sled-{index}"), + part_number: "test-sled".to_string(), + revision: 0, + }, + SledSystemHardware { + is_scrimlet: false, + usable_hardware_threads: 128, + usable_physical_ram: (128 << 30).try_into().unwrap(), + reservoir_size: (64 << 30).try_into().unwrap(), + }, + self.rack_id, + ) + }) + } + + fn db_services( + &self, + ) -> impl Iterator< + Item = (db::model::Service, db::model::IncompleteNetworkInterface), + > + '_ { + self.sled_ids.iter().zip(&self.nexuses).map(|(sled_id, nexus)| { + let service = db::model::Service::new( + nexus.id, + *sled_id, + Some(nexus.id), + "[::1]:0".parse().unwrap(), + db::model::ServiceKind::Nexus, + ); + let name = format!("test-nexus-{}", nexus.id); + let nic = db::model::IncompleteNetworkInterface::new_service( + nexus.nic_id, + nexus.id, + NEXUS_VPC_SUBNET.clone(), + IdentityMetadataCreateParams { + name: name.parse().unwrap(), + description: name, + }, + nexus.ip, + nexus.mac, + 0, + ) + .expect("failed to create incomplete Nexus NIC"); + (service, nic) + }) + } + + fn omicron_zone_configs( + &self, + ) -> impl Iterator + '_ { + self.db_services().map(|(service, nic)| { + let zone_config = OmicronZoneConfig { + id: service.id(), + underlay_address: "::1".parse().unwrap(), + zone_type: OmicronZoneType::Nexus { + internal_address: "[::1]:0".to_string(), + external_ip: "::1".parse().unwrap(), + nic: NetworkInterface { + id: nic.identity.id, + kind: NetworkInterfaceKind::Service(service.id()), + name: format!("test-nic-{}", nic.identity.id) + .parse() + .unwrap(), + ip: nic.ip.unwrap(), + mac: nic.mac.unwrap(), + subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, + slot: nic.slot.unwrap(), + }, + external_tls: false, + external_dns_servers: Vec::new(), + }, + }; + (service.sled_id, zone_config) + }) + } + } + + #[tokio::test] + async fn test_vpc_resolve_to_sleds_uses_current_target_blueprint() { + // Test setup. + usdt::register_probes().unwrap(); + let logctx = dev::test_setup_log( + "test_vpc_resolve_to_sleds_uses_current_target_blueprint", + ); + let db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // Helper function to fetch and sort the IDs of sleds we've resolved the + // SERVICES_VPC_ID to. + let fetch_service_sled_ids = || async { + let mut service_sled_ids = datastore + .vpc_resolve_to_sleds(*SERVICES_VPC_ID, &[]) + .await + .expect("failed to resolve to sleds") + .into_iter() + .map(|sled| sled.id()) + .collect::>(); + service_sled_ids.sort(); + service_sled_ids + }; + + // Create four sleds. + let harness = Harness::new(4); + for sled in harness.db_sleds() { + datastore + .sled_upsert(sled) + .await + .expect("failed to upsert sled") + .unwrap(); + } + + // Insert two Nexus records into `service`, emulating RSS. + for (service, nic) in harness.db_services().take(2) { + datastore + .service_upsert(&opctx, service) + .await + .expect("failed to insert RSS-like service"); + datastore + .service_create_network_interface_raw(&opctx, nic) + .await + .expect("failed to insert Nexus NIC"); + } + + // Ensure we find the two sleds we expect after adding Nexus records. + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Create a blueprint that has a Nexus on our third sled. (This + // blueprint is completely invalid in many ways, but all we care about + // here is inserting relevant records in `bp_omicron_zone`.) + let bp1_omicron_zones = { + let (sled_id, zone_config) = harness + .omicron_zone_configs() + .nth(2) + .expect("fewer than 3 services in test harness"); + let mut zones = BTreeMap::new(); + zones.insert( + sled_id, + OmicronZonesConfig { + generation: Generation::new(), + zones: vec![zone_config], + }, + ); + zones + }; + let bp1_id = Uuid::new_v4(); + let bp1 = Blueprint { + id: bp1_id, + omicron_zones: bp1_omicron_zones, + zones_in_service: BTreeSet::new(), + parent_blueprint_id: None, + internal_dns_version: Generation::new(), + time_created: Utc::now(), + creator: "test".to_string(), + comment: "test".to_string(), + }; + datastore + .blueprint_insert(&opctx, &bp1) + .await + .expect("failed to insert blueprint"); + + // We haven't set a blueprint target yet, so we should still only see + // the two RSS-inserted service-running sleds. + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Make bp1 the current target. + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: bp1_id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("failed to set blueprint target"); + + // bp1 is the target, but we haven't yet inserted a vNIC record, so + // we'll still only see the original 2 sleds. + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Insert the relevant service NIC record (normally performed by the + // reconfigurator's executor). + datastore + .service_create_network_interface_raw( + &opctx, + harness.db_services().nth(2).unwrap().1, + ) + .await + .expect("failed to insert service VNIC"); + + // We should now see _three_ sleds running services. + assert_eq!(&harness.sled_ids[..3], fetch_service_sled_ids().await); + + // Create another blueprint with no services and make it the target. + let bp2_id = Uuid::new_v4(); + let bp2 = Blueprint { + id: bp2_id, + omicron_zones: BTreeMap::new(), + zones_in_service: BTreeSet::new(), + parent_blueprint_id: Some(bp1_id), + internal_dns_version: Generation::new(), + time_created: Utc::now(), + creator: "test".to_string(), + comment: "test".to_string(), + }; + datastore + .blueprint_insert(&opctx, &bp2) + .await + .expect("failed to insert blueprint"); + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: bp2_id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("failed to set blueprint target"); + + // We haven't removed the service NIC record, but we should no longer + // see the third sled here, because we should be back to just the + // original two services in the `service` table. + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Insert a service NIC record for our fourth sled's Nexus. This + // shouldn't change our VPC resolution. + datastore + .service_create_network_interface_raw( + &opctx, + harness.db_services().nth(3).unwrap().1, + ) + .await + .expect("failed to insert service VNIC"); + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Create a blueprint that has a Nexus on our fourth sled. This + // shouldn't change our VPC resolution. + let bp3_omicron_zones = { + let (sled_id, zone_config) = harness + .omicron_zone_configs() + .nth(3) + .expect("fewer than 3 services in test harness"); + let mut zones = BTreeMap::new(); + zones.insert( + sled_id, + OmicronZonesConfig { + generation: Generation::new(), + zones: vec![zone_config], + }, + ); + zones + }; + let bp3_id = Uuid::new_v4(); + let bp3 = Blueprint { + id: bp3_id, + omicron_zones: bp3_omicron_zones, + zones_in_service: BTreeSet::new(), + parent_blueprint_id: Some(bp2_id), + internal_dns_version: Generation::new(), + time_created: Utc::now(), + creator: "test".to_string(), + comment: "test".to_string(), + }; + datastore + .blueprint_insert(&opctx, &bp3) + .await + .expect("failed to insert blueprint"); + assert_eq!(&harness.sled_ids[..2], fetch_service_sled_ids().await); + + // Make this blueprint the target. We've already created the service + // VNIC, so we should immediately see our fourth sled in VPC resolution. + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: bp3_id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("failed to set blueprint target"); + assert_eq!( + &[harness.sled_ids[0], harness.sled_ids[1], harness.sled_ids[3]] + as &[Uuid], + fetch_service_sled_ids().await + ); + + // Finally, create a blueprint that includes our third and fourth sleds, + // make it the target, and ensure we resolve to all four sleds. + let bp4_omicron_zones = { + let mut zones = BTreeMap::new(); + for (sled_id, zone_config) in harness.omicron_zone_configs().skip(2) + { + zones.insert( + sled_id, + OmicronZonesConfig { + generation: Generation::new(), + zones: vec![zone_config], + }, + ); + } + zones + }; + let bp4_id = Uuid::new_v4(); + let bp4 = Blueprint { + id: bp4_id, + omicron_zones: bp4_omicron_zones, + zones_in_service: BTreeSet::new(), + parent_blueprint_id: Some(bp3_id), + internal_dns_version: Generation::new(), + time_created: Utc::now(), + creator: "test".to_string(), + comment: "test".to_string(), + }; + datastore + .blueprint_insert(&opctx, &bp4) + .await + .expect("failed to insert blueprint"); + datastore + .blueprint_target_set_current( + &opctx, + BlueprintTarget { + target_id: bp4_id, + enabled: true, + time_made_target: Utc::now(), + }, + ) + .await + .expect("failed to set blueprint target"); + assert_eq!(harness.sled_ids, fetch_service_sled_ids().await); + } } diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index 2f4807d38d..949aa2779c 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -12,6 +12,7 @@ dns-service-client.workspace = true futures.workspace = true illumos-utils.workspace = true internal-dns.workspace = true +nexus-capabilities.workspace = true nexus-db-model.workspace = true nexus-db-queries.workspace = true nexus-types.workspace = true diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index 74b4764d7a..cb4109c643 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -7,6 +7,8 @@ //! See `nexus_reconfigurator_planning` crate-level docs for background. use anyhow::{anyhow, Context}; +use nexus_capabilities::Base; +use nexus_capabilities::FirewallRules; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::Blueprint; @@ -14,6 +16,7 @@ use nexus_types::identity::Asset; use omicron_common::address::Ipv6Subnet; use omicron_common::address::SLED_PREFIX; use slog::info; +use slog::Logger; use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; use std::net::SocketAddrV6; @@ -46,6 +49,26 @@ impl From for Sled { } } +// A vastly-restricted `Nexus` object that allows us access to some of Nexus +// proper's capabilities. +struct NexusContext<'a> { + opctx: OpContext, + datastore: &'a DataStore, +} + +impl Base for NexusContext<'_> { + fn log(&self) -> &Logger { + &self.opctx.log + } + + fn datastore(&self) -> &DataStore { + self.datastore + } +} + +impl nexus_capabilities::SledAgent for NexusContext<'_> {} +impl nexus_capabilities::FirewallRules for NexusContext<'_> {} + /// Make one attempt to realize the given blueprint, meaning to take actions to /// alter the real system to match the blueprint /// @@ -64,43 +87,59 @@ where "comment".to_string(), blueprint.comment.clone(), )])); + let nexusctx = NexusContext { opctx, datastore }; info!( - opctx.log, + nexusctx.log(), "attempting to realize blueprint"; - "blueprint_id" => ?blueprint.id + "blueprint_id" => %blueprint.id ); resource_allocation::ensure_zone_resources_allocated( - &opctx, - datastore, + &nexusctx.opctx, + nexusctx.datastore, &blueprint.omicron_zones, ) .await .map_err(|err| vec![err])?; - let sleds_by_id: BTreeMap = datastore - .sled_list_all_batched(&opctx) + let sleds_by_id: BTreeMap = nexusctx + .datastore + .sled_list_all_batched(&nexusctx.opctx) .await .context("listing all sleds") .map_err(|e| vec![e])? .into_iter() .map(|db_sled| (db_sled.id(), Sled::from(db_sled))) .collect(); - omicron_zones::deploy_zones(&opctx, &sleds_by_id, &blueprint.omicron_zones) - .await?; + omicron_zones::deploy_zones( + &nexusctx, + &sleds_by_id, + &blueprint.omicron_zones, + ) + .await?; + + // After deploying omicron zones, we may need to refresh OPTE service + // firewall rules. This is an idempotent operation, so we don't attempt + // to optimize out calling it in unnecessary cases, although we expect + // _most_ cases this is not needed. + nexusctx + .plumb_service_firewall_rules(&nexusctx.opctx, &[], &nexusctx.opctx) + .await + .context("failed to plumb service firewall rules to sleds") + .map_err(|err| vec![err])?; datasets::ensure_crucible_dataset_records_exist( - &opctx, - datastore, + &nexusctx.opctx, + nexusctx.datastore, blueprint.all_omicron_zones().map(|(_sled_id, zone)| zone), ) .await .map_err(|err| vec![err])?; dns::deploy_dns( - &opctx, - datastore, + &nexusctx.opctx, + nexusctx.datastore, String::from(nexus_label), blueprint, &sleds_by_id, diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index 1d5c4444b1..f8755212d1 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -4,14 +4,15 @@ //! Manges deployment of Omicron zones to Sled Agents +use crate::NexusContext; use crate::Sled; use anyhow::anyhow; use anyhow::Context; use futures::stream; use futures::StreamExt; -use nexus_db_queries::context::OpContext; +use nexus_capabilities::Base; +use nexus_capabilities::SledAgent; use nexus_types::deployment::OmicronZonesConfig; -use sled_agent_client::Client as SledAgentClient; use slog::info; use slog::warn; use std::collections::BTreeMap; @@ -20,7 +21,7 @@ use uuid::Uuid; /// Idempotently ensure that the specified Omicron zones are deployed to the /// corresponding sleds pub(crate) async fn deploy_zones( - opctx: &OpContext, + nexusctx: &NexusContext<'_>, sleds_by_id: &BTreeMap, zones: &BTreeMap, ) -> Result<(), Vec> { @@ -30,23 +31,24 @@ pub(crate) async fn deploy_zones( Some(sled) => sled, None => { let err = anyhow!("sled not found in db list: {}", sled_id); - warn!(opctx.log, "{err:#}"); + warn!(nexusctx.log(), "{err:#}"); return Some(err); } }; - let client = sled_client(opctx, &db_sled); + let client = + nexusctx.sled_client(*sled_id, db_sled.sled_agent_address); let result = - client.omicron_zones_put(&config).await.with_context(|| { + client.omicron_zones_put(config).await.with_context(|| { format!("Failed to put {config:#?} to sled {sled_id}") }); match result { Err(error) => { - warn!(opctx.log, "{error:#}"); + warn!(nexusctx.log(), "{error:#}"); Some(error) } Ok(_) => { info!( - opctx.log, + nexusctx.log(), "Successfully deployed zones for sled agent"; "sled_id" => %sled_id, "generation" => %config.generation, @@ -65,29 +67,10 @@ pub(crate) async fn deploy_zones( } } -// This is a modified copy of the functionality from `nexus/src/app/sled.rs`. -// There's no good way to access this functionality right now since it is a -// method on the `Nexus` type. We want to have a more constrained type we can -// pass into background tasks for this type of functionality, but for now we -// just copy the functionality. -fn sled_client(opctx: &OpContext, sled: &Sled) -> SledAgentClient { - let dur = std::time::Duration::from_secs(60); - let client = reqwest::ClientBuilder::new() - .connect_timeout(dur) - .timeout(dur) - .build() - .unwrap(); - SledAgentClient::new_with_client( - &format!("http://{}", sled.sled_agent_address), - client, - opctx.log.clone(), - ) -} - #[cfg(test)] mod test { use super::deploy_zones; - use crate::Sled; + use crate::{NexusContext, Sled}; use httptest::matchers::{all_of, json_decoded, request}; use httptest::responders::status_code; use httptest::Expectation; @@ -135,10 +118,13 @@ mod test { async fn test_deploy_omicron_zones(cptestctx: &ControlPlaneTestContext) { let nexus = &cptestctx.server.apictx().nexus; let datastore = nexus.datastore(); - let opctx = OpContext::for_tests( - cptestctx.logctx.log.clone(), - datastore.clone(), - ); + let nexusctx = NexusContext { + opctx: OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ), + datastore, + }; // Create some fake sled-agent servers to respond to zone puts and add // sleds to CRDB. @@ -165,7 +151,7 @@ mod test { // Get a success result back when the blueprint has an empty set of // zones. let blueprint = Arc::new(create_blueprint(BTreeMap::new())); - deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones) + deploy_zones(&nexusctx, &sleds_by_id, &blueprint.1.omicron_zones) .await .expect("failed to deploy no zones"); @@ -220,7 +206,7 @@ mod test { } // Execute it. - deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones) + deploy_zones(&nexusctx, &sleds_by_id, &blueprint.1.omicron_zones) .await .expect("failed to deploy initial zones"); @@ -237,7 +223,7 @@ mod test { .respond_with(status_code(204)), ); } - deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones) + deploy_zones(&nexusctx, &sleds_by_id, &blueprint.1.omicron_zones) .await .expect("failed to deploy same zones"); s1.verify_and_clear(); @@ -261,7 +247,7 @@ mod test { ); let errors = - deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones) + deploy_zones(&nexusctx, &sleds_by_id, &blueprint.1.omicron_zones) .await .expect_err("unexpectedly succeeded in deploying zones"); @@ -311,7 +297,7 @@ mod test { } // Activate the task - deploy_zones(&opctx, &sleds_by_id, &blueprint.1.omicron_zones) + deploy_zones(&nexusctx, &sleds_by_id, &blueprint.1.omicron_zones) .await .expect("failed to deploy last round of zones"); s1.verify_and_clear(); diff --git a/nexus/src/app/background/blueprint_execution.rs b/nexus/src/app/background/blueprint_execution.rs index 3c2530a3d3..7e670f9a3f 100644 --- a/nexus/src/app/background/blueprint_execution.rs +++ b/nexus/src/app/background/blueprint_execution.rs @@ -109,6 +109,7 @@ mod test { use nexus_db_model::{ ByteCount, SledBaseboard, SledSystemHardware, SledUpdate, }; + use nexus_db_queries::authn; use nexus_db_queries::context::OpContext; use nexus_test_utils_macros::nexus_test; use nexus_types::deployment::OmicronZonesConfig; @@ -158,8 +159,10 @@ mod test { // Set up the test. let nexus = &cptestctx.server.apictx().nexus; let datastore = nexus.datastore(); - let opctx = OpContext::for_tests( + let opctx = OpContext::for_background( cptestctx.logctx.log.clone(), + nexus.authz.clone(), + authn::Context::internal_api(), datastore.clone(), ); diff --git a/nexus/src/app/capabilities.rs b/nexus/src/app/capabilities.rs new file mode 100644 index 0000000000..047e158615 --- /dev/null +++ b/nexus/src/app/capabilities.rs @@ -0,0 +1,22 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::Nexus; +use nexus_db_queries::db; +use slog::Logger; + +impl nexus_capabilities::Base for Nexus { + fn log(&self) -> &Logger { + &self.log + } + + fn datastore(&self) -> &db::DataStore { + &self.db_datastore + } +} + +// `Nexus` proper has all capabilities. Other contexts (background tasks, sagas) +// may choose to implement objects with a subset. +impl nexus_capabilities::SledAgent for Nexus {} +impl nexus_capabilities::FirewallRules for Nexus {} diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 88a427af6a..661629dfad 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -18,6 +18,8 @@ use crate::external_api::params; use cancel_safe_futures::prelude::*; use futures::future::Fuse; use futures::{FutureExt, SinkExt, StreamExt}; +use nexus_capabilities::FirewallRules; +use nexus_capabilities::SledAgent; use nexus_db_model::IpAttachState; use nexus_db_model::IpKind; use nexus_db_queries::authn; @@ -535,7 +537,7 @@ impl super::Nexus { .lookup_for(authz::Action::Modify) .await?; - let sa = self.sled_client(&sled_id).await?; + let sa = self.sled_client_by_id(&self.opctx_alloc, sled_id).await?; let instance_put_result = sa .instance_put_migration_ids( &instance_id, @@ -606,7 +608,7 @@ impl super::Nexus { assert!(prev_instance_runtime.migration_id.is_some()); assert!(prev_instance_runtime.dst_propolis_id.is_some()); - let sa = self.sled_client(&sled_id).await?; + let sa = self.sled_client_by_id(&self.opctx_alloc, sled_id).await?; let instance_put_result = sa .instance_put_migration_ids( &instance_id, @@ -797,7 +799,7 @@ impl super::Nexus { ) -> Result, InstanceStateChangeError> { opctx.authorize(authz::Action::Modify, authz_instance).await?; - let sa = self.sled_client(&sled_id).await?; + let sa = self.sled_client_by_id(&self.opctx_alloc, *sled_id).await?; sa.instance_unregister(&authz_instance.id()) .await .map(|res| res.into_inner().updated_runtime.map(Into::into)) @@ -969,7 +971,8 @@ impl super::Nexus { )? { InstanceStateChangeRequestAction::AlreadyDone => Ok(()), InstanceStateChangeRequestAction::SendToSled(sled_id) => { - let sa = self.sled_client(&sled_id).await?; + let sa = + self.sled_client_by_id(&self.opctx_alloc, sled_id).await?; let instance_put_result = sa .instance_put_state( &instance_id, @@ -1211,7 +1214,9 @@ impl super::Nexus { )), }; - let sa = self.sled_client(&initial_vmm.sled_id).await?; + let sa = self + .sled_client_by_id(&self.opctx_alloc, initial_vmm.sled_id) + .await?; let instance_register_result = sa .instance_register( &db_instance.id(), diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index c0bc5d237b..ac4e8f1a82 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -7,6 +7,7 @@ use crate::app::sagas::retry_until_known_result; use ipnetwork::IpNetwork; use ipnetwork::Ipv6Network; +use nexus_capabilities::SledAgent; use nexus_db_model::ExternalIp; use nexus_db_model::IpAttachState; use nexus_db_model::Ipv4NatEntry; @@ -108,7 +109,7 @@ impl super::Nexus { // Look up the supplied sled's physical host IP. let physical_host_ip = - *self.sled_lookup(&self.opctx_alloc, &sled_id)?.fetch().await?.1.ip; + *self.sled_lookup(&self.opctx_alloc, sled_id).fetch().await?.1.ip; let mut last_sled_id: Option = None; loop { @@ -134,7 +135,9 @@ impl super::Nexus { } for nic in &instance_nics { - let client = self.sled_client(&sled.id()).await?; + let client = self + .sled_client_by_id(&self.opctx_alloc, sled.id()) + .await?; let nic_id = nic.id; let mapping = SetVirtualNetworkInterfaceHost { virtual_ip: nic.ip, @@ -223,7 +226,9 @@ impl super::Nexus { for sled in &sleds_page { for nic in &instance_nics { - let client = self.sled_client(&sled.id()).await?; + let client = self + .sled_client_by_id(&self.opctx_alloc, sled.id()) + .await?; let nic_id = nic.id; let mapping = DeleteVirtualNetworkInterfaceHost { virtual_ip: nic.ip, diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index 969320617f..d0b6c3374b 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -38,6 +38,7 @@ mod address_lot; pub(crate) mod background; mod bfd; mod bgp; +mod capabilities; mod certificate; mod deployment; mod device_auth; diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index a137f19434..5bb5e1c96d 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -11,6 +11,7 @@ use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; use gateway_client::types::SpType; use ipnetwork::{IpNetwork, Ipv6Network}; +use nexus_capabilities::FirewallRules; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; use nexus_db_model::{SwitchLinkFec, SwitchLinkSpeed}; @@ -239,7 +240,8 @@ impl super::Nexus { .await?; // Plumb the firewall rules for the built-in services - self.plumb_service_firewall_rules(opctx, &[]).await?; + self.plumb_service_firewall_rules(opctx, &[], &self.opctx_alloc) + .await?; // We've potentially updated the list of DNS servers and the DNS // configuration for both internal and external DNS, plus the Silo diff --git a/nexus/src/app/sagas/instance_common.rs b/nexus/src/app/sagas/instance_common.rs index e915c026dd..3a406b9944 100644 --- a/nexus/src/app/sagas/instance_common.rs +++ b/nexus/src/app/sagas/instance_common.rs @@ -8,6 +8,7 @@ use std::net::{IpAddr, Ipv6Addr}; use crate::Nexus; use chrono::Utc; +use nexus_capabilities::SledAgent; use nexus_db_model::{ ByteCount, ExternalIp, IpAttachState, Ipv4NatEntry, SledReservationConstraints, SledResource, @@ -392,7 +393,7 @@ pub async fn instance_ip_add_opte( osagactx .nexus() - .sled_client(&sled_uuid) + .sled_client_by_id(&osagactx.nexus().opctx_alloc, sled_uuid) .await .map_err(|_| { ActionError::action_failed(Error::unavail( @@ -447,7 +448,7 @@ pub async fn instance_ip_remove_opte( osagactx .nexus() - .sled_client(&sled_uuid) + .sled_client_by_id(&osagactx.nexus().opctx_alloc, sled_uuid) .await .map_err(|_| { ActionError::action_failed(Error::unavail( diff --git a/nexus/src/app/sagas/snapshot_create.rs b/nexus/src/app/sagas/snapshot_create.rs index e017ab377b..0802fe830d 100644 --- a/nexus/src/app/sagas/snapshot_create.rs +++ b/nexus/src/app/sagas/snapshot_create.rs @@ -104,6 +104,7 @@ use crate::app::{authn, authz, db}; use crate::external_api::params; use anyhow::anyhow; use crucible_agent_client::{types::RegionId, Client as CrucibleAgentClient}; +use nexus_capabilities::SledAgent; use nexus_db_model::Generation; use nexus_db_queries::db::identity::{Asset, Resource}; use nexus_db_queries::db::lookup::LookupPath; @@ -695,7 +696,7 @@ async fn ssc_send_snapshot_request_to_sled_agent( let sled_agent_client = osagactx .nexus() - .sled_client(&sled_id) + .sled_client_by_id(&osagactx.nexus().opctx_alloc, sled_id) .await .map_err(ActionError::action_failed)?; @@ -1974,7 +1975,8 @@ mod test { let sled_id = instance_state .sled_id() .expect("starting instance should have a sled"); - let sa = nexus.sled_client(&sled_id).await.unwrap(); + let sa = + nexus.sled_client_by_id(&nexus.opctx_alloc, sled_id).await.unwrap(); sa.instance_finish_transition(instance.identity.id).await; let instance_state = nexus diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 6b48e4087a..a62216b416 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -8,6 +8,7 @@ use super::NexusSaga; use super::ACTION_GENERATE_ID; use crate::app::sagas::declare_saga_actions; use crate::external_api::params; +use nexus_capabilities::FirewallRules; use nexus_db_queries::db::queries::vpc_subnet::SubnetError; use nexus_db_queries::{authn, authz, db}; use nexus_defaults as defaults; @@ -432,7 +433,13 @@ async fn svc_notify_sleds( osagactx .nexus() - .send_sled_agents_firewall_rules(&opctx, &db_vpc, &rules, &[]) + .send_sled_agents_firewall_rules( + &opctx, + &db_vpc, + &rules, + &[], + &osagactx.nexus().opctx_alloc, + ) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 88955d78e9..e3b319ddc0 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -8,6 +8,7 @@ use crate::internal_api::params::{ PhysicalDiskDeleteRequest, PhysicalDiskPutRequest, SledAgentStartupInfo, SledRole, ZpoolPutRequest, }; +use nexus_capabilities::FirewallRules; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -19,22 +20,11 @@ use nexus_types::external_api::views::SledProvisionPolicy; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; -use omicron_common::api::external::LookupResult; -use sled_agent_client::Client as SledAgentClient; use std::net::SocketAddrV6; -use std::sync::Arc; use uuid::Uuid; impl super::Nexus { // Sleds - pub fn sled_lookup<'a>( - &'a self, - opctx: &'a OpContext, - sled_id: &Uuid, - ) -> LookupResult> { - let sled = LookupPath::new(opctx, &self.db_datastore).sled_id(*sled_id); - Ok(sled) - } // TODO-robustness we should have a limit on how many sled agents there can // be (for graceful degradation at large scale). @@ -93,7 +83,8 @@ impl super::Nexus { id: Uuid, ) -> Result<(), Error> { info!(self.log, "requesting firewall rules"; "sled_uuid" => id.to_string()); - self.plumb_service_firewall_rules(opctx, &[id]).await?; + self.plumb_service_firewall_rules(opctx, &[id], &self.opctx_alloc) + .await?; Ok(()) } @@ -105,34 +96,6 @@ impl super::Nexus { self.db_datastore.sled_list(&opctx, pagparams).await } - pub async fn sled_client( - &self, - id: &Uuid, - ) -> Result, Error> { - // TODO: We should consider injecting connection pooling here, - // but for now, connections to sled agents are constructed - // on an "as requested" basis. - // - // Frankly, returning an "Arc" here without a connection pool is a - // little silly; it's not actually used if each client connection exists - // as a one-shot. - let (.., sled) = - self.sled_lookup(&self.opctx_alloc, id)?.fetch().await?; - - let log = self.log.new(o!("SledAgent" => id.clone().to_string())); - let dur = std::time::Duration::from_secs(60); - let client = reqwest::ClientBuilder::new() - .connect_timeout(dur) - .timeout(dur) - .build() - .unwrap(); - Ok(Arc::new(SledAgentClient::new_with_client( - &format!("http://{}", sled.address()), - client, - log, - ))) - } - pub(crate) async fn reserve_on_random_sled( &self, resource_id: Uuid, @@ -292,25 +255,4 @@ impl super::Nexus { self.db_datastore.dataset_upsert(dataset).await?; Ok(()) } - - /// Ensure firewall rules for internal services get reflected on all the relevant sleds. - pub(crate) async fn plumb_service_firewall_rules( - &self, - opctx: &OpContext, - sleds_filter: &[Uuid], - ) -> Result<(), Error> { - let svcs_vpc = LookupPath::new(opctx, &self.db_datastore) - .vpc_id(*db::fixed_data::vpc::SERVICES_VPC_ID); - let svcs_fw_rules = - self.vpc_list_firewall_rules(opctx, &svcs_vpc).await?; - let (_, _, _, svcs_vpc) = svcs_vpc.fetch().await?; - self.send_sled_agents_firewall_rules( - opctx, - &svcs_vpc, - &svcs_fw_rules, - sleds_filter, - ) - .await?; - Ok(()) - } } diff --git a/nexus/src/app/test_interfaces.rs b/nexus/src/app/test_interfaces.rs index 581b9a89bb..a1fd574398 100644 --- a/nexus/src/app/test_interfaces.rs +++ b/nexus/src/app/test_interfaces.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use async_trait::async_trait; +use nexus_capabilities::SledAgent; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::Error; @@ -86,7 +87,7 @@ impl TestInterfaces for super::Nexus { ) -> Result>, Error> { let sled_id = self.instance_sled_id_with_opctx(id, opctx).await?; if let Some(sled_id) = sled_id { - Ok(Some(self.sled_client(&sled_id).await?)) + Ok(Some(self.sled_client_by_id(&self.opctx_alloc, sled_id).await?)) } else { Ok(None) } diff --git a/nexus/src/app/vpc.rs b/nexus/src/app/vpc.rs index 3a6278053a..da1fee4b2d 100644 --- a/nexus/src/app/vpc.rs +++ b/nexus/src/app/vpc.rs @@ -6,12 +6,11 @@ use crate::app::sagas; use crate::external_api::params; +use nexus_capabilities::FirewallRules; use nexus_db_queries::authn; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; -use nexus_db_queries::db::identity::Asset; -use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::Name; @@ -22,20 +21,13 @@ use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; use omicron_common::api::external::InternalContext; -use omicron_common::api::external::IpNet; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; use omicron_common::api::external::LookupType; use omicron_common::api::external::NameOrId; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::VpcFirewallRuleUpdateParams; -use omicron_common::api::internal::nexus::HostIdentifier; -use sled_agent_client::types::NetworkInterface; -use futures::future::join_all; -use ipnetwork::IpNetwork; -use std::collections::{HashMap, HashSet}; -use std::net::IpAddr; use std::sync::Arc; use uuid::Uuid; @@ -165,20 +157,6 @@ impl super::Nexus { // Firewall rules - pub(crate) async fn vpc_list_firewall_rules( - &self, - opctx: &OpContext, - vpc_lookup: &lookup::Vpc<'_>, - ) -> ListResultVec { - let (.., authz_vpc) = - vpc_lookup.lookup_for(authz::Action::Read).await?; - let rules = self - .db_datastore - .vpc_list_firewall_rules(&opctx, &authz_vpc) - .await?; - Ok(rules) - } - pub(crate) async fn vpc_update_firewall_rules( &self, opctx: &OpContext, @@ -195,8 +173,14 @@ impl super::Nexus { .db_datastore .vpc_update_firewall_rules(opctx, &authz_vpc, rules) .await?; - self.send_sled_agents_firewall_rules(opctx, &db_vpc, &rules, &[]) - .await?; + self.send_sled_agents_firewall_rules( + opctx, + &db_vpc, + &rules, + &[], + &self.opctx_alloc, + ) + .await?; Ok(rules) } @@ -244,392 +228,4 @@ impl super::Nexus { debug!(self.log, "default firewall rules for vpc {}", vpc_name; "rules" => ?&rules); Ok(rules) } - - pub(crate) async fn send_sled_agents_firewall_rules( - &self, - opctx: &OpContext, - vpc: &db::model::Vpc, - rules: &[db::model::VpcFirewallRule], - sleds_filter: &[Uuid], - ) -> Result<(), Error> { - let rules_for_sled = self - .resolve_firewall_rules_for_sled_agent(opctx, &vpc, rules) - .await?; - debug!(self.log, "resolved {} rules for sleds", rules_for_sled.len()); - let sled_rules_request = - sled_agent_client::types::VpcFirewallRulesEnsureBody { - vni: vpc.vni.0, - rules: rules_for_sled, - }; - - let vpc_to_sleds = self - .db_datastore - .vpc_resolve_to_sleds(vpc.id(), sleds_filter) - .await?; - debug!(self.log, "resolved sleds for vpc {}", vpc.name(); "vpc_to_sled" => ?vpc_to_sleds); - - let mut sled_requests = Vec::with_capacity(vpc_to_sleds.len()); - for sled in &vpc_to_sleds { - let sled_id = sled.id(); - let vpc_id = vpc.id(); - let sled_rules_request = sled_rules_request.clone(); - sled_requests.push(async move { - self.sled_client(&sled_id) - .await? - .vpc_firewall_rules_put(&vpc_id, &sled_rules_request) - .await - .map_err(|e| Error::internal_error(&e.to_string())) - }); - } - - debug!(self.log, "sending firewall rules to sled agents"); - let results = join_all(sled_requests).await; - // TODO-correctness: handle more than one failure in the sled-agent requests - // https://github.com/oxidecomputer/omicron/issues/1791 - for (sled, result) in vpc_to_sleds.iter().zip(results) { - if let Err(e) = result { - warn!(self.log, "failed to update firewall rules on sled agent"; - "sled_id" => %sled.id(), - "vpc_id" => %vpc.id(), - "error" => %e); - return Err(e); - } - } - info!( - self.log, - "updated firewall rules on {} sleds", - vpc_to_sleds.len() - ); - - Ok(()) - } - - pub(crate) async fn resolve_firewall_rules_for_sled_agent( - &self, - opctx: &OpContext, - vpc: &db::model::Vpc, - rules: &[db::model::VpcFirewallRule], - ) -> Result, Error> { - // Collect the names of instances, subnets, and VPCs that are either - // targets or host filters. We have to find the sleds for all the - // targets, and we'll need information about the IP addresses or - // subnets for things that are specified as host filters as well. - let mut instances: HashSet = HashSet::new(); - let mut subnets: HashSet = HashSet::new(); - let mut vpcs: HashSet = HashSet::new(); - for rule in rules { - for target in &rule.targets { - match &target.0 { - external::VpcFirewallRuleTarget::Instance(name) => { - instances.insert(name.clone().into()); - } - external::VpcFirewallRuleTarget::Subnet(name) => { - subnets.insert(name.clone().into()); - } - external::VpcFirewallRuleTarget::Vpc(name) => { - if name != vpc.name() { - return Err(Error::invalid_request( - "cross-VPC firewall target unsupported", - )); - } - vpcs.insert(name.clone().into()); - } - external::VpcFirewallRuleTarget::Ip(_) - | external::VpcFirewallRuleTarget::IpNet(_) => { - vpcs.insert(vpc.name().clone().into()); - } - } - } - - for host in rule.filter_hosts.iter().flatten() { - match &host.0 { - external::VpcFirewallRuleHostFilter::Instance(name) => { - instances.insert(name.clone().into()); - } - external::VpcFirewallRuleHostFilter::Subnet(name) => { - subnets.insert(name.clone().into()); - } - external::VpcFirewallRuleHostFilter::Vpc(name) => { - if name != vpc.name() { - return Err(Error::invalid_request( - "cross-VPC firewall host filter unsupported", - )); - } - vpcs.insert(name.clone().into()); - } - // We don't need to resolve anything for Ip(Net)s. - external::VpcFirewallRuleHostFilter::Ip(_) => (), - external::VpcFirewallRuleHostFilter::IpNet(_) => (), - } - } - } - - // Resolve named instances, VPCs, and subnets. - // TODO-correctness: It's possible the resolving queries produce - // inconsistent results due to concurrent changes. They should be - // transactional. - type NetMap = HashMap>; - type NicMap = HashMap>; - let no_networks: Vec = Vec::new(); - let no_interfaces: Vec = Vec::new(); - - let mut instance_interfaces: NicMap = HashMap::new(); - for instance_name in &instances { - if let Ok((.., authz_instance)) = - LookupPath::new(opctx, &self.db_datastore) - .project_id(vpc.project_id) - .instance_name(instance_name) - .lookup_for(authz::Action::ListChildren) - .await - { - for iface in self - .db_datastore - .derive_guest_network_interface_info(opctx, &authz_instance) - .await? - { - instance_interfaces - .entry(instance_name.0.clone()) - .or_insert_with(Vec::new) - .push(iface); - } - } - } - - let mut vpc_interfaces: NicMap = HashMap::new(); - for vpc_name in &vpcs { - if let Ok((.., authz_vpc)) = - LookupPath::new(opctx, &self.db_datastore) - .project_id(vpc.project_id) - .vpc_name(vpc_name) - .lookup_for(authz::Action::ListChildren) - .await - { - for iface in self - .db_datastore - .derive_vpc_network_interface_info(opctx, &authz_vpc) - .await? - { - vpc_interfaces - .entry(vpc_name.0.clone()) - .or_insert_with(Vec::new) - .push(iface); - } - } - } - - let mut subnet_interfaces: NicMap = HashMap::new(); - for subnet_name in &subnets { - if let Ok((.., authz_subnet)) = - LookupPath::new(opctx, &self.db_datastore) - .project_id(vpc.project_id) - .vpc_name(&Name::from(vpc.name().clone())) - .vpc_subnet_name(subnet_name) - .lookup_for(authz::Action::ListChildren) - .await - { - for iface in self - .db_datastore - .derive_subnet_network_interface_info(opctx, &authz_subnet) - .await? - { - subnet_interfaces - .entry(subnet_name.0.clone()) - .or_insert_with(Vec::new) - .push(iface); - } - } - } - - let subnet_networks: NetMap = self - .db_datastore - .resolve_vpc_subnets_to_ip_networks(vpc, subnets) - .await? - .into_iter() - .map(|(name, v)| (name.0, v)) - .collect(); - - debug!( - self.log, - "resolved names for firewall rules"; - "instance_interfaces" => ?instance_interfaces, - "vpc_interfaces" => ?vpc_interfaces, - "subnet_interfaces" => ?subnet_interfaces, - "subnet_networks" => ?subnet_networks, - ); - - // Compile resolved rules for the sled agents. - let mut sled_agent_rules = Vec::with_capacity(rules.len()); - for rule in rules { - // TODO: what is the correct behavior when a name is not found? - // Options: - // (1) Fail update request (though note this can still arise - // from things like instance deletion) - // (2) Allow update request, ignore this rule (but store it - // in case it becomes valid later). This is consistent - // with the semantics of the rules. Rules with bad - // references should likely at least be flagged to users. - // We currently adopt option (2), as this allows users to add - // firewall rules (including default rules) before instances - // and their interfaces are instantiated. - - // Collect unique network interface targets. - // This would be easier if `NetworkInterface` were `Hash`, - // but that's not easy because it's a generated type. We - // use the pair (VNI, MAC) as a unique interface identifier. - let mut nics = HashSet::new(); - let mut targets = Vec::with_capacity(rule.targets.len()); - let mut push_target_nic = |nic: &NetworkInterface| { - if nics.insert((nic.vni, *nic.mac)) { - targets.push(nic.clone()); - } - }; - for target in &rule.targets { - match &target.0 { - external::VpcFirewallRuleTarget::Vpc(name) => { - vpc_interfaces - .get(&name) - .unwrap_or(&no_interfaces) - .iter() - .for_each(&mut push_target_nic); - } - external::VpcFirewallRuleTarget::Subnet(name) => { - subnet_interfaces - .get(&name) - .unwrap_or(&no_interfaces) - .iter() - .for_each(&mut push_target_nic); - } - external::VpcFirewallRuleTarget::Instance(name) => { - instance_interfaces - .get(&name) - .unwrap_or(&no_interfaces) - .iter() - .for_each(&mut push_target_nic); - } - external::VpcFirewallRuleTarget::Ip(addr) => { - vpc_interfaces - .get(vpc.name()) - .unwrap_or(&no_interfaces) - .iter() - .filter(|nic| nic.ip == *addr) - .for_each(&mut push_target_nic); - } - external::VpcFirewallRuleTarget::IpNet(net) => { - vpc_interfaces - .get(vpc.name()) - .unwrap_or(&no_interfaces) - .iter() - .filter(|nic| match (net, nic.ip) { - (IpNet::V4(net), IpAddr::V4(ip)) => { - net.contains(ip) - } - (IpNet::V6(net), IpAddr::V6(ip)) => { - net.contains(ip) - } - (_, _) => false, - }) - .for_each(&mut push_target_nic); - } - } - } - if !rule.targets.is_empty() && targets.is_empty() { - // Target not found; skip this rule. - continue; - } - - let filter_hosts = match &rule.filter_hosts { - None => None, - Some(hosts) => { - let mut host_addrs = Vec::with_capacity(hosts.len()); - for host in hosts { - match &host.0 { - external::VpcFirewallRuleHostFilter::Instance( - name, - ) => { - for interface in instance_interfaces - .get(&name) - .unwrap_or(&no_interfaces) - { - host_addrs.push( - HostIdentifier::Ip(IpNet::from( - interface.ip, - )) - .into(), - ) - } - } - external::VpcFirewallRuleHostFilter::Subnet( - name, - ) => { - for subnet in subnet_networks - .get(&name) - .unwrap_or(&no_networks) - { - host_addrs.push( - HostIdentifier::Ip(IpNet::from( - *subnet, - )) - .into(), - ); - } - } - external::VpcFirewallRuleHostFilter::Ip(addr) => { - host_addrs.push( - HostIdentifier::Ip(IpNet::from(*addr)) - .into(), - ) - } - external::VpcFirewallRuleHostFilter::IpNet(net) => { - host_addrs.push(HostIdentifier::Ip(*net).into()) - } - external::VpcFirewallRuleHostFilter::Vpc(name) => { - for interface in vpc_interfaces - .get(&name) - .unwrap_or(&no_interfaces) - { - host_addrs.push( - HostIdentifier::Vpc(interface.vni) - .into(), - ) - } - } - } - } - if !hosts.is_empty() && host_addrs.is_empty() { - // Filter host not found; skip this rule. - continue; - } - Some(host_addrs) - } - }; - - let filter_ports = rule - .filter_ports - .as_ref() - .map(|ports| ports.iter().map(|v| v.0.into()).collect()); - - let filter_protocols = - rule.filter_protocols.as_ref().map(|protocols| { - protocols.iter().map(|v| v.0.into()).collect() - }); - - sled_agent_rules.push(sled_agent_client::types::VpcFirewallRule { - status: rule.status.0.into(), - direction: rule.direction.0.into(), - targets, - filter_hosts, - filter_ports, - filter_protocols, - action: rule.action.0.into(), - priority: rule.priority.0 .0, - }); - } - debug!( - self.log, - "resolved firewall rules for sled agents"; - "sled_agent_rules" => ?sled_agent_rules, - ); - - Ok(sled_agent_rules) - } } diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index b007cc6217..765cbd4367 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -35,6 +35,8 @@ use dropshot::{ }; use dropshot::{ApiDescription, StreamingBody}; use ipnetwork::IpNetwork; +use nexus_capabilities::FirewallRules; +use nexus_capabilities::SledAgent; use nexus_db_queries::authz; use nexus_db_queries::db; use nexus_db_queries::db::identity::Resource; @@ -5160,7 +5162,7 @@ async fn sled_view( let path = path_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; let (.., sled) = - nexus.sled_lookup(&opctx, &path.sled_id)?.fetch().await?; + nexus.sled_lookup(&opctx, path.sled_id).fetch().await?; Ok(HttpResponseOk(sled.into())) }; apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await @@ -5186,7 +5188,7 @@ async fn sled_set_provision_policy( let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; + let sled_lookup = nexus.sled_lookup(&opctx, path.sled_id); let old_state = nexus .sled_set_provision_policy(&opctx, &sled_lookup, new_state) @@ -5217,7 +5219,7 @@ async fn sled_instance_list( let path = path_params.into_inner(); let query = query_params.into_inner(); let opctx = crate::context::op_context_for_external_api(&rqctx).await?; - let sled_lookup = nexus.sled_lookup(&opctx, &path.sled_id)?; + let sled_lookup = nexus.sled_lookup(&opctx, path.sled_id); let sled_instances = nexus .sled_instance_list( &opctx, diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 0df2b83008..5c7481159e 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -12,6 +12,9 @@ use camino::Utf8Path; use http::method::Method; use http::StatusCode; use itertools::Itertools; +use nexus_capabilities::SledAgent as NexusSledAgentCapabilities; +use nexus_db_queries::authn; +use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::fixed_data::silo::SILO_ID; @@ -4676,8 +4679,15 @@ async fn instance_simulate_on_sled( sled_id: Uuid, instance_id: Uuid, ) { - info!(&cptestctx.logctx.log, "Poking simulated instance on sled"; + let log = &cptestctx.logctx.log; + info!(log, "Poking simulated instance on sled"; "instance_id" => %instance_id, "sled_id" => %sled_id); - let sa = nexus.sled_client(&sled_id).await.unwrap(); + let opctx = OpContext::for_background( + log.clone(), + Arc::new(authz::Authz::new(log)), + authn::Context::internal_read(), + nexus.datastore().clone(), + ); + let sa = nexus.sled_client_by_id(&opctx, sled_id).await.unwrap(); sa.instance_finish_transition(instance_id).await; } diff --git a/schema/crdb/39.0.0/up.sql b/schema/crdb/39.0.0/up.sql new file mode 100644 index 0000000000..ce482b9e48 --- /dev/null +++ b/schema/crdb/39.0.0/up.sql @@ -0,0 +1,3 @@ +CREATE INDEX IF NOT EXISTS lookup_bp_target_by_id ON omicron.public.bp_target ( + blueprint_id +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 40a6fd463f..14c1b123ad 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3159,6 +3159,10 @@ CREATE TABLE IF NOT EXISTS omicron.public.bp_target ( time_made_target TIMESTAMPTZ NOT NULL ); +CREATE INDEX IF NOT EXISTS lookup_bp_target_by_id ON omicron.public.bp_target ( + blueprint_id +); + -- see inv_sled_omicron_zones, which is identical except it references a -- collection whereas this table references a blueprint CREATE TABLE IF NOT EXISTS omicron.public.bp_sled_omicron_zones ( @@ -3552,7 +3556,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '38.0.0', NULL) + ( TRUE, NOW(), NOW(), '39.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT;