diff --git a/bootstore/src/schemes/v0/peer.rs b/bootstore/src/schemes/v0/peer.rs index efb916a61fa..da8839dc170 100644 --- a/bootstore/src/schemes/v0/peer.rs +++ b/bootstore/src/schemes/v0/peer.rs @@ -53,7 +53,7 @@ pub enum NodeRequestError { #[error( "Network config update failed because it is out of date. Attempted - update generation: {attempted_update_generation}, current generation: + update generation: {attempted_update_generation}, current generation: {current_generation}" )] StaleNetworkConfig { diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 4eecd74a048..bf73d121a99 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -9,6 +9,7 @@ mod error; pub mod http_pagination; +pub use crate::api::internal::shared::AllowedSourceIps; pub use crate::api::internal::shared::SwitchLocation; use crate::update::ArtifactHash; use crate::update::ArtifactId; @@ -847,6 +848,7 @@ impl JsonSchema for Hostname { pub enum ResourceType { AddressLot, AddressLotBlock, + AllowedSourceIps, BackgroundTask, BgpConfig, BgpAnnounceSet, @@ -1431,6 +1433,18 @@ impl IpNet { } } } + + /// Return true if the provided address is contained in self. + /// + /// This returns false if the address and the network are of different IP + /// families. + pub fn contains(&self, addr: IpAddr) -> bool { + match (self, addr) { + (IpNet::V4(net), IpAddr::V4(ip)) => net.contains(ip), + (IpNet::V6(net), IpAddr::V6(ip)) => net.contains(ip), + (_, _) => false, + } + } } impl From for IpNet { diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 09c2e9b6edc..27020efcd53 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -6,7 +6,7 @@ use crate::{ address::NUM_SOURCE_NAT_PORTS, - api::external::{self, BfdMode, Name}, + api::external::{self, BfdMode, IpNet, Name}, }; use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network}; use schemars::JsonSchema; @@ -184,7 +184,7 @@ pub struct BgpConfig { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, JsonSchema)] pub struct BgpPeerConfig { - /// The autonomous sysetm number of the router the peer belongs to. + /// The autonomous system number of the router the peer belongs to. pub asn: u32, /// Switch port the peer is reachable on. pub port: String, @@ -422,3 +422,68 @@ impl fmt::Display for PortFec { } } } + +/// Description of source IPs allowed to reach rack services. +#[derive( + Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize, +)] +#[serde(rename_all = "snake_case", tag = "allow", content = "ips")] +pub enum AllowedSourceIps { + /// Allow traffic from any external IP address. + #[default] + Any, + /// Restrict access to a specific set of source IP addresses or subnets. + /// + /// All others are prevented from reaching rack services. + List(Vec), +} + +#[cfg(test)] +mod tests { + use crate::api::{ + external::{IpNet, Ipv4Net, Ipv6Net}, + internal::shared::AllowedSourceIps, + }; + use ipnetwork::{Ipv4Network, Ipv6Network}; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn test_deserialize_allowed_source_ips() { + let parsed: AllowedSourceIps = serde_json::from_str( + "{ \"allow\" : \"list\", \"ips\" : [\"127.0.0.1\", \"10.0.0.0/24\", \"fd00::1/64\"]}", + ) + .unwrap(); + assert_eq!( + parsed, + AllowedSourceIps::List(vec![ + IpNet::from(Ipv4Addr::LOCALHOST), + IpNet::V4(Ipv4Net( + Ipv4Network::new(Ipv4Addr::new(10, 0, 0, 0), 24).unwrap() + )), + IpNet::V6(Ipv6Net( + Ipv6Network::new( + Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1), + 64 + ) + .unwrap() + )), + ]) + ); + } + + #[test] + fn test_deserialize_unknown_string() { + serde_json::from_str::("{\"allow\": \"wat\"}") + .expect_err( + "Should not be able to deserialize from unknown variant name", + ); + } + + #[test] + fn test_deserialize_any_into_allowed_external_ips() { + assert_eq!( + AllowedSourceIps::Any, + serde_json::from_str("{\"allow\": \"any\"}").unwrap(), + ); + } +} diff --git a/nexus/db-model/src/allowed_source_ip.rs b/nexus/db-model/src/allowed_source_ip.rs new file mode 100644 index 00000000000..a4f23d9172c --- /dev/null +++ b/nexus/db-model/src/allowed_source_ip.rs @@ -0,0 +1,94 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/5.0/. + +// Copyright 2024 Oxide Computer Company + +//! Database representation of allowed source IP address, for implementing basic +//! IP allowlisting. + +use crate::schema::allowed_source_ip; +use crate::Generation; +use chrono::DateTime; +use chrono::Utc; +use ipnetwork::IpNetwork; +use nexus_types::external_api::params; +use nexus_types::external_api::views; +use omicron_common::api::external; +use omicron_common::api::external::IpNet; +use serde::Deserialize; +use serde::Serialize; +use uuid::Uuid; + +/// Database model for an allowlist of source IP addresses. +#[derive( + Queryable, Insertable, Selectable, Clone, Debug, Serialize, Deserialize, +)] +#[diesel(table_name = allowed_source_ip)] +pub struct AllowedSourceIp { + pub id: Uuid, + pub time_created: DateTime, + pub time_modified: DateTime, + pub generation: Generation, + pub allowed_ips: Option>, +} + +impl AllowedSourceIp { + /// Construct a new allowlist record. + pub fn new(id: Uuid, allowed_ips: external::AllowedSourceIps) -> Self { + let now = Utc::now(); + let allowed_ips = match allowed_ips { + external::AllowedSourceIps::Any => None, + external::AllowedSourceIps::List(list) => { + Some(list.into_iter().map(Into::into).collect()) + } + }; + Self { + id, + time_created: now, + time_modified: now, + generation: Generation::new(), + allowed_ips, + } + } +} + +#[derive(AsChangeset)] +#[diesel(table_name = allowed_source_ip, treat_none_as_null = true)] +pub struct AllowedSourceIpUpdate { + /// The new list of allowed IPs. + pub allowed_ips: Option>, +} + +impl From for AllowedSourceIpUpdate { + fn from(params: params::AllowedSourceIpsUpdate) -> Self { + let allowed_ips = match params.allowed_ips { + external::AllowedSourceIps::Any => None, + external::AllowedSourceIps::List(list) => { + Some(list.into_iter().map(Into::into).collect()) + } + }; + Self { allowed_ips } + } +} + +impl From for views::AllowedSourceIps { + fn from(db: AllowedSourceIp) -> Self { + let allowed_ips = if let Some(list) = db.allowed_ips { + assert!( + !list.is_empty(), + "IP allowlist must not be empty, use NULL instead" + ); + external::AllowedSourceIps::List( + list.into_iter().map(IpNet::from).collect(), + ) + } else { + external::AllowedSourceIps::Any + }; + Self { + time_created: db.time_created, + time_modified: db.time_modified, + allowed_ips, + } + } +} diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 6495a0c9604..0a51229cbc1 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -10,6 +10,7 @@ extern crate diesel; extern crate newtype_derive; mod address_lot; +mod allowed_source_ip; mod bfd; mod bgp; mod block_size; @@ -114,6 +115,7 @@ mod db { pub use self::macaddr::*; pub use self::unsigned::*; pub use address_lot::*; +pub use allowed_source_ip::*; pub use bfd::*; pub use bgp::*; pub use block_size::*; diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index fa03aca4fb8..70f1d86f10a 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -1624,6 +1624,16 @@ table! { } } +table! { + allowed_source_ip (id) { + id -> Uuid, + time_created -> Timestamptz, + time_modified -> Timestamptz, + generation -> Int8, + allowed_ips -> Nullable>, + } +} + table! { db_metadata (singleton) { singleton -> Bool, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 753d2a84798..419c3461d5c 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(55, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(56, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(56, "add-allowed-source-ips"), KnownVersion::new(55, "add-lookup-sled-by-policy-and-state-index"), KnownVersion::new(54, "blueprint-add-external-ip-id"), KnownVersion::new(53, "drop-service-table"), diff --git a/nexus/db-queries/src/db/datastore/allowed_source_ips.rs b/nexus/db-queries/src/db/datastore/allowed_source_ips.rs new file mode 100644 index 00000000000..5752272bbe1 --- /dev/null +++ b/nexus/db-queries/src/db/datastore/allowed_source_ips.rs @@ -0,0 +1,250 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Datastore methods for operating on source IP allowlists. + +use crate::authz; +use crate::context::OpContext; +use crate::db::error::public_error_from_diesel; +use crate::db::error::ErrorHandler; +use crate::db::fixed_data::allowed_source_ips::ALLOWED_SOURCE_IPS_ID; +use crate::db::DbConnection; +use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::ExpressionMethods; +use diesel::QueryDsl; +use diesel::SelectableHelper; +use nexus_db_model::schema::allowed_source_ip; +use nexus_db_model::AllowedSourceIp; +use omicron_common::api::external::AllowedSourceIps; +use omicron_common::api::external::Error; +use omicron_common::api::external::LookupType; +use omicron_common::api::external::ResourceType; + +impl super::DataStore { + /// Fetch the list of allowed source IPs. + /// + /// This is currently effectively a singleton, since it is populated by RSS + /// and we only provide APIs in Nexus to update it, not create / delete. + pub async fn allowed_source_ips_view( + &self, + opctx: &OpContext, + ) -> Result { + let conn = self.pool_connection_authorized(opctx).await?; + opctx.authorize(authz::Action::Read, &authz::FLEET).await?; + allowed_source_ip::dsl::allowed_source_ip + .find(ALLOWED_SOURCE_IPS_ID) + .first_async::(&*conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::AllowedSourceIps, + LookupType::ById(ALLOWED_SOURCE_IPS_ID), + ), + ) + }) + } + + /// Upsert and return a list of allowed source IPs. + pub async fn allowed_source_ips_upsert( + &self, + opctx: &OpContext, + allowed_ips: AllowedSourceIps, + ) -> Result { + let conn = self.pool_connection_authorized(opctx).await?; + Self::allowed_source_ips_upsert_on_connection( + opctx, + &conn, + allowed_ips, + ) + .await + } + + pub(crate) async fn allowed_source_ips_upsert_on_connection( + opctx: &OpContext, + conn: &async_bb8_diesel::Connection, + allowed_ips: AllowedSourceIps, + ) -> Result { + use allowed_source_ip::dsl; + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + let record = AllowedSourceIp::new(ALLOWED_SOURCE_IPS_ID, allowed_ips); + diesel::insert_into(dsl::allowed_source_ip) + .values(record.clone()) + .returning(AllowedSourceIp::as_returning()) + .on_conflict(dsl::id) + .do_update() + .set(( + dsl::allowed_ips.eq(record.allowed_ips), + dsl::time_modified.eq(record.time_modified), + dsl::generation.eq(dsl::generation + 1), + )) + .get_result_async(conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } +} + +#[cfg(test)] +mod tests { + use crate::db::{ + datastore::test_utils::datastore_test, + fixed_data::allowed_source_ips::ALLOWED_SOURCE_IPS_ID, + }; + use nexus_db_model::AllowedSourceIp; + use nexus_test_utils::db::test_setup_database; + use omicron_common::api::external::{ + self, Error, IpNet, LookupType, ResourceType, + }; + use omicron_test_utils::dev; + + #[tokio::test] + async fn test_allowed_source_ip_database_ops() { + let logctx = dev::test_setup_log("test_allowed_source_ip_database_ops"); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = datastore_test(&logctx, &db).await; + + // There should be nothing to begin with. + let result = datastore + .allowed_source_ips_view(&opctx) + .await + .expect_err("Expected query to fail when there are no records"); + assert_eq!( + result, + Error::ObjectNotFound { + type_name: ResourceType::AllowedSourceIps, + lookup_type: LookupType::ById(ALLOWED_SOURCE_IPS_ID) + }, + "Expected an ObjectNotFound error when there is no IP allowlist" + ); + + // Helper to get IP addresses from a record. + fn get_ips(record: &AllowedSourceIp) -> Vec { + record + .allowed_ips + .as_ref() + .expect("Record should have non-NULL allowlist") + .iter() + .copied() + .map(IpNet::from) + .collect() + } + + // Upsert an allowlist, with some specific IPs. + let ips = + vec!["10.0.0.0/8".parse().unwrap(), "::1/64".parse().unwrap()]; + let allowed_ips = external::AllowedSourceIps::List(ips.clone()); + let record = datastore + .allowed_source_ips_upsert(&opctx, allowed_ips) + .await + .expect("Expected this insert to succeed"); + assert_eq!( + record.id, ALLOWED_SOURCE_IPS_ID, + "Record should have hard-coded allowlist ID" + ); + assert_eq!( + u64::from(record.generation.0), + 1, + "Generation should start at 1" + ); + assert_eq!( + ips, + get_ips(&record), + "Inserted and re-read incorrect allowed source ips" + ); + + // Upsert a new list, verify it's changed. + let new_ips = + vec!["10.0.0.0/4".parse().unwrap(), "fd00::10/32".parse().unwrap()]; + let allowed_ips = external::AllowedSourceIps::List(new_ips.clone()); + let new_record = datastore + .allowed_source_ips_upsert(&opctx, allowed_ips) + .await + .expect("Expected this insert to succeed"); + assert_eq!( + new_record.id, ALLOWED_SOURCE_IPS_ID, + "Record should have hard-coded allowlist ID" + ); + assert_eq!( + u64::from(new_record.generation.0), + 2, + "Generation should now be 2" + ); + assert_eq!( + record.time_created, new_record.time_created, + "Time created should not have changed" + ); + assert!( + record.time_modified < new_record.time_modified, + "Time modified should have changed" + ); + assert_eq!( + new_ips, + get_ips(&new_record), + "Updated allowed IPs are incorrect" + ); + + // Insert an allowlist letting anything in, and check it. + let record = new_record; + let allowed_ips = external::AllowedSourceIps::Any; + let new_record = datastore + .allowed_source_ips_upsert(&opctx, allowed_ips) + .await + .expect("Expected this insert to succeed"); + assert_eq!( + new_record.id, ALLOWED_SOURCE_IPS_ID, + "Record should have hard-coded allowlist ID" + ); + assert_eq!( + u64::from(new_record.generation.0), + 3, + "Generation should now be 3" + ); + assert_eq!( + record.time_created, new_record.time_created, + "Time created should not have changed" + ); + assert!( + record.time_modified < new_record.time_modified, + "Time modified should have changed" + ); + assert!( + new_record.allowed_ips.is_none(), + "NULL should be used in the DB to represent any allowed source IPs", + ); + + // Lastly change it back to a real list again. + let record = new_record; + let allowed_ips = external::AllowedSourceIps::List(new_ips.clone()); + let new_record = datastore + .allowed_source_ips_upsert(&opctx, allowed_ips) + .await + .expect("Expected this insert to succeed"); + assert_eq!( + new_record.id, ALLOWED_SOURCE_IPS_ID, + "Record should have hard-coded allowlist ID" + ); + assert_eq!( + u64::from(new_record.generation.0), + 4, + "Generation should now be 4" + ); + assert_eq!( + record.time_created, new_record.time_created, + "Time created should not have changed" + ); + assert!( + record.time_modified < new_record.time_modified, + "Time modified should have changed" + ); + assert_eq!( + new_ips, + get_ips(&new_record), + "Updated allowed IPs are incorrect" + ); + + db.cleanup().await.expect("failed to cleanup database"); + logctx.cleanup_successful(); + } +} diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index b91b4cc9604..15c810dcf1f 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -48,6 +48,7 @@ use std::sync::Arc; use uuid::Uuid; mod address_lot; +mod allowed_source_ips; mod bfd; mod bgp; mod bootstore; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 43c55e104e1..3faa489d6f5 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -55,6 +55,7 @@ use nexus_types::external_api::shared::IdentityType; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::SiloRole; use nexus_types::identity::Resource; +use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -86,6 +87,7 @@ pub struct RackInit { pub recovery_user_id: external_params::UserId, pub recovery_user_password_hash: omicron_passwords::PasswordHashString, pub dns_update: DnsVersionUpdateBuilder, + pub allowed_source_ips: AllowedSourceIps, } /// Possible errors while trying to initialize rack @@ -106,6 +108,8 @@ enum RackInitError { Retryable(DieselError), // Other non-retryable database error Database(DieselError), + // Error adding initial allowed source IP list + AllowedSourceIpError(Error), } // Catch-all for Diesel error conversion into RackInitError, which @@ -169,6 +173,7 @@ impl From for Error { "failed operation due to database error: {:#}", err )), + RackInitError::AllowedSourceIpError(err) => err, } } } @@ -859,6 +864,17 @@ impl DataStore { } })?; + // Insert the initial source IP allowlist for requests to + // user-facing services. + Self::allowed_source_ips_upsert_on_connection( + opctx, + &conn, + rack_init.allowed_source_ips, + ).await.map_err(|e| { + err.set(RackInitError::AllowedSourceIpError(e)).unwrap(); + DieselError::RollbackTransaction + })?; + let rack = diesel::update(rack_dsl::rack) .filter(rack_dsl::id.eq(rack_id)) .set(( @@ -1054,6 +1070,7 @@ mod test { "test suite".to_string(), "test suite".to_string(), ), + allowed_source_ips: AllowedSourceIps::Any, } } } diff --git a/nexus/db-queries/src/db/fixed_data/allowed_source_ips.rs b/nexus/db-queries/src/db/fixed_data/allowed_source_ips.rs new file mode 100644 index 00000000000..ce8aeb0e271 --- /dev/null +++ b/nexus/db-queries/src/db/fixed_data/allowed_source_ips.rs @@ -0,0 +1,11 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! Fixed data for source IP allowlist implementation. + +/// UUID of singleton source IP allowlist. +pub static ALLOWED_SOURCE_IPS_ID: uuid::Uuid = + uuid::uuid!("001de000-a110-4000-8000-000000000000"); diff --git a/nexus/db-queries/src/db/fixed_data/mod.rs b/nexus/db-queries/src/db/fixed_data/mod.rs index 4f896eb5d10..258df6bbd3f 100644 --- a/nexus/db-queries/src/db/fixed_data/mod.rs +++ b/nexus/db-queries/src/db/fixed_data/mod.rs @@ -30,9 +30,11 @@ // 001de000-4401 built-in services project // 001de000-074c built-in services vpc // 001de000-c470 built-in services vpc subnets +// 001de000-all0 singleton ID for source IP allowlist ("all0" is like "allow") use once_cell::sync::Lazy; +pub mod allowed_source_ips; pub mod project; pub mod role_assignment; pub mod role_builtin; @@ -65,6 +67,7 @@ fn assert_valid_uuid(id: &uuid::Uuid) { #[cfg(test)] mod test { + use super::allowed_source_ips::ALLOWED_SOURCE_IPS_ID; use super::assert_valid_uuid; use super::FLEET_ID; @@ -72,4 +75,9 @@ mod test { fn test_builtin_fleet_id_is_valid() { assert_valid_uuid(&FLEET_ID); } + + #[test] + fn test_allowlist_id_is_valid() { + assert_valid_uuid(&ALLOWED_SOURCE_IPS_ID); + } } diff --git a/nexus/networking/src/firewall_rules.rs b/nexus/networking/src/firewall_rules.rs index 85c457d5226..45bb9630a17 100644 --- a/nexus/networking/src/firewall_rules.rs +++ b/nexus/networking/src/firewall_rules.rs @@ -9,6 +9,8 @@ use ipnetwork::IpNetwork; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; +use nexus_db_queries::db::datastore::SERVICES_DB_NAME; +use nexus_db_queries::db::fixed_data::vpc::SERVICES_VPC_ID; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::identity::Resource; use nexus_db_queries::db::lookup; @@ -16,12 +18,18 @@ use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::model::Name; use nexus_db_queries::db::DataStore; use omicron_common::api::external; +use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::Error; use omicron_common::api::external::IpNet; +use omicron_common::api::external::L4PortRange; use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::VpcFirewallRuleAction; +use omicron_common::api::external::VpcFirewallRuleDirection; +use omicron_common::api::external::VpcFirewallRuleStatus; use omicron_common::api::internal::nexus::HostIdentifier; use omicron_common::api::internal::shared::NetworkInterface; use slog::debug; +use slog::error; use slog::info; use slog::warn; use slog::Logger; @@ -345,6 +353,45 @@ pub async fn resolve_firewall_rules_for_sled_agent( priority: rule.priority.0 .0, }); } + + // Add rules that implement our IP allowlist for user-facing services. + // + // These rules are implicit, and not stored in the firewall rule table, + // since they're logically a different thing. We'll convert them into + // sled-agent firewall rules here and insert them. + // + // Note that this only applies to the internal services VPC, so we ignore + // these rules if we have no such interfaces for that. + if vpc.id() == *SERVICES_VPC_ID { + if let AllowedSourceIps::List(allowed_ips) = + lookup_allowed_source_ips(datastore, opctx, log).await? + { + let name = SERVICES_DB_NAME.parse().unwrap(); + if let Some(interfaces) = vpc_interfaces.get(&name) { + debug!( + log, + "constructing firewall rules implementing \ + user-facing services IP allowlist"; + "allowed_ips" => ?allowed_ips, + ); + if let Some(mut new_rules) = + allowlist_to_firewall_rules(interfaces, allowed_ips).await + { + sled_agent_rules.append(&mut new_rules); + } + } else { + warn!( + log, + "Found services VPC ID but no NICs for this service!" + ); + } + } else { + debug!( + log, + "User-facing services allowlist is set to allow any inbound traffic" + ); + } + } debug!( log, "resolved firewall rules for sled agents"; @@ -439,3 +486,100 @@ pub async fn plumb_service_firewall_rules( .await?; Ok(()) } + +/// Compute firewall rules implied by an IP allowlist. +/// +/// This looks up the source IP allowlist in the database, and generates +/// firewall rules attached to each provided NIC that implements the allowlist. +/// If the allowlist is "any", there are no restrictions and no rules are +/// returned. +/// +/// Otherwise, two sets of rules are returned: +/// +/// - A low-priority rule that denies all traffic +/// - A higher-priority rule that allows it from the specific IPs. +async fn allowlist_to_firewall_rules( + nics: &[NetworkInterface], + allowed_ips: Vec, +) -> Option> { + if nics.is_empty() { + return None; + } + + // Convert to host identifiers, how the sled-agent types expose these. + assert!(!allowed_ips.is_empty(), "Allowlist must not be empty here"); + let filter_hosts = Some( + allowed_ips + .into_iter() + .map(|net| sled_agent_client::types::HostIdentifier::Ip(net.into())) + .collect(), + ); + + let low_priority_deny = sled_agent_client::types::VpcFirewallRule { + status: VpcFirewallRuleStatus::Enabled.into(), + direction: VpcFirewallRuleDirection::Inbound.into(), + targets: nics.to_vec(), + filter_hosts: filter_hosts.clone(), + filter_ports: Some(vec![L4PortRange { + first: 1.try_into().unwrap(), + last: u16::MAX.try_into().unwrap(), + } + .into()]), + filter_protocols: None, + action: VpcFirewallRuleAction::Deny.into(), + priority: ALLOW_LIST_LOW_PRIORTY, + }; + + let allow = sled_agent_client::types::VpcFirewallRule { + status: VpcFirewallRuleStatus::Enabled.into(), + direction: VpcFirewallRuleDirection::Inbound.into(), + targets: nics.to_vec(), + filter_hosts, + + // TODO-correctness: Having no filter on ports or protocols is not + // techniically correct. Nexus should only receive TCP traffic, on HTTP + // or HTTPs traffic, for example. However, at this point in the system, + // we don't really have access to _which service_ this NIC is for, only + // that it is for _one_ of our services. + // + // We may want to reorganize this code to apply these rules on a + // per-service basis. This is discussed in RFD 474. + filter_ports: None, + filter_protocols: None, + action: VpcFirewallRuleAction::Allow.into(), + priority: ALLOWLIST_HIGH_PRIORITY, + }; + + Some(vec![low_priority_deny, allow]) +} + +/// Return the list of allowed IPs from the database. +async fn lookup_allowed_source_ips( + datastore: &DataStore, + opctx: &OpContext, + log: &Logger, +) -> Result { + match datastore.allowed_source_ips_view(opctx).await { + Ok(allowed) => { + debug!(log, "fetched allowlist from DB"; "allowed" => ?allowed); + let ips = if let Some(list) = allowed.allowed_ips { + AllowedSourceIps::List( + list.into_iter().map(Into::into).collect(), + ) + } else { + AllowedSourceIps::Any + }; + Ok(ips) + } + Err(e) => { + error!(log, "failed to fetch allowlist from DB"; "err" => ?e); + Err(e) + } + } +} + +/// FW rule priority for allow-list default-deny rule. +const ALLOW_LIST_LOW_PRIORTY: u16 = u16::MAX - 1; + +/// FW rule priority for allow-list explicit allow rule. +const ALLOWLIST_HIGH_PRIORITY: u16 = ALLOW_LIST_LOW_PRIORTY - 1; diff --git a/nexus/src/app/allowed_source_ips.rs b/nexus/src/app/allowed_source_ips.rs new file mode 100644 index 00000000000..dac48241ca3 --- /dev/null +++ b/nexus/src/app/allowed_source_ips.rs @@ -0,0 +1,147 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2024 Oxide Computer Company + +//! Nexus methods for operating on source IP allowlists. + +use std::net::IpAddr; + +use nexus_db_queries::context::OpContext; +use nexus_types::external_api::params; +use nexus_types::external_api::views::AllowedSourceIps; +use omicron_common::api::external; +use omicron_common::api::external::Error; + +impl super::Nexus { + /// Fetch the allowlist of source IPs that can reach user-facing services. + pub async fn allowed_source_ips_view( + &self, + opctx: &OpContext, + ) -> Result { + self.db_datastore + .allowed_source_ips_view(opctx) + .await + .map(AllowedSourceIps::from) + } + + /// Upsert the allowlist of source IPs that can reach user-facing services. + pub async fn allowed_source_ips_upsert( + &self, + opctx: &OpContext, + remote_addr: IpAddr, + params: params::AllowedSourceIpsUpdate, + ) -> Result<(), Error> { + if let external::AllowedSourceIps::List(list) = ¶ms.allowed_ips { + // Size limits on the allowlist. + const MAX_ALLOWLIST_LENGTH: usize = 1000; + if list.len() > MAX_ALLOWLIST_LENGTH { + let message = format!( + "Source IP allowlist is limited to {} entries, found {}", + MAX_ALLOWLIST_LENGTH, + list.len(), + ); + return Err(Error::invalid_request(message)); + } + if list.is_empty() { + return Err(Error::invalid_request( + "Source IP allow list must not be empty", + )); + } + + // Some basic sanity-checks on the addresses in the allowlist. + // + // The most important part here is checking that the source address + // the request came from is on the allowlist. This is our only real + // guardrail to prevent accidentally preventing any future access to + // the rack! + for entry in list.iter() { + if !entry.contains(remote_addr) { + return Err(Error::invalid_request( + "The source IP allow list would prevent access \ + from the current client! Ensure that the allowlist \ + contains an entry that continues to allow access \ + from this peer.", + )); + } + if entry.ip().is_unspecified() { + return Err(Error::invalid_request( + "Source IP allowlist may not contain the \ + unspecified address. Use \"any\" to allow \ + any source to connect to user-facing services.", + )); + } + if entry.prefix() == 0 { + return Err(Error::invalid_request( + "Source IP allowlist entries may not have \ + a netmask of /0.", + )); + } + } + }; + + // Actually insert the new allowlist. + self.db_datastore + .allowed_source_ips_upsert(opctx, params.allowed_ips) + .await + .map(|_| ())?; + + // Notify the sled-agents of the updated firewall rules. + match nexus_networking::plumb_service_firewall_rules( + self.datastore(), + &opctx, + &[], + &opctx, + &opctx.log, + ) + .await + { + Ok(_) => { + info!(self.log, "plumbed updated IP allowlist to sled-agents"); + Ok(()) + } + Err(e) => { + error!( + self.log, + "failed to update sled-agents with new allowlist"; + "error" => ?e + ); + Err(e) + } + } + } + + /// Wait until we've applied the user-facing services allowlist. + /// + /// This will block until we've plumbed this allowlist and passed it to the + /// sled-agents responsible. This should only be called from + /// rack-initialization handling. + pub(crate) async fn await_ip_allowlist_plumbing(&self) { + let opctx = self.opctx_for_internal_api(); + loop { + match nexus_networking::plumb_service_firewall_rules( + self.datastore(), + &opctx, + &[], + &opctx, + &opctx.log, + ) + .await + { + Ok(_) => { + info!(self.log, "plumbed initial IP allowlist"); + return; + } + Err(e) => { + error!( + self.log, + "failed to plumb initial IP allowlist"; + "error" => ?e + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + } + } + } +} diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index a2cbe2f7ae9..406d8c0dde3 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -39,6 +39,7 @@ use uuid::Uuid; // The implementation of Nexus is large, and split into a number of submodules // by resource. mod address_lot; +mod allowed_source_ips; pub(crate) mod background; mod bfd; mod bgp; diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index be00c99fb88..e66352413aa 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -635,6 +635,7 @@ impl super::Nexus { .user_password_hash .into(), dns_update, + allowed_source_ips: request.allowed_source_ips, }, ) .await?; diff --git a/nexus/src/app/sled.rs b/nexus/src/app/sled.rs index 8aa0573d468..c7fc651823e 100644 --- a/nexus/src/app/sled.rs +++ b/nexus/src/app/sled.rs @@ -273,7 +273,13 @@ impl super::Nexus { address: SocketAddrV6, kind: DatasetKind, ) -> Result<(), Error> { - info!(self.log, "upserting dataset"; "zpool_id" => zpool_id.to_string(), "dataset_id" => id.to_string(), "address" => address.to_string()); + info!( + self.log, + "upserting dataset"; + "zpool_id" => zpool_id.to_string(), + "dataset_id" => id.to_string(), + "address" => address.to_string() + ); let dataset = db::model::Dataset::new(id, zpool_id, address, kind); self.db_datastore.dataset_upsert(dataset).await?; Ok(()) diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 293ae31da71..8c91d6005aa 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -288,6 +288,9 @@ pub(crate) fn external_api() -> NexusApiDescription { api.register(networking_bfd_disable)?; api.register(networking_bfd_status)?; + api.register(networking_allowed_source_ips_view)?; + api.register(networking_allowed_source_ips_update)?; + api.register(utilization_view)?; // Fleet-wide API operations @@ -3742,6 +3745,53 @@ async fn networking_bfd_status( apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await } +/// Get user-facing services IP allowlist +#[endpoint { + method = GET, + path = "/v1/system/networking/allowed-source-ips", + tags = ["system/networking"], +}] +async fn networking_allowed_source_ips_view( + rqctx: RequestContext>, +) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + nexus + .allowed_source_ips_view(&opctx) + .await + .map(HttpResponseOk) + .map_err(HttpError::from) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + +/// Update user-facing services IP allowlist +#[endpoint { + method = PUT, + path = "/v1/system/networking/allowed-source-ips", + tags = ["system/networking"], +}] +async fn networking_allowed_source_ips_update( + rqctx: RequestContext>, + params: TypedBody, +) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.nexus; + let params = params.into_inner(); + let opctx = crate::context::op_context_for_external_api(&rqctx).await?; + let remote_addr = rqctx.request.remote_addr().ip(); + nexus + .allowed_source_ips_upsert(&opctx, remote_addr, params) + .await + .map(|_| HttpResponseUpdatedNoContent()) + .map_err(HttpError::from) + }; + apictx.external_latencies.instrument_dropshot_handler(&rqctx, handler).await +} + // Images /// List images diff --git a/nexus/src/lib.rs b/nexus/src/lib.rs index 81a40743925..306fd90508f 100644 --- a/nexus/src/lib.rs +++ b/nexus/src/lib.rs @@ -137,6 +137,11 @@ impl Server { let opctx = apictx.nexus.opctx_for_service_balancer(); apictx.nexus.await_rack_initialization(&opctx).await; + // While we've started our internal server, we need to wait until we've + // definitely implemented our source IP allowlist for making requests to + // the external server we're about to start. + apictx.nexus.await_ip_allowlist_plumbing().await; + // Launch the external server. let tls_config = apictx .nexus @@ -317,6 +322,7 @@ impl nexus_test_interface::NexusServer for Server { bgp: Vec::new(), bfd: Vec::new(), }, + allowed_source_ips: Default::default(), }, ) .await diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index 02ab1385e31..9dd65cbc2a9 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -25,6 +25,7 @@ use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::Ipv4Range; use nexus_types::external_api::views::SledProvisionPolicy; use omicron_common::api::external::AddressLotKind; +use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::ByteCount; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::IdentityMetadataUpdateParams; @@ -866,6 +867,14 @@ pub static DEMO_USER_CREATE: Lazy = password: params::UserPassword::LoginDisallowed, }); +// Allowlist for user-facing services. +pub static ALLOWED_SOURCE_IP_URL: Lazy = + Lazy::new(|| String::from("/v1/system/networking/allowed-source-ips")); +pub static ALLOWED_SOURCE_IP_UPDATE: Lazy = + Lazy::new(|| params::AllowedSourceIpsUpdate { + allowed_ips: AllowedSourceIps::Any, + }); + /// Describes an API endpoint to be verified by the "unauthorized" test /// /// These structs are also used to check whether we're covering all endpoints in @@ -2378,5 +2387,18 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { ), ], }, + + // User-facing services IP allowlist + VerifyEndpoint { + url: &ALLOWED_SOURCE_IP_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Get, + AllowedMethod::Put( + serde_json::to_value(&*ALLOWED_SOURCE_IP_UPDATE).unwrap(), + ), + ], + }, ] }); diff --git a/nexus/tests/output/nexus_tags.txt b/nexus/tests/output/nexus_tags.txt index 3e40e8293d0..61ae6dc7e2c 100644 --- a/nexus/tests/output/nexus_tags.txt +++ b/nexus/tests/output/nexus_tags.txt @@ -173,6 +173,8 @@ networking_address_lot_block_list GET /v1/system/networking/address- networking_address_lot_create POST /v1/system/networking/address-lot networking_address_lot_delete DELETE /v1/system/networking/address-lot/{address_lot} networking_address_lot_list GET /v1/system/networking/address-lot +networking_allowed_source_ips_update PUT /v1/system/networking/allowed-source-ips +networking_allowed_source_ips_view GET /v1/system/networking/allowed-source-ips networking_bfd_disable POST /v1/system/networking/bfd-disable networking_bfd_enable POST /v1/system/networking/bfd-enable networking_bfd_status GET /v1/system/networking/bfd-status diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 101d1eaf334..6ab10208734 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -9,10 +9,10 @@ use crate::external_api::shared; use base64::Engine; use chrono::{DateTime, Utc}; use omicron_common::api::external::{ - AddressLotKind, BfdMode, ByteCount, Hostname, IdentityMetadataCreateParams, - IdentityMetadataUpdateParams, InstanceCpuCount, IpNet, Ipv4Net, Ipv6Net, - Name, NameOrId, PaginationOrder, RouteDestination, RouteTarget, - SemverVersion, + AddressLotKind, AllowedSourceIps, BfdMode, ByteCount, Hostname, + IdentityMetadataCreateParams, IdentityMetadataUpdateParams, + InstanceCpuCount, IpNet, Ipv4Net, Ipv6Net, Name, NameOrId, PaginationOrder, + RouteDestination, RouteTarget, SemverVersion, }; use schemars::JsonSchema; use serde::{ @@ -2062,3 +2062,12 @@ pub struct TimeseriesQuery { /// A timeseries query string, written in the Oximeter query language. pub query: String, } + +// Allowed source IPs + +/// Parameters for updating allowed source IPs +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct AllowedSourceIpsUpdate { + /// The new list of allowed source IPs. + pub allowed_ips: AllowedSourceIps, +} diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index 80ef24fcb22..c1e0669b554 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -12,8 +12,9 @@ use api_identity::ObjectIdentity; use chrono::DateTime; use chrono::Utc; use omicron_common::api::external::{ - ByteCount, Digest, Error, IdentityMetadata, InstanceState, Ipv4Net, - Ipv6Net, Name, ObjectIdentity, RoleName, SimpleIdentity, + AllowedSourceIps as ExternalAllowedSourceIps, ByteCount, Digest, Error, + IdentityMetadata, InstanceState, Ipv4Net, Ipv6Net, Name, ObjectIdentity, + RoleName, SimpleIdentity, }; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; @@ -948,3 +949,16 @@ pub struct Ping { /// returns anything at all. pub status: PingStatus, } + +// ALLOWED SOURCE IPS + +/// Allowlist of IPs or subnets that can make requests to user-facing services. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct AllowedSourceIps { + /// Time the list was created. + pub time_created: DateTime, + /// Time the list was last modified. + pub time_modified: DateTime, + /// The allowlist of IPs or subnets. + pub allowed_ips: ExternalAllowedSourceIps, +} diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index 15ab1549529..143ca1be8b8 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -13,6 +13,7 @@ use omicron_common::api::external::ByteCount; use omicron_common::api::external::Generation; use omicron_common::api::external::MacAddr; use omicron_common::api::external::Name; +use omicron_common::api::internal::shared::AllowedSourceIps; use omicron_common::api::internal::shared::ExternalPortDiscovery; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::api::internal::shared::SourceNatConfig; @@ -243,6 +244,8 @@ pub struct RackInitializationRequest { pub external_port_count: ExternalPortDiscovery, /// Initial rack network configuration pub rack_network_config: RackNetworkConfig, + /// IPs or subnets allowed to make requests to user-facing services + pub allowed_source_ips: AllowedSourceIps, } pub type DnsConfigParams = dns_service_client::types::DnsConfigParams; diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 19e4330e7b6..71f32992abd 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1386,6 +1386,48 @@ "dependency" ] }, + "AllowedSourceIps": { + "description": "Description of source IPs allowed to reach rack services.", + "oneOf": [ + { + "description": "Allow traffic from any external IP address.", + "type": "object", + "properties": { + "allow": { + "type": "string", + "enum": [ + "any" + ] + } + }, + "required": [ + "allow" + ] + }, + { + "description": "Restrict access to a specific set of source IP addresses or subnets.\n\nAll others are prevented from reaching rack services.", + "type": "object", + "properties": { + "allow": { + "type": "string", + "enum": [ + "list" + ] + }, + "ips": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + } + }, + "required": [ + "allow", + "ips" + ] + } + ] + }, "BackgroundTask": { "description": "Background tasks\n\nThese are currently only intended for observability by developers. We will eventually want to flesh this out into something more observable for end users.", "type": "object", @@ -1544,7 +1586,7 @@ "format": "ipv4" }, "asn": { - "description": "The autonomous sysetm number of the router the peer belongs to.", + "description": "The autonomous system number of the router the peer belongs to.", "type": "integer", "format": "uint32", "minimum": 0 @@ -3806,6 +3848,14 @@ "RackInitializationRequest": { "type": "object", "properties": { + "allowed_source_ips": { + "description": "IPs or subnets allowed to make requests to user-facing services", + "allOf": [ + { + "$ref": "#/components/schemas/AllowedSourceIps" + } + ] + }, "blueprint": { "description": "Blueprint describing services initialized by RSS.", "allOf": [ @@ -3887,6 +3937,7 @@ } }, "required": [ + "allowed_source_ips", "blueprint", "certs", "datasets", diff --git a/openapi/nexus.json b/openapi/nexus.json index ca49abd76a1..2e9309eae25 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -6248,6 +6248,61 @@ } } }, + "/v1/system/networking/allowed-source-ips": { + "get": { + "tags": [ + "system/networking" + ], + "summary": "Get user-facing services IP allowlist", + "operationId": "networking_allowed_source_ips_view", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AllowedSourceIps" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "system/networking" + ], + "summary": "Update user-facing services IP allowlist", + "operationId": "networking_allowed_source_ips_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AllowedSourceIpsUpdate" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/v1/system/networking/bfd-disable": { "post": { "tags": [ @@ -9130,6 +9185,94 @@ "switch_histories" ] }, + "AllowedSourceIps": { + "description": "Allowlist of IPs or subnets that can make requests to user-facing services.", + "type": "object", + "properties": { + "allowed_ips": { + "description": "The allowlist of IPs or subnets.", + "allOf": [ + { + "$ref": "#/components/schemas/AllowedSourceIps2" + } + ] + }, + "time_created": { + "description": "Time the list was created.", + "type": "string", + "format": "date-time" + }, + "time_modified": { + "description": "Time the list was last modified.", + "type": "string", + "format": "date-time" + } + }, + "required": [ + "allowed_ips", + "time_created", + "time_modified" + ] + }, + "AllowedSourceIps2": { + "description": "Description of source IPs allowed to reach rack services.", + "oneOf": [ + { + "description": "Allow traffic from any external IP address.", + "type": "object", + "properties": { + "allow": { + "type": "string", + "enum": [ + "any" + ] + } + }, + "required": [ + "allow" + ] + }, + { + "description": "Restrict access to a specific set of source IP addresses or subnets.\n\nAll others are prevented from reaching rack services.", + "type": "object", + "properties": { + "allow": { + "type": "string", + "enum": [ + "list" + ] + }, + "ips": { + "type": "array", + "items": { + "$ref": "#/components/schemas/IpNet" + } + } + }, + "required": [ + "allow", + "ips" + ] + } + ] + }, + "AllowedSourceIpsUpdate": { + "description": "Parameters for updating allowed source IPs", + "type": "object", + "properties": { + "allowed_ips": { + "description": "The new list of allowed source IPs.", + "allOf": [ + { + "$ref": "#/components/schemas/AllowedSourceIps2" + } + ] + } + }, + "required": [ + "allowed_ips" + ] + }, "Baseboard": { "description": "Properties that uniquely identify an Oxide hardware component", "type": "object", diff --git a/schema/crdb/add-allowed-source-ips/up.sql b/schema/crdb/add-allowed-source-ips/up.sql new file mode 100644 index 00000000000..5df0e4801bd --- /dev/null +++ b/schema/crdb/add-allowed-source-ips/up.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS omicron.public.allowed_source_ip ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + generation INT8 NOT NULL, + allowed_ips INET[] +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index b42b63deccc..e09d4ce84b1 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3760,6 +3760,20 @@ CREATE TABLE IF NOT EXISTS omicron.public.db_metadata ( CHECK (singleton = true) ); +-- An allowlist of IP addresses that can make requests to user-facing services. +CREATE TABLE IF NOT EXISTS omicron.public.allowed_source_ip ( + id UUID PRIMARY KEY, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + generation INT8 NOT NULL, + -- A nullable list of allowed source IPs. + -- + -- NULL is used to indicate _any_ source IP is allowed. A _non-empty_ list + -- represents an explicit allow list of IPs or IP subnets. Note that the + -- list itself may never be empty. + allowed_ips INET[] +); + /* * Keep this at the end of file so that the database does not contain a version * until it is fully populated. @@ -3771,7 +3785,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '55.0.0', NULL) + (TRUE, NOW(), NOW(), '56.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/bootstrap/params.rs b/sled-agent/src/bootstrap/params.rs index c901fb31ac5..84571f74f33 100644 --- a/sled-agent/src/bootstrap/params.rs +++ b/sled-agent/src/bootstrap/params.rs @@ -7,6 +7,7 @@ use anyhow::{bail, Result}; use async_trait::async_trait; use omicron_common::address::{self, Ipv6Subnet, SLED_PREFIX}; +use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::internal::shared::RackNetworkConfig; use omicron_common::ledger::Ledgerable; use schemars::JsonSchema; @@ -41,6 +42,7 @@ struct UnvalidatedRackInitializeRequest { external_certificates: Vec, recovery_silo: RecoverySiloConfig, rack_network_config: RackNetworkConfig, + allowed_source_ips: AllowedSourceIps, } /// Configuration for the "rack setup service". @@ -87,6 +89,9 @@ pub struct RackInitializeRequest { /// Initial rack network configuration pub rack_network_config: RackNetworkConfig, + + /// IPs or subnets allowed to make requests to user-facing services + pub allowed_source_ips: AllowedSourceIps, } // This custom debug implementation hides the private keys. @@ -105,6 +110,7 @@ impl std::fmt::Debug for RackInitializeRequest { external_certificates: _, recovery_silo, rack_network_config, + allowed_source_ips, } = &self; f.debug_struct("RackInitializeRequest") @@ -121,6 +127,7 @@ impl std::fmt::Debug for RackInitializeRequest { .field("external_certificates", &"") .field("recovery_silo", recovery_silo) .field("rack_network_config", rack_network_config) + .field("allowed_source_ips", allowed_source_ips) .finish() } } @@ -161,6 +168,7 @@ impl TryFrom for RackInitializeRequest { external_certificates: value.external_certificates, recovery_silo: value.recovery_silo, rack_network_config: value.rack_network_config, + allowed_source_ips: value.allowed_source_ips, }) } } @@ -495,6 +503,7 @@ mod tests { bgp: Vec::new(), bfd: Vec::new(), }, + allowed_source_ips: Default::default(), }; // Valid configs: all external DNS IPs are contained in the IP pool diff --git a/sled-agent/src/rack_setup/config.rs b/sled-agent/src/rack_setup/config.rs index fba753657e9..b540fdf1dde 100644 --- a/sled-agent/src/rack_setup/config.rs +++ b/sled-agent/src/rack_setup/config.rs @@ -129,6 +129,7 @@ mod test { bgp: Vec::new(), bfd: Vec::new(), }, + allowed_source_ips: Default::default(), }; assert_eq!( diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 6e3ce4a6acd..c22f0c5a2d2 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -1267,6 +1267,7 @@ mod tests { bgp: Vec::new(), bfd: Vec::new(), }, + allowed_source_ips: Default::default(), }; let mut svp = ServicePortBuilder::new(&config); diff --git a/wicket/src/cli/rack_setup/config_template.toml b/wicket/src/cli/rack_setup/config_template.toml index d091237b5f5..a37359011c0 100644 --- a/wicket/src/cli/rack_setup/config_template.toml +++ b/wicket/src/cli/rack_setup/config_template.toml @@ -38,6 +38,16 @@ internal_services_ip_pool_ranges = [] # Confirm this list contains all expected sleds before continuing! bootstrap_sleds = [] +# Allowlist of source IPs that can make requests to user-facing services. +[allowed_source_ips] +# Any external IPs to make requests. This is the default. +allow = "any" + +# Use the below two lines to only allow requests from the specified IPs or +# subnets. Requests from any other source IP are refused. +# allow = "list" +# ips = [ "1.2.3.4", "5.6.7.8/10" ] + # TODO: docs on network config [rack_network_config] infra_ip_first = ""