From 3f712ad42f4f98f9f542c9532667fefe83636a62 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Mon, 20 Nov 2023 21:57:48 +0000 Subject: [PATCH] wip --- nexus/db-queries/src/db/datastore/rack.rs | 27 +++++++++++++++----- nexus/src/app/rack.rs | 28 ++++++++++++-------- schema/crdb/11.0.0/up2.sql | 1 - schema/crdb/{11.0.0 => 12.0.0}/up1.sql | 19 +++++--------- schema/crdb/12.0.0/up2.sql | 5 ++++ schema/crdb/dbinit.sql | 31 +++++++++-------------- 6 files changed, 61 insertions(+), 50 deletions(-) delete mode 100644 schema/crdb/11.0.0/up2.sql rename schema/crdb/{11.0.0 => 12.0.0}/up1.sql (82%) create mode 100644 schema/crdb/12.0.0/up2.sql diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index ab927a2815..60055d2587 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -215,16 +215,16 @@ impl DataStore { Ok(()) } - // Return the rack subnet, reconfiguration epoch, and all current - // underlay allocations for the rack. - // - // Order allocations by `subnet_octet` + /// Return the rack subnet and all current underlay allocations for the + /// rack. + /// + /// Order allocations by `subnet_octet` pub async fn rack_subnet_allocations( &self, opctx: &OpContext, rack_id: Uuid, ) -> Result< - Vec<(Option, i64, Option)>, + Vec<(Option, Option)>, Error, > { opctx.authorize(authz::Action::Read, &authz::FLEET).await?; @@ -239,7 +239,6 @@ impl DataStore { ) .select(( rack_dsl::rack_subnet, - rack_dsl::reconfiguration_epoch, Option::::as_select(), )) .order_by(subnet_dsl::subnet_octet.asc()) @@ -248,6 +247,22 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + /// Store a new sled subnet allocation in the database + pub async fn sled_subnet_allocation_insert( + &self, + opctx: &OpContext, + allocation: SledUnderlaySubnetAllocation, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, &authz::FLEET).await?; + use db::schema::sled_underlay_subnet_allocation::dsl; + diesel::insert_into(dsl::sled_underlay_subnet_allocation) + .values(allocation) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + Ok(()) + } + // The following methods which return a `TxnError` take a `conn` parameter // which comes from the transaction created in `rack_set_initialized`. diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 46946173f2..d007e69175 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -776,19 +776,11 @@ impl super::Nexus { opctx: &OpContext, sled: UninitializedSled, ) -> Result<(), Error> { - // First, get the hw_baseboard_id for the given UninitializedSled - let limit = NonZeroU32::new(50).unwrap(); - let collection = self - .db_datastore - .inventory_get_latest_collection(opctx, limit) - .await?; - let baseboard_id = sled.baseboard.into(); let hw_baseboard_id = self.db_datastore.find_hw_baseboard_id(opctx, baseboard_id).await?; // Fetch all the existing allocations via self.rack_id - // This only works for a single rack right now let allocations = self .db_datastore .rack_subnet_allocations(opctx, sled.rack_id) @@ -798,6 +790,8 @@ impl super::Nexus { // octet. The returned allocations are ordered by octet, so we will know // when we have a free one. However, if we already have an allocation // for the given sled then reuse that one. + // TODO: This could all actually be done in SQL using a `next_item` query. + // See XXX const MIN_SUBNET_OCTET: i16 = 33; let mut new_allocation = SledUnderlaySubnetAllocation { rack_id: sled.rack_id, @@ -805,12 +799,16 @@ impl super::Nexus { subnet_octet: MIN_SUBNET_OCTET, hw_baseboard_id, }; - for (subnet, epoch, allocation) in allocations { + let mut subnet; + let mut allocation_already_exists = false; + for (subnet_col, allocation) in allocations { + subnet = subnet_col; if let Some(allocation) = allocation { if allocation.hw_baseboard_id == new_allocation.hw_baseboard_id { // We already have an allocation for this sled. new_allocation = allocation; + allocation_already_exists = true; break; } if allocation.subnet_octet == new_allocation.subnet_octet { @@ -822,8 +820,16 @@ impl super::Nexus { } } } - - // Then write the allocation and bump the reconfiguration epoch + // Write the new allocation row to CRDB. The UNIQUE constraint + // on `subnet_octet` will prevent dueling administrators reusing + // allocations when sleds are being added. We will need another + // mechanism ala generation numbers when we must interleave additions + // and removals of sleds. + if !allocation_already_exists { + self.db_datastore + .sled_subnet_allocation_insert(opctx, new_allocation) + .await?; + } // Then make the call to sled-agent todo!() diff --git a/schema/crdb/11.0.0/up2.sql b/schema/crdb/11.0.0/up2.sql deleted file mode 100644 index 204b5c24df..0000000000 --- a/schema/crdb/11.0.0/up2.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE omicron.public.rack ADD COLUMN IF NOT EXISTS reconfiguration_epoch INT8 NOT NULL DEFAULT 0; diff --git a/schema/crdb/11.0.0/up1.sql b/schema/crdb/12.0.0/up1.sql similarity index 82% rename from schema/crdb/11.0.0/up1.sql rename to schema/crdb/12.0.0/up1.sql index 45c22ddd52..21a9f4b5be 100644 --- a/schema/crdb/11.0.0/up1.sql +++ b/schema/crdb/12.0.0/up1.sql @@ -12,16 +12,17 @@ -- all of the octets start at 33. This makes the data in this table purely additive -- post-RSS, which also implies that we cannot re-use subnet octets if an original -- sled that was part of RSS was removed from the cluster. --- --- All modifications to this table should be guarded by the --- `reconfiguration_epoch` row in the `rack` table. -CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocations { +CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation { + -- The physical identity of the sled + -- (foreign key into `hw_baseboard_id` table) + hw_baseboard_id UUID PRIMARY KEY, + -- The rack to which a sled is being added -- (foreign key into `rack` table) -- -- We require this because the sled is not yet part of the sled table when -- we first allocate a subnet for it. - rack_id UUID NOT NULL + rack_id UUID NOT NULL, -- The sled to which a subnet is being allocated -- @@ -33,12 +34,4 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocations { -- -- Always between 33 and 255 inclusive subnet_octet INT2 NOT NULL UNIQUE, - - -- The physical identity of the sled - -- (foreign key into `hw_baseboard_id` table) - hw_baseboard_id UUID NOT NULL UNIQUE, - - -- Each update will grab all allocations for a rack to figure out what - -- subnet octets are free - PRIMARY KEY (rack_id, sled_id) }; diff --git a/schema/crdb/12.0.0/up2.sql b/schema/crdb/12.0.0/up2.sql new file mode 100644 index 0000000000..e307ed9817 --- /dev/null +++ b/schema/crdb/12.0.0/up2.sql @@ -0,0 +1,5 @@ +-- Add an index which allows pagination by {rack_id, sled_id} pairs. +CREATE UNIQUE INDEX IF NOT EXISTS lookup_subnet_allocation_by_rack_and_sled ON omicron.public.sled_underlay_subnet_allocation { + rack_id, + sled_id +}; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 3cb2e105f8..3f87526db1 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -66,13 +66,7 @@ CREATE TABLE IF NOT EXISTS omicron.public.rack ( tuf_base_url STRING(512), /* The IPv6 underlay /56 prefix for the rack */ - rack_subnet INET, - - /* A generation number to allow conditional updates and optimisitic - * concurrency control when performing rack cluster reconfigurations, such - * as adding or removing sleds. - */ - reconfiguration_epoch INT8 NOT NULL DEFAULT 0 + rack_subnet INET ); /* @@ -179,16 +173,17 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_resource_by_sled ON omicron.public.sled -- all of the octets start at 33. This makes the data in this table purely additive -- post-RSS, which also implies that we cannot re-use subnet octets if an original -- sled that was part of RSS was removed from the cluster. --- --- All modifications to this table should be guarded by the --- `reconfiguration_epoch` row in the `rack` table. CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation { + -- The physical identity of the sled + -- (foreign key into `hw_baseboard_id` table) + hw_baseboard_id UUID PRIMARY KEY, + -- The rack to which a sled is being added -- (foreign key into `rack` table) -- -- We require this because the sled is not yet part of the sled table when -- we first allocate a subnet for it. - rack_id UUID NOT NULL + rack_id UUID NOT NULL, -- The sled to which a subnet is being allocated -- @@ -200,14 +195,12 @@ CREATE TABLE IF NOT EXISTS omicron.public.sled_underlay_subnet_allocation { -- -- Always between 33 and 255 inclusive subnet_octet INT2 NOT NULL UNIQUE, +}; - -- The physical identity of the sled - -- (foreign key into `hw_baseboard_id` table) - hw_baseboard_id UUID NOT NULL UNIQUE, - - -- Each update will grab all allocations for a rack to figure out what - -- subnet octets are free - PRIMARY KEY (rack_id, sled_id) +-- Add an index which allows pagination by {rack_id, sled_id} pairs. +CREATE UNIQUE INDEX IF NOT EXISTS lookup_subnet_allocation_by_rack_and_sled ON omicron.public.sled_underlay_subnet_allocation { + rack_id, + sled_id }; /* @@ -2890,7 +2883,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '11.0.0', NULL) + ( TRUE, NOW(), NOW(), '12.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT;