From 91f6c67434441227f6cc26a193f21518a5fc167d Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 11:05:53 -0500 Subject: [PATCH 1/7] Update non-discoverable silos to be hidden from utilization by default --- nexus/db-model/src/schema.rs | 1 + nexus/db-model/src/utilization.rs | 1 + nexus/db-queries/src/db/datastore/utilization.rs | 7 +++++++ schema/crdb/dbinit.sql | 1 + 4 files changed, 10 insertions(+) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 9259532c52..11a4ef8196 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -431,6 +431,7 @@ table! { silo_utilization(silo_id) { silo_id -> Uuid, silo_name -> Text, + silo_discoverable -> Bool, cpus_provisioned -> Int8, memory_provisioned -> Int8, storage_provisioned -> Int8, diff --git a/nexus/db-model/src/utilization.rs b/nexus/db-model/src/utilization.rs index 9bef4f59c7..b0e6324bc9 100644 --- a/nexus/db-model/src/utilization.rs +++ b/nexus/db-model/src/utilization.rs @@ -9,6 +9,7 @@ use uuid::Uuid; pub struct SiloUtilization { pub silo_id: Uuid, pub silo_name: Name, + pub silo_discoverable: bool, pub cpus_allocated: i64, pub memory_allocated: ByteCount, diff --git a/nexus/db-queries/src/db/datastore/utilization.rs b/nexus/db-queries/src/db/datastore/utilization.rs index 4fbe215fe2..972bbd1662 100644 --- a/nexus/db-queries/src/db/datastore/utilization.rs +++ b/nexus/db-queries/src/db/datastore/utilization.rs @@ -8,6 +8,7 @@ use crate::db::model::Name; use crate::db::model::SiloUtilization; use crate::db::pagination::paginated; use async_bb8_diesel::AsyncRunQueryDsl; +use diesel::BoolExpressionMethods; use diesel::{ExpressionMethods, QueryDsl, SelectableHelper}; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::Error; @@ -50,6 +51,12 @@ impl DataStore { ), } .select(SiloUtilization::as_select()) + .filter( + dsl::silo_discoverable.eq(true).or(dsl::cpus_allocated + .gt(0) + .or(dsl::memory_allocated.gt(0)) + .or(dsl::storage_allocated.gt(0))), + ) .load_async(&*self.pool_connection_authorized(opctx).await?) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 4cb347b260..059feb9f2d 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -857,6 +857,7 @@ CREATE VIEW IF NOT EXISTS omicron.public.silo_utilization AS SELECT c.id AS silo_id, s.name AS silo_name, + s.discoverable as silo_discoverable, c.cpus_provisioned AS cpus_provisioned, c.ram_provisioned AS memory_provisioned, c.virtual_disk_bytes_provisioned AS storage_provisioned, From 633d36ea3845137bb87d1ae8006957167e589a80 Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 11:17:24 -0500 Subject: [PATCH 2/7] Add migration --- schema/crdb/31.0.0/up.sql | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 schema/crdb/31.0.0/up.sql diff --git a/schema/crdb/31.0.0/up.sql b/schema/crdb/31.0.0/up.sql new file mode 100644 index 0000000000..12d01d6418 --- /dev/null +++ b/schema/crdb/31.0.0/up.sql @@ -0,0 +1,21 @@ +ALTER VIEW omicron.public.silo_utilization +AS SELECT + c.id AS silo_id, + s.name AS silo_name, + s.discoverable as silo_discoverable, + c.cpus_provisioned AS cpus_provisioned, + c.ram_provisioned AS memory_provisioned, + c.virtual_disk_bytes_provisioned AS storage_provisioned, + q.cpus AS cpus_allocated, + q.memory_bytes AS memory_allocated, + q.storage_bytes AS storage_allocated +FROM + omicron.public.virtual_provisioning_collection AS c + RIGHT JOIN omicron.public.silo_quotas AS q + ON c.id = q.silo_id + INNER JOIN omicron.public.silo AS s + ON c.id = s.id +WHERE + c.collection_type = 'Silo' +AND + s.time_deleted IS NULL; \ No newline at end of file From 8a372499e7ddbae7c87b924d23ddcf15c53ba07e Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 11:21:18 -0500 Subject: [PATCH 3/7] Bump schema version --- nexus/db-model/src/schema.rs | 2 +- schema/crdb/dbinit.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 11a4ef8196..146d468899 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(30, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(31, 0, 0); table! { disk (id) { diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 059feb9f2d..c9d483678c 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3419,7 +3419,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '30.0.0', NULL) + ( TRUE, NOW(), NOW(), '31.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From eaf76f97c03bab88614c37db6ce21d694b887fb0 Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 12:42:44 -0500 Subject: [PATCH 4/7] simplify query; add test --- .../src/db/datastore/utilization.rs | 7 +-- nexus/tests/integration_tests/utilization.rs | 43 ++++++++++++++++++- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/utilization.rs b/nexus/db-queries/src/db/datastore/utilization.rs index 972bbd1662..826c66043a 100644 --- a/nexus/db-queries/src/db/datastore/utilization.rs +++ b/nexus/db-queries/src/db/datastore/utilization.rs @@ -52,10 +52,11 @@ impl DataStore { } .select(SiloUtilization::as_select()) .filter( - dsl::silo_discoverable.eq(true).or(dsl::cpus_allocated - .gt(0) + dsl::silo_discoverable + .eq(true) + .or(dsl::cpus_allocated.gt(0)) .or(dsl::memory_allocated.gt(0)) - .or(dsl::storage_allocated.gt(0))), + .or(dsl::storage_allocated.gt(0)), ) .load_async(&*self.pool_connection_authorized(opctx).await?) .await diff --git a/nexus/tests/integration_tests/utilization.rs b/nexus/tests/integration_tests/utilization.rs index e09e71a9e3..7d733c66b8 100644 --- a/nexus/tests/integration_tests/utilization.rs +++ b/nexus/tests/integration_tests/utilization.rs @@ -29,6 +29,16 @@ async fn test_utilization(cptestctx: &ControlPlaneTestContext) { create_default_ip_pool(&client).await; + // set high quota for test silo + let _ = NexusRequest::object_put( + client, + "/v1/system/silos/test-suite-silo/quotas", + Some(¶ms::SiloQuotasCreate::arbitrarily_high_default()), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await; + let current_util = objects_list_page_authz::( client, "/v1/system/utilization/silos", @@ -36,6 +46,8 @@ async fn test_utilization(cptestctx: &ControlPlaneTestContext) { .await .items; + // `default-silo` should be the only silo that shows up because + // it has a default quota set assert_eq!(current_util.len(), 2); assert_eq!(current_util[0].silo_name, "default-silo"); @@ -47,7 +59,36 @@ async fn test_utilization(cptestctx: &ControlPlaneTestContext) { assert_eq!(current_util[1].silo_name, "test-suite-silo"); assert_eq!(current_util[1].provisioned, SiloQuotasCreate::empty().into()); - assert_eq!(current_util[1].allocated, SiloQuotasCreate::empty().into()); + assert_eq!( + current_util[1].allocated, + SiloQuotasCreate::arbitrarily_high_default().into() + ); + + let _ = NexusRequest::object_put( + client, + "/v1/system/silos/test-suite-silo/quotas", + Some(¶ms::SiloQuotasCreate::empty()), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await; + + let current_util = objects_list_page_authz::( + client, + "/v1/system/utilization/silos", + ) + .await + .items; + + // Now that default-silo is the only one with a quota, it should be the only result + assert_eq!(current_util.len(), 1); + + assert_eq!(current_util[0].silo_name, "default-silo"); + assert_eq!(current_util[0].provisioned, SiloQuotasCreate::empty().into()); + assert_eq!( + current_util[0].allocated, + SiloQuotasCreate::arbitrarily_high_default().into() + ); let _ = create_project(&client, &PROJECT_NAME).await; let _ = create_instance(client, &PROJECT_NAME, &INSTANCE_NAME).await; From f0dda108351b7758ffc49048b77d4d52eef2703a Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 13:36:39 -0500 Subject: [PATCH 5/7] Use CREATE OR REPLACE VIEW, bump schema version --- nexus/db-model/src/schema.rs | 2 +- schema/crdb/{31.0.0 => 32.0.0}/up.sql | 3 ++- schema/crdb/dbinit.sql | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename schema/crdb/{31.0.0 => 32.0.0}/up.sql (87%) diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index 146d468899..46a726a889 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(31, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(32, 0, 0); table! { disk (id) { diff --git a/schema/crdb/31.0.0/up.sql b/schema/crdb/32.0.0/up.sql similarity index 87% rename from schema/crdb/31.0.0/up.sql rename to schema/crdb/32.0.0/up.sql index 12d01d6418..cd5cc74099 100644 --- a/schema/crdb/31.0.0/up.sql +++ b/schema/crdb/32.0.0/up.sql @@ -1,7 +1,8 @@ -ALTER VIEW omicron.public.silo_utilization +CREATE OR REPLACE VIEW omicron.public.silo_utilization AS SELECT c.id AS silo_id, s.name AS silo_name, + -- This is the added column s.discoverable as silo_discoverable, c.cpus_provisioned AS cpus_provisioned, c.ram_provisioned AS memory_provisioned, diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index c9d483678c..191b662630 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3419,7 +3419,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '31.0.0', NULL) + ( TRUE, NOW(), NOW(), '32.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; From 1baaba77fb54021382df9eee25f4f2b9fdc3adbf Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 16:11:01 -0500 Subject: [PATCH 6/7] Fix migration by adding new column at end of select --- schema/crdb/32.0.0/up.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/crdb/32.0.0/up.sql b/schema/crdb/32.0.0/up.sql index cd5cc74099..d6cd1e2cde 100644 --- a/schema/crdb/32.0.0/up.sql +++ b/schema/crdb/32.0.0/up.sql @@ -2,14 +2,14 @@ CREATE OR REPLACE VIEW omicron.public.silo_utilization AS SELECT c.id AS silo_id, s.name AS silo_name, - -- This is the added column - s.discoverable as silo_discoverable, c.cpus_provisioned AS cpus_provisioned, c.ram_provisioned AS memory_provisioned, c.virtual_disk_bytes_provisioned AS storage_provisioned, q.cpus AS cpus_allocated, q.memory_bytes AS memory_allocated, - q.storage_bytes AS storage_allocated + q.storage_bytes AS storage_allocated, + -- This is the added column + s.discoverable as silo_discoverable FROM omicron.public.virtual_provisioning_collection AS c RIGHT JOIN omicron.public.silo_quotas AS q From c9c3a43d6303c0946171ec46af5f4be2ca062a43 Mon Sep 17 00:00:00 2001 From: Justin Bennett Date: Wed, 31 Jan 2024 19:13:03 -0500 Subject: [PATCH 7/7] Rearrange view order to appease tests --- schema/crdb/dbinit.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a425caff14..103eb2e0c7 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -857,13 +857,13 @@ CREATE VIEW IF NOT EXISTS omicron.public.silo_utilization AS SELECT c.id AS silo_id, s.name AS silo_name, - s.discoverable as silo_discoverable, c.cpus_provisioned AS cpus_provisioned, c.ram_provisioned AS memory_provisioned, c.virtual_disk_bytes_provisioned AS storage_provisioned, q.cpus AS cpus_allocated, q.memory_bytes AS memory_allocated, - q.storage_bytes AS storage_allocated + q.storage_bytes AS storage_allocated, + s.discoverable as silo_discoverable FROM omicron.public.virtual_provisioning_collection AS c RIGHT JOIN omicron.public.silo_quotas AS q