WIP: [Reconfigurator] Add new Clickhouse discretionary zones

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "certificates",
     "clickhouse-admin",
     "clickhouse-admin/api",
+    "clickhouse-admin/types",
     "clients/bootstrap-agent-client",
     "clients/cockroach-admin-client",
     "clients/ddm-admin-client",
@@ -315,7 +316,7 @@ ciborium = "0.2.2"
 clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] }
 clickhouse-admin-api = { path = "clickhouse-admin/api" }
 clickhouse-admin-types = { path = "clickhouse-admin/types" }
-clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" }
+clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "4ee0f74db55f440d589232256458c0750f6a641e" }
 cockroach-admin-api = { path = "cockroach-admin/api" }
 cockroach-admin-client = { path = "clients/cockroach-admin-client" }
 cockroach-admin-types = { path = "cockroach-admin/types" }

nexus/db-model/Cargo.toml

@@ -14,6 +14,7 @@ omicron-rpaths.workspace = true
 anyhow.workspace = true
 camino.workspace = true
 chrono.workspace = true
+clickhouse-admin-types.workspace = true
 derive-where.workspace = true
 diesel = { workspace = true, features = ["postgres", "r2d2", "chrono", "serde_json", "network-address", "uuid"] }
 hex.workspace = true

nexus/db-model/src/deployment.rs

@@ -8,9 +8,9 @@
 use crate::inventory::ZoneType;
 use crate::omicron_zone_config::{self, OmicronZoneNic};
 use crate::schema::{
-    blueprint, bp_omicron_physical_disk, bp_omicron_zone, bp_omicron_zone_nic,
-    bp_sled_omicron_physical_disks, bp_sled_omicron_zones, bp_sled_state,
-    bp_target,
+    blueprint, bp_clickhouse_cluster_config, bp_omicron_physical_disk,
+    bp_omicron_zone, bp_omicron_zone_nic, bp_sled_omicron_physical_disks,
+    bp_sled_omicron_zones, bp_sled_state, bp_target,
 };
 use crate::typed_uuid::DbTypedUuid;
 use crate::{
@@ -21,7 +21,6 @@ use anyhow::{anyhow, bail, Context, Result};
 use chrono::{DateTime, Utc};
 use ipnetwork::IpNetwork;
 use nexus_sled_agent_shared::inventory::OmicronZoneDataset;
-use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintZoneConfig;
 use nexus_types::deployment::BlueprintZoneDisposition;
 use nexus_types::deployment::BlueprintZonesConfig;
@@ -30,6 +29,7 @@ use nexus_types::deployment::{
     blueprint_zone_type, BlueprintPhysicalDisksConfig,
 };
 use nexus_types::deployment::{BlueprintPhysicalDiskConfig, BlueprintZoneType};
+use nexus_types::deployment::{BlueprintTarget, ClickhouseClusterConfig};
 use nexus_types::deployment::{
     OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp,
     OmicronZoneExternalSnatIp,
@@ -259,6 +259,9 @@ pub struct BpOmicronZone {
 
     pub external_ip_id: Option<DbTypedUuid<ExternalIpKind>>,
     pub filesystem_pool: Option<DbTypedUuid<ZpoolKind>>,
+
+    clickhouse_keeper_id: Option<i64>,
+    clickhouse_server_id: Option<i64>,
 }
 
 impl BpOmicronZone {
@@ -308,6 +311,8 @@ impl BpOmicronZone {
             snat_ip: None,
             snat_first_port: None,
             snat_last_port: None,
+            clickhouse_keeper_id: None,
+            clickhouse_server_id: None,
         };
 
         match &blueprint_zone.zone_type {
@@ -350,18 +355,40 @@ impl BpOmicronZone {
                 bp_omicron_zone.set_zpool_name(dataset);
             }
             BlueprintZoneType::ClickhouseKeeper(
-                blueprint_zone_type::ClickhouseKeeper { address, dataset },
+                blueprint_zone_type::ClickhouseKeeper {
+                    keeper_id,
+                    address,
+                    dataset,
+                },
             ) => {
                 // Set the common fields
                 bp_omicron_zone.set_primary_service_ip_and_port(address);
                 bp_omicron_zone.set_zpool_name(dataset);
+
+                // Set the zone specific fields
+                bp_omicron_zone.clickhouse_keeper_id = Some(
+                    keeper_id
+                        .0
+                        .try_into()
+                        .expect("no more than 2^63 keeper IDs please"),
+                );
             }
             BlueprintZoneType::ClickhouseServer(
-                blueprint_zone_type::ClickhouseServer { address, dataset },
+                blueprint_zone_type::ClickhouseServer {
+                    server_id,
+                    address,
+                    dataset,
+                },
             ) => {
                 // Set the common fields
                 bp_omicron_zone.set_primary_service_ip_and_port(address);
                 bp_omicron_zone.set_zpool_name(dataset);
+
+                // Set the zone specific fields
+                bp_omicron_zone.clickhouse_server_id =
+                    Some(server_id.0.try_into().expect(
+                        "no more than 2^63 clickhouse server IDs please",
+                    ));
             }
             BlueprintZoneType::CockroachDb(
                 blueprint_zone_type::CockroachDb { address, dataset },
@@ -588,12 +615,22 @@ impl BpOmicronZone {
             }
             ZoneType::ClickhouseKeeper => BlueprintZoneType::ClickhouseKeeper(
                 blueprint_zone_type::ClickhouseKeeper {
+                    keeper_id: clickhouse_admin_types::KeeperId(
+                        self.clickhouse_keeper_id.ok_or_else(|| {
+                            anyhow!("missing clickhouse_keeper_id")
+                        })? as u64,
+                    ),
                     address: primary_address,
                     dataset: dataset?,
                 },
             ),
             ZoneType::ClickhouseServer => BlueprintZoneType::ClickhouseServer(
                 blueprint_zone_type::ClickhouseServer {
+                    server_id: clickhouse_admin_types::ServerId(
+                        self.clickhouse_server_id.ok_or_else(|| {
+                            anyhow!("missing clickhouse_keeper_id")
+                        })? as u64,
+                    ),
                     address: primary_address,
                     dataset: dataset?,
                 },
@@ -803,6 +840,64 @@ impl From<BpOmicronZoneNic> for OmicronZoneNic {
     }
 }
 
+#[derive(Queryable, Clone, Debug, Selectable, Insertable)]
+#[diesel(table_name = bp_clickhouse_cluster_config)]
+pub struct BpClickhouseClusterConfig {
+    blueprint_id: Uuid,
+    generation: Generation,
+    max_used_server_id: i64,
+    max_used_keeper_id: i64,
+    cluster_name: String,
+    cluster_secret: String,
+}
+
+impl BpClickhouseClusterConfig {
+    pub fn new(
+        blueprint_id: Uuid,
+        config: &ClickhouseClusterConfig,
+    ) -> anyhow::Result<BpClickhouseClusterConfig> {
+        Ok(BpClickhouseClusterConfig {
+            blueprint_id,
+            generation: Generation(config.generation),
+            max_used_server_id: config
+                .max_used_server_id
+                .0
+                .try_into()
+                .context("more than 2^63 IDs in use")?,
+            max_used_keeper_id: config
+                .max_used_keeper_id
+                .0
+                .try_into()
+                .context("more than 2^63 IDs in use")?,
+            cluster_name: config.cluster_name.clone(),
+            cluster_secret: config.cluster_secret.clone(),
+        })
+    }
+}
+
+impl TryFrom<BpClickhouseClusterConfig> for ClickhouseClusterConfig {
+    type Error = anyhow::Error;
+    fn try_from(value: BpClickhouseClusterConfig) -> Result<Self, Self::Error> {
+        Ok(ClickhouseClusterConfig {
+            generation: value.generation.0,
+            max_used_server_id: clickhouse_admin_types::ServerId(
+                value
+                    .max_used_server_id
+                    .try_into()
+                    .context("negative ID in database?")?,
+            ),
+            max_used_keeper_id: clickhouse_admin_types::KeeperId(
+                value
+                    .max_used_keeper_id
+                    .try_into()
+                    .context("negative ID in database?")?,
+            ),
+            cluster_name: value.cluster_name.clone(),
+            cluster_secret: value.cluster_secret.clone(),
+        })
+    }
+}
+
 mod diesel_util {
     use crate::{
         schema::bp_omicron_zone::disposition, to_db_bp_zone_disposition,

nexus/db-model/src/schema.rs

@@ -1512,6 +1512,7 @@ table! {
         cockroachdb_fingerprint -> Text,
 
         cockroachdb_setting_preserve_downgrade -> Nullable<Text>,
+
     }
 }
 
@@ -1595,6 +1596,10 @@ table! {
         disposition -> crate::DbBpZoneDispositionEnum,
         external_ip_id -> Nullable<Uuid>,
         filesystem_pool -> Nullable<Uuid>,
+
+        clickhouse_keeper_id -> Nullable<Int8>,
+        clickhouse_server_id -> Nullable<Int8>
+
     }
 }
 
@@ -1612,6 +1617,18 @@ table! {
     }
 }
 
+table! {
+    bp_clickhouse_cluster_config (blueprint_id) {
+        blueprint_id -> Uuid,
+
+        generation-> Int8,
+        max_used_server_id -> Int8,
+        max_used_keeper_id -> Int8,
+        cluster_name -> Text,
+        cluster_secret -> Text,
+    }
+}
+
 table! {
     cockroachdb_zone_id_to_node_id (omicron_zone_id, crdb_node_id) {
         omicron_zone_id -> Uuid,

nexus/db-queries/src/db/datastore/deployment.rs

@@ -49,6 +49,7 @@ use nexus_types::deployment::BlueprintPhysicalDisksConfig;
 use nexus_types::deployment::BlueprintTarget;
 use nexus_types::deployment::BlueprintZoneFilter;
 use nexus_types::deployment::BlueprintZonesConfig;
+use nexus_types::deployment::ClickhouseClusterConfig;
 use nexus_types::deployment::CockroachDbPreserveDowngrade;
 use nexus_types::external_api::views::SledState;
 use omicron_common::api::external::DataPageParams;
@@ -632,6 +633,7 @@ impl DataStore {
             external_dns_version,
             cockroachdb_fingerprint,
             cockroachdb_setting_preserve_downgrade,
+            clickhouse_cluster_config: None,
             time_created,
             creator,
             comment,

nexus/db-queries/src/db/datastore/inventory.rs

@@ -2087,6 +2087,8 @@ impl DataStore {
             rot_pages_found,
             sled_agents,
             omicron_zones,
+            // TODO: fill this in once we collect it
+            clickhouse_keeper_cluster_membership: BTreeMap::new(),
         })
     }
 }

nexus/inventory/src/builder.rs

@@ -147,6 +147,8 @@ impl CollectionBuilder {
             rot_pages_found: self.rot_pages_found,
             sled_agents: self.sleds,
             omicron_zones: self.omicron_zones,
+            // TODO: Fill when we actually gather keeper inventory
+            clickhouse_keeper_cluster_membership: BTreeMap::new(),
         }
     }
 

nexus/reconfigurator/planning/Cargo.toml

@@ -9,6 +9,7 @@ workspace = true
 [dependencies]
 anyhow.workspace = true
 chrono.workspace = true
+clickhouse-admin-types.workspace = true
 debug-ignore.workspace = true
 gateway-client.workspace = true
 indexmap.workspace = true