Accept notifications from Crucible

clients/nexus-client/Cargo.toml

@@ -20,3 +20,4 @@ serde_json.workspace = true
 slog.workspace = true
 uuid.workspace = true
 omicron-workspace-hack.workspace = true
+omicron-uuid-kinds.workspace = true

clients/nexus-client/src/lib.rs

@@ -33,6 +33,10 @@ progenitor::generate_api!(
         MacAddr = omicron_common::api::external::MacAddr,
         Name = omicron_common::api::external::Name,
         NewPasswordHash = omicron_passwords::NewPasswordHash,
+
+        TypedUuidForUpstairsKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsKind>,
+        TypedUuidForUpstairsRepairKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsRepairKind>,
+        TypedUuidForUpstairsSessionKind = omicron_uuid_kinds::TypedUuid<omicron_uuid_kinds::UpstairsSessionKind>,
     }
 );
 

common/src/api/internal/nexus.rs

@@ -9,6 +9,10 @@ use crate::api::external::{
     InstanceState, IpNet, SemverVersion, Vni,
 };
 use chrono::{DateTime, Utc};
+use omicron_uuid_kinds::DownstairsRegionKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsRepairKind;
+use omicron_uuid_kinds::UpstairsSessionKind;
 use parse_display::{Display, FromStr};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
@@ -251,3 +255,42 @@ pub enum HostIdentifier {
     Ip(IpNet),
     Vpc(Vni),
 }
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+pub enum UpstairsRepairType {
+    Live,
+    Reconciliation,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct DownstairsUnderRepair {
+    pub region_uuid: TypedUuid<DownstairsRegionKind>,
+    pub target_addr: std::net::SocketAddrV6,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairStartInfo {
+    pub time: DateTime<Utc>,
+    pub session_id: TypedUuid<UpstairsSessionKind>,
+    pub repair_id: TypedUuid<UpstairsRepairKind>,
+    pub repair_type: UpstairsRepairType,
+    pub repairs: Vec<DownstairsUnderRepair>,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairFinishInfo {
+    pub time: DateTime<Utc>,
+    pub session_id: TypedUuid<UpstairsSessionKind>,
+    pub repair_id: TypedUuid<UpstairsRepairKind>,
+    pub repair_type: UpstairsRepairType,
+    pub repairs: Vec<DownstairsUnderRepair>,
+    pub aborted: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)]
+pub struct RepairProgress {
+    pub time: DateTime<Utc>,
+    pub current_item: i64,
+    pub total_items: i64,
+}

nexus/Cargo.toml

@@ -76,7 +76,6 @@ tokio-postgres = { workspace = true, features = ["with-serde_json-1"] }
 tough.workspace = true
 trust-dns-resolver.workspace = true
 uuid.workspace = true
-
 nexus-blueprint-execution.workspace = true
 nexus-defaults.workspace = true
 nexus-db-model.workspace = true
@@ -93,6 +92,7 @@ rustls = { workspace = true }
 rustls-pemfile = { workspace = true }
 update-common.workspace = true
 omicron-workspace-hack.workspace = true
+omicron-uuid-kinds.workspace = true
 
 [dev-dependencies]
 async-bb8-diesel.workspace = true

nexus/db-model/src/lib.rs

@@ -84,6 +84,7 @@ mod switch;
 mod tuf_repo;
 mod typed_uuid;
 mod unsigned;
+mod upstairs_repair;
 mod user_builtin;
 mod utilization;
 mod virtual_provisioning_collection;
@@ -174,6 +175,7 @@ pub use switch_interface::*;
 pub use switch_port::*;
 pub use tuf_repo::*;
 pub use typed_uuid::to_db_typed_uuid;
+pub use upstairs_repair::*;
 pub use user_builtin::*;
 pub use utilization::*;
 pub use virtual_provisioning_collection::*;

nexus/db-model/src/schema.rs

@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(37, 0, 1);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(38, 0, 0);
 
 table! {
     disk (id) {
@@ -1518,6 +1518,32 @@ table! {
     }
 }
 
+table! {
+    upstairs_repair_notification (repair_id, upstairs_id, session_id, region_id, notification_type) {
+        time -> Timestamptz,
+
+        repair_id -> Uuid,
+        repair_type -> crate::UpstairsRepairTypeEnum,
+        upstairs_id -> Uuid,
+        session_id -> Uuid,
+
+        region_id -> Uuid,
+        target_ip -> Inet,
+        target_port -> Int4,
+
+        notification_type -> crate::UpstairsRepairNotificationTypeEnum,
+    }
+}
+
+table! {
+    upstairs_repair_progress (repair_id, time, current_item, total_items) {
+        repair_id -> Uuid,
+        time -> Timestamptz,
+        current_item -> Int8,
+        total_items -> Int8,
+    }
+}
+
 table! {
     db_metadata (singleton) {
         singleton -> Bool,

nexus/db-model/src/upstairs_repair.rs

@@ -0,0 +1,154 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use super::impl_enum_type;
+use crate::ipv6;
+use crate::schema::upstairs_repair_notification;
+use crate::schema::upstairs_repair_progress;
+use crate::typed_uuid::DbTypedUuid;
+use crate::SqlU16;
+use chrono::{DateTime, Utc};
+use omicron_common::api::internal;
+use omicron_uuid_kinds::DownstairsRegionKind;
+use omicron_uuid_kinds::TypedUuid;
+use omicron_uuid_kinds::UpstairsKind;
+use omicron_uuid_kinds::UpstairsRepairKind;
+use omicron_uuid_kinds::UpstairsSessionKind;
+use serde::{Deserialize, Serialize};
+use std::net::SocketAddrV6; // internal::nexus::UpstairsRepairType;
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "upstairs_repair_notification_type", schema = "public"))]
+    pub struct UpstairsRepairNotificationTypeEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = UpstairsRepairNotificationTypeEnum)]
+    pub enum UpstairsRepairNotificationType;
+
+    // Notification types
+    Started => b"started"
+    Succeeded => b"succeeded"
+    Failed => b"failed"
+);
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "upstairs_repair_type", schema = "public"))]
+    pub struct UpstairsRepairTypeEnum;
+
+    #[derive(Copy, Clone, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq, Eq, Hash)]
+    #[diesel(sql_type = UpstairsRepairTypeEnum)]
+    pub enum UpstairsRepairType;
+
+    // Types of repair a Crucible Upstairs can do
+    Live => b"live"
+    Reconciliation => b"reconciliation"
+);
+
+impl From<internal::nexus::UpstairsRepairType> for UpstairsRepairType {
+    fn from(v: internal::nexus::UpstairsRepairType) -> UpstairsRepairType {
+        match v {
+            internal::nexus::UpstairsRepairType::Live => {
+                UpstairsRepairType::Live
+            }
+            internal::nexus::UpstairsRepairType::Reconciliation => {
+                UpstairsRepairType::Reconciliation
+            }
+        }
+    }
+}
+
+/// A record of Crucible Upstairs repair notifications: when a repair started,
+/// succeeded, failed, etc.
+///
+/// Each repair attempt is uniquely identified by the repair ID, upstairs ID,
+/// session ID, and region ID. How those change tells Nexus about what is going
+/// on:
+///
+/// - if all IDs are the same for different requests, Nexus knows that the
+///   client is retrying the notification.
+///
+/// - if the upstairs ID, session ID, and region ID are all the same, but the
+///   repair ID is different, then the same Upstairs is trying to repair that
+///   region again. This could be due to a failed first attempt, or that
+///   downstairs may have been kicked out again.
+///
+/// - if the upstairs ID and region ID are the same, but the session ID and
+///   repair ID are different, then a different session of the same Upstairs is
+///   trying to repair that Downstairs. Session IDs change each time the
+///   Upstairs is created, so it could have crashed, or it could have been
+///   migrated and the destination Propolis' Upstairs is attempting to repair
+///   the same region.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = upstairs_repair_notification)]
+pub struct UpstairsRepairNotification {
+    // Importantly, this is client time, not Nexus' time that it received the
+    // notification.
+    pub time: DateTime<Utc>,
+
+    pub repair_id: DbTypedUuid<UpstairsRepairKind>,
+
+    // There's a difference between the live repairs and reconciliation: the
+    // Upstairs can go through reconciliation without there being any error from
+    // a downstairs, or any region replacement request from Nexus. One example
+    // is if the rack power is pulled: if everything is powered back up again
+    // reconciliation could be required but this isn't the fault of any problem
+    // with a physical disk, or any error that was returned.
+    //
+    // Alternatively any record of a live repair means that there was a problem:
+    // Currently, either an Upstairs kicked out a Downstairs (or two) due to
+    // some error or because it lagged behind the others, or Nexus has
+    // instructed an Upstairs to perform a region replacement.
+    pub repair_type: UpstairsRepairType,
+
+    pub upstairs_id: DbTypedUuid<UpstairsKind>,
+    pub session_id: DbTypedUuid<UpstairsSessionKind>,
+
+    pub region_id: DbTypedUuid<DownstairsRegionKind>,
+    pub target_ip: ipv6::Ipv6Addr,
+    pub target_port: SqlU16,
+
+    pub notification_type: UpstairsRepairNotificationType,
+}
+
+impl UpstairsRepairNotification {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        time: DateTime<Utc>,
+        repair_id: TypedUuid<UpstairsRepairKind>,
+        repair_type: UpstairsRepairType,
+        upstairs_id: TypedUuid<UpstairsKind>,
+        session_id: TypedUuid<UpstairsSessionKind>,
+        region_id: TypedUuid<DownstairsRegionKind>,
+        target_addr: SocketAddrV6,
+        notification_type: UpstairsRepairNotificationType,
+    ) -> Self {
+        Self {
+            time,
+            repair_id: repair_id.into(),
+            repair_type,
+            upstairs_id: upstairs_id.into(),
+            session_id: session_id.into(),
+            region_id: region_id.into(),
+            target_ip: target_addr.ip().into(),
+            target_port: target_addr.port().into(),
+            notification_type,
+        }
+    }
+
+    pub fn address(&self) -> SocketAddrV6 {
+        SocketAddrV6::new(*self.target_ip, *self.target_port, 0, 0)
+    }
+}
+
+/// A record of Crucible Upstairs repair progress.
+#[derive(Queryable, Insertable, Debug, Clone, Selectable)]
+#[diesel(table_name = upstairs_repair_progress)]
+pub struct UpstairsRepairProgress {
+    pub repair_id: DbTypedUuid<UpstairsRepairKind>,
+    pub time: DateTime<Utc>,
+    pub current_item: i64,
+    pub total_items: i64,
+}