From 72a21d610943e4c6933ff435331004855e195125 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 12 Jun 2024 08:42:22 -0400 Subject: [PATCH] Support updating the RoT bootloader (stage0) --- Cargo.lock | 13 +- clients/sled-agent-client/src/lib.rs | 9 + common/src/api/internal/nexus.rs | 10 +- common/src/update.rs | 17 + openapi/sled-agent.json | 5 +- openapi/wicketd.json | 42 ++ .../tests/output/self-stat-schema.json | 4 +- sp-sim/src/gimlet.rs | 40 +- tufaceous-lib/src/assemble/manifest.rs | 9 +- tufaceous/manifests/fake.toml | 17 + update-common/src/artifacts/update_plan.rs | 214 +++++++++- wicket-common/src/update_events.rs | 12 + wicket/README.md | 13 +- wicket/src/cli/rack_update.rs | 5 + wicket/src/runner.rs | 4 + wicket/src/state/force_update.rs | 27 +- wicket/src/state/inventory.rs | 23 ++ wicket/src/state/update.rs | 21 +- wicket/src/ui/panes/overview.rs | 16 +- wicket/src/ui/panes/update.rs | 83 +++- wicketd/src/http_entrypoints.rs | 11 +- wicketd/src/update_tracker.rs | 391 +++++++++++++++++- 22 files changed, 906 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 01e6f04557b..b43c6fe327f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3225,9 +3225,10 @@ dependencies = [ [[package]] name = "hubtools" -version = "0.4.1" -source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#73cd5a84689d59ecce9da66ad4389c540d315168" +version = "0.4.6" +source = "git+https://github.com/oxidecomputer/hubtools.git?branch=main#943c4bbe6b50d1ab635d085d6204895fb4154e79" dependencies = [ + "hex", "lpc55_areas", "lpc55_sign", "object 0.30.4", @@ -4131,8 +4132,8 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lpc55_areas" -version = "0.2.4" -source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" +version = "0.2.5" +source = "git+https://github.com/oxidecomputer/lpc55_support#131520fc913ecce9b80557e854751953f743a7d2" dependencies = [ "bitfield", "clap", @@ -4142,8 +4143,8 @@ dependencies = [ [[package]] name = "lpc55_sign" -version = "0.3.3" -source = "git+https://github.com/oxidecomputer/lpc55_support#96f064eaae5e95930efaab6c29fd1b2e22225dac" +version = "0.3.4" +source = "git+https://github.com/oxidecomputer/lpc55_support#131520fc913ecce9b80557e854751953f743a7d2" dependencies = [ "byteorder", "const-oid", diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 862ae00cc91..a0c0b030dc0 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -440,6 +440,15 @@ impl From use omicron_common::api::internal::nexus::KnownArtifactKind; match s { + KnownArtifactKind::GimletRotBootloader => { + types::KnownArtifactKind::GimletRotBootloader + } + KnownArtifactKind::PscRotBootloader => { + types::KnownArtifactKind::PscRotBootloader + } + KnownArtifactKind::SwitchRotBootloader => { + types::KnownArtifactKind::SwitchRotBootloader + } KnownArtifactKind::GimletSp => types::KnownArtifactKind::GimletSp, KnownArtifactKind::GimletRot => types::KnownArtifactKind::GimletRot, KnownArtifactKind::Host => types::KnownArtifactKind::Host, diff --git a/common/src/api/internal/nexus.rs b/common/src/api/internal/nexus.rs index b569437f433..623c22e9e16 100644 --- a/common/src/api/internal/nexus.rs +++ b/common/src/api/internal/nexus.rs @@ -180,16 +180,11 @@ pub struct UpdateArtifactId { // // 1. Add it here. // -// 2. Add the new kind to /{nexus-client,sled-agent-client}/lib.rs. +// 2. Add the new kind to /clients/src/lib.rs. // The mapping from `UpdateArtifactKind::*` to `types::UpdateArtifactKind::*` // must be left as a `todo!()` for now; `types::UpdateArtifactKind` will not // be updated with the new variant until step 5 below. // -// 3. Add it to the sql database schema under (CREATE TYPE -// omicron.public.update_artifact_kind). -// -// TODO: After omicron ships this would likely involve a DB migration. -// // 4. Add the new kind and the mapping to its `update_artifact_kind` to // /nexus/db-model/src/update_artifact.rs // @@ -231,6 +226,7 @@ pub enum KnownArtifactKind { // Sled Artifacts GimletSp, GimletRot, + GimletRotBootloader, Host, Trampoline, ControlPlane, @@ -238,10 +234,12 @@ pub enum KnownArtifactKind { // PSC Artifacts PscSp, PscRot, + PscRotBootloader, // Switch Artifacts SwitchSp, SwitchRot, + SwitchRotBootloader, } impl KnownArtifactKind { diff --git a/common/src/update.rs b/common/src/update.rs index 9feff1f8688..fc747cf16d9 100644 --- a/common/src/update.rs +++ b/common/src/update.rs @@ -177,6 +177,12 @@ impl ArtifactKind { /// These artifact kinds are not stored anywhere, but are derived from stored /// kinds and used as internal identifiers. impl ArtifactKind { + /// Gimlet root of trust bootloader slot image identifier. + /// + /// Derived from [`KnownArtifactKind::GimletRotBootloader`]. + pub const GIMLET_ROT_STAGE0: Self = + Self::from_static("gimlet_rot_bootloader"); + /// Gimlet root of trust A slot image identifier. /// /// Derived from [`KnownArtifactKind::GimletRot`]. @@ -189,6 +195,11 @@ impl ArtifactKind { pub const GIMLET_ROT_IMAGE_B: Self = Self::from_static("gimlet_rot_image_b"); + /// PSC root of trust stage0 image identifier. + /// + /// Derived from [`KnownArtifactKind::PscRotBootloader`]. + pub const PSC_ROT_STAGE0: Self = Self::from_static("psc_rot_bootloader"); + /// PSC root of trust A slot image identifier. /// /// Derived from [`KnownArtifactKind::PscRot`]. @@ -199,6 +210,12 @@ impl ArtifactKind { /// Derived from [`KnownArtifactKind::PscRot`]. pub const PSC_ROT_IMAGE_B: Self = Self::from_static("psc_rot_image_b"); + /// Switch root of trust A slot image identifier. + /// + /// Derived from [`KnownArtifactKind::SwitchRotBootloader`]. + pub const SWITCH_ROT_STAGE0: Self = + Self::from_static("switch_rot_bootloader"); + /// Switch root of trust A slot image identifier. /// /// Derived from [`KnownArtifactKind::SwitchRot`]. diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 68513345e22..aaf6dad7235 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -3379,13 +3379,16 @@ "enum": [ "gimlet_sp", "gimlet_rot", + "gimlet_rot_bootloader", "host", "trampoline", "control_plane", "psc_sp", "psc_rot", + "psc_rot_bootloader", "switch_sp", - "switch_rot" + "switch_rot", + "switch_rot_bootloader" ] }, "L4PortRange": { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index edef5b98138..922a62366ec 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -3274,6 +3274,10 @@ "StartUpdateOptions": { "type": "object", "properties": { + "skip_rot_bootloader_version_check": { + "description": "If true, skip the check on the current RoT version and always update it regardless of whether the update appears to be neeeded.", + "type": "boolean" + }, "skip_rot_version_check": { "description": "If true, skip the check on the current RoT version and always update it regardless of whether the update appears to be neeeded.", "type": "boolean" @@ -3291,6 +3295,15 @@ } ] }, + "test_simulate_rot_bootloader_result": { + "nullable": true, + "description": "If passed in, simulates a result for the RoT Bootloader update.\n\nThis is used for testing.", + "allOf": [ + { + "$ref": "#/components/schemas/UpdateSimulatedResult" + } + ] + }, "test_simulate_rot_result": { "nullable": true, "description": "If passed in, simulates a result for the RoT update.\n\nThis is used for testing.", @@ -3318,6 +3331,7 @@ } }, "required": [ + "skip_rot_bootloader_version_check", "skip_rot_version_check", "skip_sp_version_check" ] @@ -4734,6 +4748,20 @@ }, "UpdateComponent": { "oneOf": [ + { + "type": "object", + "properties": { + "component": { + "type": "string", + "enum": [ + "rot_bootloader" + ] + } + }, + "required": [ + "component" + ] + }, { "type": "object", "properties": { @@ -4822,6 +4850,20 @@ "state" ] }, + { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "interrogate_rot_bootloader" + ] + } + }, + "required": [ + "id" + ] + }, { "type": "object", "properties": { diff --git a/oximeter/collector/tests/output/self-stat-schema.json b/oximeter/collector/tests/output/self-stat-schema.json index 019e05b4946..f5e439d40f5 100644 --- a/oximeter/collector/tests/output/self-stat-schema.json +++ b/oximeter/collector/tests/output/self-stat-schema.json @@ -39,7 +39,7 @@ } ], "datum_type": "cumulative_u64", - "created": "2024-06-04T20:49:05.675711686Z" + "created": "2024-06-11T21:04:40.129429782Z" }, "oximeter_collector:failed_collections": { "timeseries_name": "oximeter_collector:failed_collections", @@ -86,6 +86,6 @@ } ], "datum_type": "cumulative_u64", - "created": "2024-06-04T20:49:05.676050088Z" + "created": "2024-06-11T21:04:40.130239084Z" } } \ No newline at end of file diff --git a/sp-sim/src/gimlet.rs b/sp-sim/src/gimlet.rs index 280248d0349..4e0b264e64f 100644 --- a/sp-sim/src/gimlet.rs +++ b/sp-sim/src/gimlet.rs @@ -1275,12 +1275,13 @@ impl SpHandler for Handler { "port" => ?port, "component" => ?component, ); - if component == SpComponent::ROT { - Ok(rot_slot_id_to_u16(self.rot_active_slot)) - } else { + match component { + SpComponent::ROT => Ok(rot_slot_id_to_u16(self.rot_active_slot)), + // The only active component is stage0 + SpComponent::STAGE0 => Ok(0), // The real SP returns `RequestUnsupportedForComponent` for anything // other than the RoT, including SP_ITSELF. - Err(SpError::RequestUnsupportedForComponent) + _ => Err(SpError::RequestUnsupportedForComponent), } } @@ -1300,16 +1301,27 @@ impl SpHandler for Handler { "slot" => slot, "persist" => persist, ); - if component == SpComponent::ROT { - self.rot_active_slot = rot_slot_id_from_u16(slot)?; - Ok(()) - } else if component == SpComponent::HOST_CPU_BOOT_FLASH { - self.update_state.set_active_host_slot(slot); - Ok(()) - } else { - // The real SP returns `RequestUnsupportedForComponent` for anything - // other than the RoT and host boot flash, including SP_ITSELF. - Err(SpError::RequestUnsupportedForComponent) + match component { + SpComponent::ROT => { + self.rot_active_slot = rot_slot_id_from_u16(slot)?; + Ok(()) + } + SpComponent::STAGE0 => { + if slot == 1 { + return Ok(()); + } else { + Err(SpError::RequestUnsupportedForComponent) + } + } + SpComponent::HOST_CPU_BOOT_FLASH => { + self.update_state.set_active_host_slot(slot); + Ok(()) + } + _ => { + // The real SP returns `RequestUnsupportedForComponent` for anything + // other than the RoT and host boot flash, including SP_ITSELF. + Err(SpError::RequestUnsupportedForComponent) + } } } diff --git a/tufaceous-lib/src/assemble/manifest.rs b/tufaceous-lib/src/assemble/manifest.rs index 1c4a676f4c1..e9187ff0aff 100644 --- a/tufaceous-lib/src/assemble/manifest.rs +++ b/tufaceous-lib/src/assemble/manifest.rs @@ -279,6 +279,9 @@ impl<'a> FakeDataAttributes<'a> { use hubtools::{CabooseBuilder, HubrisArchiveBuilder}; let board = match self.kind { + KnownArtifactKind::GimletRotBootloader + | KnownArtifactKind::PscRotBootloader + | KnownArtifactKind::SwitchRotBootloader => "SimRotStage0", // non-Hubris artifacts: just make fake data KnownArtifactKind::Host | KnownArtifactKind::Trampoline @@ -287,11 +290,11 @@ impl<'a> FakeDataAttributes<'a> { // hubris artifacts: build a fake archive (SimGimletSp and // SimGimletRot are used by sp-sim) KnownArtifactKind::GimletSp => "SimGimletSp", - KnownArtifactKind::GimletRot => "SimGimletRot", + KnownArtifactKind::GimletRot => "SimRot", KnownArtifactKind::PscSp => "fake-psc-sp", KnownArtifactKind::PscRot => "fake-psc-rot", - KnownArtifactKind::SwitchSp => "fake-sidecar-sp", - KnownArtifactKind::SwitchRot => "fake-sidecar-rot", + KnownArtifactKind::SwitchSp => "SimSidecarSp", + KnownArtifactKind::SwitchRot => "SimRot", }; let caboose = CabooseBuilder::default() diff --git a/tufaceous/manifests/fake.toml b/tufaceous/manifests/fake.toml index cc7ccabd740..a71a5e853fd 100644 --- a/tufaceous/manifests/fake.toml +++ b/tufaceous/manifests/fake.toml @@ -68,3 +68,20 @@ version = "1.0.0" kind = "composite-rot" archive_a = { kind = "fake", size = "512KiB" } archive_b = { kind = "fake", size = "512KiB" } + +[[artifact.gimlet_rot_bootloader]] +name = "fake-gimlet-rot-bootloader" +version = "1.0.0" +source = { kind = "fake", size = "1MiB" } + +[[artifact.psc_rot_bootloader]] +name = "fake-psc-rot-bootloader" +version = "1.0.0" +source = { kind = "fake", size = "1MiB" } + +[[artifact.switch_rot_bootloader]] +name = "fake-switch-rot-bootloader" +version = "1.0.0" +source = { kind = "fake", size = "1MiB" } + + diff --git a/update-common/src/artifacts/update_plan.rs b/update-common/src/artifacts/update_plan.rs index c5b171d6483..0c34ea292b6 100644 --- a/update-common/src/artifacts/update_plan.rs +++ b/update-common/src/artifacts/update_plan.rs @@ -44,12 +44,15 @@ pub struct UpdatePlan { pub gimlet_sp: BTreeMap, pub gimlet_rot_a: Vec, pub gimlet_rot_b: Vec, + pub gimlet_rot_bootloader: Vec, pub psc_sp: BTreeMap, pub psc_rot_a: Vec, pub psc_rot_b: Vec, + pub psc_rot_bootloader: Vec, pub sidecar_sp: BTreeMap, pub sidecar_rot_a: Vec, pub sidecar_rot_b: Vec, + pub sidecar_rot_bootloader: Vec, // Note: The Trampoline image is broken into phase1/phase2 as part of our // update plan (because they go to different destinations), but the two @@ -84,12 +87,15 @@ pub struct UpdatePlanBuilder<'a> { gimlet_sp: BTreeMap, gimlet_rot_a: Vec, gimlet_rot_b: Vec, + gimlet_rot_bootloader: Vec, psc_sp: BTreeMap, psc_rot_a: Vec, psc_rot_b: Vec, + psc_rot_bootloader: Vec, sidecar_sp: BTreeMap, sidecar_rot_a: Vec, sidecar_rot_b: Vec, + sidecar_rot_bootloader: Vec, // We always send phase 1 images (regardless of host or trampoline) to the // SP via MGS, so we retain their data. @@ -130,12 +136,15 @@ impl<'a> UpdatePlanBuilder<'a> { gimlet_sp: BTreeMap::new(), gimlet_rot_a: Vec::new(), gimlet_rot_b: Vec::new(), + gimlet_rot_bootloader: Vec::new(), psc_sp: BTreeMap::new(), psc_rot_a: Vec::new(), psc_rot_b: Vec::new(), + psc_rot_bootloader: Vec::new(), sidecar_sp: BTreeMap::new(), sidecar_rot_a: Vec::new(), sidecar_rot_b: Vec::new(), + sidecar_rot_bootloader: Vec::new(), host_phase_1: None, trampoline_phase_1: None, trampoline_phase_2: None, @@ -187,6 +196,17 @@ impl<'a> UpdatePlanBuilder<'a> { | KnownArtifactKind::SwitchRot => { self.add_rot_artifact(artifact_id, artifact_kind, stream).await } + KnownArtifactKind::GimletRotBootloader + | KnownArtifactKind::PscRotBootloader + | KnownArtifactKind::SwitchRotBootloader => { + self.add_rot_bootloader_artifact( + artifact_id, + artifact_kind, + artifact_hash, + stream, + ) + .await + } KnownArtifactKind::Host => { self.add_host_artifact(artifact_id, stream) } @@ -221,7 +241,10 @@ impl<'a> UpdatePlanBuilder<'a> { | KnownArtifactKind::Trampoline | KnownArtifactKind::ControlPlane | KnownArtifactKind::PscRot - | KnownArtifactKind::SwitchRot => unreachable!(), + | KnownArtifactKind::SwitchRot + | KnownArtifactKind::GimletRotBootloader + | KnownArtifactKind::PscRotBootloader + | KnownArtifactKind::SwitchRotBootloader => unreachable!(), }; let mut stream = std::pin::pin!(stream); @@ -274,6 +297,76 @@ impl<'a> UpdatePlanBuilder<'a> { Ok(()) } + async fn add_rot_bootloader_artifact( + &mut self, + artifact_id: ArtifactId, + artifact_kind: KnownArtifactKind, + artifact_hash: ArtifactHash, + stream: impl Stream> + Send, + ) -> Result<(), RepositoryError> { + // We're only called with an RoT bootloader kind. + let (bootloader, bootloader_kind) = match artifact_kind { + KnownArtifactKind::GimletRotBootloader => ( + &mut self.gimlet_rot_bootloader, + ArtifactKind::GIMLET_ROT_STAGE0, + ), + KnownArtifactKind::PscRotBootloader => { + (&mut self.psc_rot_bootloader, ArtifactKind::PSC_ROT_STAGE0) + } + KnownArtifactKind::SwitchRotBootloader => ( + &mut self.sidecar_rot_bootloader, + ArtifactKind::SWITCH_ROT_STAGE0, + ), + KnownArtifactKind::GimletRot + | KnownArtifactKind::Host + | KnownArtifactKind::Trampoline + | KnownArtifactKind::ControlPlane + | KnownArtifactKind::PscRot + | KnownArtifactKind::SwitchRot + | KnownArtifactKind::GimletSp + | KnownArtifactKind::PscSp + | KnownArtifactKind::SwitchSp => unreachable!(), + }; + + let mut stream = std::pin::pin!(stream); + + // SP images are small, and hubtools wants a `&[u8]` to parse, so we'll + // read the whole thing into memory. + let mut data = Vec::new(); + while let Some(res) = stream.next().await { + let chunk = res.map_err(|error| RepositoryError::ReadArtifact { + kind: artifact_kind.into(), + error: Box::new(error), + })?; + data.extend_from_slice(&chunk); + } + + let artifact_hash_id = + ArtifactHashId { kind: artifact_kind.into(), hash: artifact_hash }; + let data = self + .extracted_artifacts + .store( + artifact_hash_id, + futures::stream::iter([Ok(Bytes::from(data))]), + ) + .await?; + bootloader.push(ArtifactIdData { + id: artifact_id.clone(), + data: data.clone(), + //kind: bootloader_kind, + }); + + self.record_extracted_artifact( + artifact_id, + data, + //artifact_kind.into(), + bootloader_kind, + self.log, + )?; + + Ok(()) + } + async fn add_rot_artifact( &mut self, artifact_id: ArtifactId, @@ -305,7 +398,10 @@ impl<'a> UpdatePlanBuilder<'a> { | KnownArtifactKind::Trampoline | KnownArtifactKind::ControlPlane | KnownArtifactKind::PscSp - | KnownArtifactKind::SwitchSp => unreachable!(), + | KnownArtifactKind::SwitchSp + | KnownArtifactKind::GimletRotBootloader + | KnownArtifactKind::SwitchRotBootloader + | KnownArtifactKind::PscRotBootloader => unreachable!(), }; let (rot_a_data, rot_b_data) = Self::extract_nested_artifact_pair( @@ -694,6 +790,18 @@ impl<'a> UpdatePlanBuilder<'a> { KnownArtifactKind::SwitchRot, self.sidecar_rot_a.is_empty() || self.sidecar_rot_b.is_empty(), ), + ( + KnownArtifactKind::GimletRotBootloader, + self.gimlet_rot_bootloader.is_empty(), + ), + ( + KnownArtifactKind::PscRotBootloader, + self.psc_rot_bootloader.is_empty(), + ), + ( + KnownArtifactKind::SwitchRotBootloader, + self.sidecar_rot_bootloader.is_empty(), + ), ] { if no_artifacts { return Err(RepositoryError::MissingArtifactKind(kind)); @@ -732,6 +840,37 @@ impl<'a> UpdatePlanBuilder<'a> { } } + // Same check for the RoT bootloader. We are explicitly treating the + // bootloader as distinct from the main A/B images here. + for (kind, mut single_board_rot_artifacts) in [ + ( + KnownArtifactKind::GimletRotBootloader, + self.gimlet_rot_bootloader.iter(), + ), + ( + KnownArtifactKind::PscRotBootloader, + self.psc_rot_bootloader.iter(), + ), + ( + KnownArtifactKind::SwitchRotBootloader, + self.sidecar_rot_bootloader.iter(), + ), + ] { + // We know each of these iterators has at least 1 element (checked + // above) so we can safely unwrap the first. + let version = + &single_board_rot_artifacts.next().unwrap().id.version; + for artifact in single_board_rot_artifacts { + if artifact.id.version != *version { + return Err(RepositoryError::MultipleVersionsPresent { + kind, + v1: version.clone(), + v2: artifact.id.version.clone(), + }); + } + } + } + // Repeat the same version check for all SP images. (This is a separate // loop because the types of the iterators don't match.) for (kind, mut single_board_sp_artifacts) in [ @@ -758,12 +897,15 @@ impl<'a> UpdatePlanBuilder<'a> { gimlet_sp: self.gimlet_sp, // checked above gimlet_rot_a: self.gimlet_rot_a, // checked above gimlet_rot_b: self.gimlet_rot_b, // checked above + gimlet_rot_bootloader: self.gimlet_rot_bootloader, // checked above psc_sp: self.psc_sp, // checked above psc_rot_a: self.psc_rot_a, // checked above psc_rot_b: self.psc_rot_b, // checked above + psc_rot_bootloader: self.psc_rot_bootloader, // checked above sidecar_sp: self.sidecar_sp, // checked above sidecar_rot_a: self.sidecar_rot_a, // checked above sidecar_rot_b: self.sidecar_rot_b, // checked above + sidecar_rot_bootloader: self.sidecar_rot_bootloader, // checked above host_phase_1: self.host_phase_1.ok_or( RepositoryError::MissingArtifactKind(KnownArtifactKind::Host), )?, @@ -941,6 +1083,22 @@ mod tests { builder.build_to_vec().unwrap() } + fn make_fake_rot_bootloader_image(board: &str, sign: &str) -> Vec { + use hubtools::{CabooseBuilder, HubrisArchiveBuilder}; + + let caboose = CabooseBuilder::default() + .git_commit("this-is-fake-data") + .board(board) + .version("0.0.0") + .name(board) + .sign(sign) + .build(); + + let mut builder = HubrisArchiveBuilder::with_fake_image(); + builder.write_caboose(caboose.as_slice()).unwrap(); + builder.build_to_vec().unwrap() + } + // See documentation for extract_nested_artifact_pair for why multi_thread // is required. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1077,6 +1235,40 @@ mod tests { .unwrap(); } + let gimlet_rot_bootloader = + make_fake_rot_bootloader_image("test-gimlet-a", "test-gimlet-a"); + let psc_rot_bootloader = + make_fake_rot_bootloader_image("test-psc-a", "test-psc-a"); + let switch_rot_bootloader = + make_fake_rot_bootloader_image("test-sidecar-a", "test-sidecar-a"); + + for (kind, artifact) in [ + ( + KnownArtifactKind::GimletRotBootloader, + gimlet_rot_bootloader.clone(), + ), + (KnownArtifactKind::PscRotBootloader, psc_rot_bootloader.clone()), + ( + KnownArtifactKind::SwitchRotBootloader, + switch_rot_bootloader.clone(), + ), + ] { + let hash = ArtifactHash(Sha256::digest(&artifact).into()); + let id = ArtifactId { + name: format!("{kind:?}"), + version: VERSION_0, + kind: kind.into(), + }; + plan_builder + .add_artifact( + id, + hash, + futures::stream::iter([Ok(Bytes::from(artifact))]), + ) + .await + .unwrap(); + } + let UpdatePlanBuildOutput { plan, by_id, .. } = plan_builder.build().unwrap(); @@ -1142,7 +1334,10 @@ mod tests { | KnownArtifactKind::Trampoline | KnownArtifactKind::GimletRot | KnownArtifactKind::PscRot - | KnownArtifactKind::SwitchRot => {} + | KnownArtifactKind::SwitchRot + | KnownArtifactKind::SwitchRotBootloader + | KnownArtifactKind::GimletRotBootloader + | KnownArtifactKind::PscRotBootloader => {} } } @@ -1186,6 +1381,19 @@ mod tests { sidecar_rot.archive_b ); + assert_eq!( + read_to_vec(&plan.gimlet_rot_bootloader[0].data).await, + gimlet_rot_bootloader + ); + assert_eq!( + read_to_vec(&plan.sidecar_rot_bootloader[0].data).await, + switch_rot_bootloader + ); + assert_eq!( + read_to_vec(&plan.psc_rot_bootloader[0].data).await, + psc_rot_bootloader + ); + logctx.cleanup_successful(); } diff --git a/wicket-common/src/update_events.rs b/wicket-common/src/update_events.rs index fe92887646c..667ddce6b33 100644 --- a/wicket-common/src/update_events.rs +++ b/wicket-common/src/update_events.rs @@ -32,6 +32,7 @@ pub enum WicketdEngineSpec {} )] #[serde(tag = "component", rename_all = "snake_case")] pub enum UpdateComponent { + RotBootloader, Rot, Sp, Host, @@ -42,6 +43,7 @@ pub enum UpdateComponent { pub enum UpdateStepId { TestStep, SetHostPowerState { state: PowerState }, + InterrogateRotBootloader, InterrogateRot, InterrogateSp, SpComponentUpdate, @@ -257,6 +259,16 @@ pub enum SpComponentUpdateTerminalError { }, #[error("RoT booted into unexpected slot {active_slot}")] RotUnexpectedActiveSlot { active_slot: u16 }, + #[error("Getting RoT boot info failed")] + GetRotBootInfoFailed { + #[source] + error: anyhow::Error, + }, + #[error("Unexpected error returned from RoT bootloader update")] + RotBootloaderError { + #[source] + error: anyhow::Error, + }, } impl update_engine::AsError for SpComponentUpdateTerminalError { diff --git a/wicket/README.md b/wicket/README.md index 0a24acbe8e6..fc1c93fe831 100644 --- a/wicket/README.md +++ b/wicket/README.md @@ -148,7 +148,18 @@ it on an as-needed basis. ### Using a real SP -TODO +The easiest way is to change the mgs config to point to a running SP instead +of a simulated SP + +``` +[[switch.port]] +kind = "simulated" +fake-interface = "fake-sled1" +# Your SP address here +addr = "[fe80::c1d:93ff:fe20:ffe0%2]:11111" +ignition-target = 3 +location = { switch0 = ["sled", 1], switch1 = ["sled", 1] } +``` ### Running wicketd diff --git a/wicket/src/cli/rack_update.rs b/wicket/src/cli/rack_update.rs index ccacea0e38c..44a2076b227 100644 --- a/wicket/src/cli/rack_update.rs +++ b/wicket/src/cli/rack_update.rs @@ -98,6 +98,10 @@ pub(crate) struct StartRackUpdateArgs { #[clap(flatten)] component_ids: ComponentIdSelector, + /// Force update the RoT Bootloader even if the version is the same. + #[clap(long, help_heading = "Update options")] + force_update_rot_bootloader: bool, + /// Force update the RoT even if the version is the same. #[clap(long, help_heading = "Update options")] force_update_rot: bool, @@ -125,6 +129,7 @@ impl StartRackUpdateArgs { let update_ids = self.component_ids.to_component_ids()?; let options = CreateStartUpdateOptions { + force_update_rot_bootloader: self.force_update_rot_bootloader, force_update_rot: self.force_update_rot, force_update_sp: self.force_update_sp, } diff --git a/wicket/src/runner.rs b/wicket/src/runner.rs index e83d321459a..77fbb82df86 100644 --- a/wicket/src/runner.rs +++ b/wicket/src/runner.rs @@ -176,6 +176,10 @@ impl RunnerCore { Action::StartUpdate(component_id) => { if let Some(wicketd) = wicketd { let options = CreateStartUpdateOptions { + force_update_rot_bootloader: self + .state + .force_update_state + .force_update_rot_bootloader, force_update_rot: self .state .force_update_state diff --git a/wicket/src/state/force_update.rs b/wicket/src/state/force_update.rs index 72533f13786..feafac88a7b 100644 --- a/wicket/src/state/force_update.rs +++ b/wicket/src/state/force_update.rs @@ -7,6 +7,7 @@ use wicket_common::update_events::UpdateComponent; #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct ForceUpdateState { + pub force_update_rot_bootloader: bool, pub force_update_rot: bool, pub force_update_sp: bool, selected_component: UpdateComponent, @@ -15,9 +16,10 @@ pub struct ForceUpdateState { impl Default for ForceUpdateState { fn default() -> Self { Self { + force_update_rot_bootloader: false, force_update_rot: false, force_update_sp: false, - selected_component: UpdateComponent::Rot, + selected_component: UpdateComponent::RotBootloader, } } } @@ -28,20 +30,29 @@ impl ForceUpdateState { } pub fn next_component(&mut self) { - if self.selected_component == UpdateComponent::Rot { - self.selected_component = UpdateComponent::Sp; - } else { - self.selected_component = UpdateComponent::Rot; - } + self.selected_component = match self.selected_component { + UpdateComponent::RotBootloader => UpdateComponent::Rot, + UpdateComponent::Rot => UpdateComponent::Sp, + UpdateComponent::Sp => UpdateComponent::RotBootloader, + _ => unreachable!(), + }; } pub fn prev_component(&mut self) { - // We only have 2 components; next/prev are both toggles. - self.next_component(); + self.selected_component = match self.selected_component { + UpdateComponent::RotBootloader => UpdateComponent::Sp, + UpdateComponent::Rot => UpdateComponent::RotBootloader, + UpdateComponent::Sp => UpdateComponent::Rot, + _ => unreachable!(), + }; } pub fn toggle(&mut self, component: UpdateComponent) { match component { + UpdateComponent::RotBootloader => { + self.force_update_rot_bootloader = + !self.force_update_rot_bootloader + } UpdateComponent::Rot => { self.force_update_rot = !self.force_update_rot; } diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs index 5cfe536dfbe..0ab187cc483 100644 --- a/wicket/src/state/inventory.rs +++ b/wicket/src/state/inventory.rs @@ -171,6 +171,21 @@ impl Component { self.sp().rot.as_ref().and_then(|rot| rot.caboose_b.as_ref()), ) } + + pub fn stage0_version(&self) -> String { + version_or_unknown( + self.sp().rot.as_ref().and_then(|rot| rot.caboose_stage0.as_ref()), + ) + } + + pub fn stage0next_version(&self) -> String { + version_or_unknown( + self.sp() + .rot + .as_ref() + .and_then(|rot| rot.caboose_stage0next.as_ref()), + ) + } } /// The component type and its slot. @@ -256,6 +271,14 @@ impl ComponentId { } } + pub fn rot_bootloader_known_artifact_kind(&self) -> KnownArtifactKind { + match self { + ComponentId::Sled(_) => KnownArtifactKind::GimletRotBootloader, + ComponentId::Switch(_) => KnownArtifactKind::SwitchRotBootloader, + ComponentId::Psc(_) => KnownArtifactKind::PscRotBootloader, + } + } + pub fn to_string_uppercase(&self) -> String { let mut s = self.to_string(); s.make_ascii_uppercase(); diff --git a/wicket/src/state/update.rs b/wicket/src/state/update.rs index 77bbdd83d2d..31876365e22 100644 --- a/wicket/src/state/update.rs +++ b/wicket/src/state/update.rs @@ -46,6 +46,7 @@ impl RackUpdateState { *id, vec![ UpdateComponent::Rot, + UpdateComponent::RotBootloader, UpdateComponent::Sp, UpdateComponent::Host, ], @@ -55,14 +56,22 @@ impl RackUpdateState { *id, UpdateItem::new( *id, - vec![UpdateComponent::Rot, UpdateComponent::Sp], + vec![ + UpdateComponent::Rot, + UpdateComponent::RotBootloader, + UpdateComponent::Sp, + ], ), ), ComponentId::Psc(_) => ( *id, UpdateItem::new( *id, - vec![UpdateComponent::Rot, UpdateComponent::Sp], + vec![ + UpdateComponent::Rot, + UpdateComponent::RotBootloader, + UpdateComponent::Sp, + ], ), ), }) @@ -429,6 +438,7 @@ fn update_component_state( #[allow(unused)] pub fn update_component_title(component: UpdateComponent) -> &'static str { match component { + UpdateComponent::RotBootloader => "ROT_BOOTLOADER", UpdateComponent::Rot => "ROT", UpdateComponent::Sp => "SP", UpdateComponent::Host => "HOST", @@ -436,6 +446,7 @@ pub fn update_component_title(component: UpdateComponent) -> &'static str { } pub struct CreateStartUpdateOptions { + pub(crate) force_update_rot_bootloader: bool, pub(crate) force_update_rot: bool, pub(crate) force_update_sp: bool, } @@ -454,7 +465,9 @@ impl CreateStartUpdateOptions { as a u64", ) }); - + let test_simulate_rot_bootloader_result = get_update_simulated_result( + "WICKET_UPDATE_TEST_SIMULATE_ROT_BOOTLOADER_RESULT", + )?; let test_simulate_rot_result = get_update_simulated_result( "WICKET_UPDATE_TEST_SIMULATE_ROT_RESULT", )?; @@ -465,8 +478,10 @@ impl CreateStartUpdateOptions { Ok(StartUpdateOptions { test_error, test_step_seconds, + test_simulate_rot_bootloader_result, test_simulate_rot_result, test_simulate_sp_result, + skip_rot_bootloader_version_check: self.force_update_rot_bootloader, skip_rot_version_check: self.force_update_rot, skip_sp_version_check: self.force_update_sp, }) diff --git a/wicket/src/ui/panes/overview.rs b/wicket/src/ui/panes/overview.rs index 7d60c417725..45d02311aad 100644 --- a/wicket/src/ui/panes/overview.rs +++ b/wicket/src/ui/panes/overview.rs @@ -770,12 +770,12 @@ fn inventory_description(component: &Component) -> Text { .into(), ); } - if let Some(_) = slot_a_error { + if let Some(e) = slot_a_error { spans.push( vec![ nest_bullet(), Span::styled("Image status: ", label_style), - Span::styled("Error: ", bad_style), + Span::styled(format!("Error: {e:?}"), bad_style), ] .into(), ); @@ -813,12 +813,12 @@ fn inventory_description(component: &Component) -> Text { .into(), ); } - if let Some(_) = slot_b_error { + if let Some(e) = slot_b_error { spans.push( vec![ nest_bullet(), Span::styled("Image status: ", label_style), - Span::styled("Error: ", bad_style), + Span::styled(format!("Error: {e:?}"), bad_style), ] .into(), ); @@ -857,12 +857,12 @@ fn inventory_description(component: &Component) -> Text { .into(), ); } - if let Some(_) = stage0_error { + if let Some(e) = stage0_error { spans.push( vec![ nest_bullet(), Span::styled("Image status: ", label_style), - Span::styled("Error: ", bad_style), + Span::styled(format!("Error: {e:?}"), bad_style), ] .into(), ); @@ -902,12 +902,12 @@ fn inventory_description(component: &Component) -> Text { .into(), ); } - if let Some(_) = stage0next_error { + if let Some(e) = stage0next_error { spans.push( vec![ nest_bullet(), Span::styled("Image status: ", label_style), - Span::styled("Error: ", bad_style), + Span::styled(format!("Error: {e:?}"), bad_style), ] .into(), ); diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index 664c647eaca..10bdd985446 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -1711,6 +1711,7 @@ struct ComponentForceUpdateSelectionState { } struct ForceUpdateSelectionState { + rot_bootloader: Option, rot: Option, sp: Option, } @@ -1722,6 +1723,7 @@ impl From<&'_ State> for ForceUpdateSelectionState { let inventory = &state.inventory; let update_item = &state.update_state.items[&component_id]; + let mut rot_bootloader = None; let mut rot = None; let mut sp = None; @@ -1737,6 +1739,22 @@ impl From<&'_ State> for ForceUpdateSelectionState { let installed_version = active_installed_version(&component_id, component, inventory); match component { + UpdateComponent::RotBootloader => { + assert!( + rot_bootloader.is_none(), + "update item contains multiple RoT bootloader entries" + ); + if artifact_version == installed_version { + rot_bootloader = + Some(ComponentForceUpdateSelectionState { + version: artifact_version, + toggled_on: state + .force_update_state + .force_update_rot, + selected: false, // set below + }); + } + } UpdateComponent::Rot => { assert!( rot.is_none(), @@ -1773,10 +1791,13 @@ impl From<&'_ State> for ForceUpdateSelectionState { // If we only have one force-updateable component, mark it as selected; // otherwise, respect the option currently selected in `State`. - match (rot.as_mut(), sp.as_mut()) { - (Some(rot), None) => rot.selected = true, - (None, Some(sp)) => sp.selected = true, - (Some(rot), Some(sp)) => { + match (rot_bootloader.as_mut(), rot.as_mut(), sp.as_mut()) { + (Some(rot_bootloader), None, None) => { + rot_bootloader.selected = true + } + (None, Some(rot), None) => rot.selected = true, + (None, None, Some(sp)) => sp.selected = true, + (None, Some(rot), Some(sp)) => { if state.force_update_state.selected_component() == UpdateComponent::Rot { @@ -1785,16 +1806,19 @@ impl From<&'_ State> for ForceUpdateSelectionState { sp.selected = true; } } - (None, None) => (), + // XXX + _ => (), } - Self { rot, sp } + Self { rot_bootloader, rot, sp } } } impl ForceUpdateSelectionState { fn num_spans(&self) -> usize { - usize::from(self.rot.is_some()) + usize::from(self.sp.is_some()) + usize::from(self.rot.is_some()) + + usize::from(self.sp.is_some()) + + usize::from(self.rot_bootloader.is_some()) } fn next_component(&self, state: &mut State) { @@ -1826,6 +1850,13 @@ impl ForceUpdateSelectionState { state.force_update_state.toggle(UpdateComponent::Rot); } else if self.sp.as_ref().map(|sp| sp.selected).unwrap_or(false) { state.force_update_state.toggle(UpdateComponent::Sp); + } else if self + .rot_bootloader + .as_ref() + .map(|rot_bootloader| rot_bootloader.selected) + .unwrap_or(false) + { + state.force_update_state.toggle(UpdateComponent::RotBootloader); } } @@ -1850,6 +1881,9 @@ impl ForceUpdateSelectionState { } let mut spans = Vec::new(); + if let Some(rot_bootloader) = self.rot_bootloader.as_ref() { + spans.push(make_spans("RoT Bootloader", rot_bootloader)); + } if let Some(rot) = self.rot.as_ref() { spans.push(make_spans("RoT", rot)); } @@ -2201,6 +2235,10 @@ fn active_installed_version( ) -> String { let component = inventory.get_inventory(id); match update_component { + UpdateComponent::RotBootloader => component.map_or_else( + || "UNKNOWN".to_string(), + |component| component.stage0_version(), + ), UpdateComponent::Sp => component.map_or_else( || "UNKNOWN".to_string(), |component| component.sp_version_active(), @@ -2254,6 +2292,26 @@ fn all_installed_versions( ] }, ), + UpdateComponent::RotBootloader => component.map_or_else( + || { + vec![InstalledVersion { + title: base_title.into(), + version: "UNKNOWN".into(), + }] + }, + |component| { + vec![ + InstalledVersion { + title: "{base_title}".to_string().into(), + version: component.stage0_version().into(), + }, + InstalledVersion { + title: format!("{base_title}_NEXT").into(), + version: component.stage0next_version().into(), + }, + ] + }, + ), UpdateComponent::Rot => component.map_or_else( || { vec![InstalledVersion { @@ -2301,6 +2359,9 @@ fn artifact_version( versions: &BTreeMap, ) -> String { let artifact = match (id, component) { + (ComponentId::Sled(_), UpdateComponent::RotBootloader) => { + KnownArtifactKind::GimletRotBootloader + } (ComponentId::Sled(_), UpdateComponent::Rot) => { KnownArtifactKind::GimletRot } @@ -2310,12 +2371,18 @@ fn artifact_version( (ComponentId::Sled(_), UpdateComponent::Host) => { KnownArtifactKind::Host } + (ComponentId::Switch(_), UpdateComponent::RotBootloader) => { + KnownArtifactKind::SwitchRotBootloader + } (ComponentId::Switch(_), UpdateComponent::Rot) => { KnownArtifactKind::SwitchRot } (ComponentId::Switch(_), UpdateComponent::Sp) => { KnownArtifactKind::SwitchSp } + (ComponentId::Psc(_), UpdateComponent::RotBootloader) => { + KnownArtifactKind::PscRotBootloader + } (ComponentId::Psc(_), UpdateComponent::Rot) => { KnownArtifactKind::PscRot } @@ -2363,7 +2430,7 @@ impl Control for UpdatePane { [ Constraint::Length(3), Constraint::Length(3), - Constraint::Length(6), + Constraint::Length(8), Constraint::Min(0), Constraint::Length(3), ] diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 999428ff06a..001974e085d 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -697,6 +697,12 @@ pub(crate) struct StartUpdateOptions { /// This is used for testing. pub(crate) test_step_seconds: Option, + /// If passed in, simulates a result for the RoT Bootloader update. + /// + /// This is used for testing. + pub(crate) test_simulate_rot_bootloader_result: + Option, + /// If passed in, simulates a result for the RoT update. /// /// This is used for testing. @@ -709,7 +715,10 @@ pub(crate) struct StartUpdateOptions { /// If true, skip the check on the current RoT version and always update it /// regardless of whether the update appears to be neeeded. - #[allow(dead_code)] // TODO actually use this + pub(crate) skip_rot_bootloader_version_check: bool, + + /// If true, skip the check on the current RoT version and always update it + /// regardless of whether the update appears to be neeeded. pub(crate) skip_rot_version_check: bool, /// If true, skip the check on the current SP version and always update it diff --git a/wicketd/src/update_tracker.rs b/wicketd/src/update_tracker.rs index 10253bc2f77..c247c7a8e14 100644 --- a/wicketd/src/update_tracker.rs +++ b/wicketd/src/update_tracker.rs @@ -23,12 +23,15 @@ use display_error_chain::DisplayErrorChain; use dropshot::HttpError; use futures::Stream; use futures::TryFutureExt; +use gateway_client::types::GetRotBootInfoParams; use gateway_client::types::HostPhase2Progress; use gateway_client::types::HostPhase2RecoveryImageId; use gateway_client::types::HostStartupOptions; use gateway_client::types::InstallinatorImageId; use gateway_client::types::PowerState; use gateway_client::types::RotCfpaSlot; +use gateway_client::types::RotImageError; +use gateway_client::types::RotState; use gateway_client::types::SpComponentFirmwareSlot; use gateway_client::types::SpIdentifier; use gateway_client::types::SpType; @@ -862,19 +865,47 @@ impl UpdateDriver { define_test_steps(&engine, secs); } - let (rot_a, rot_b, sp_artifacts) = match update_cx.sp.type_ { - SpType::Sled => { - (&plan.gimlet_rot_a, &plan.gimlet_rot_b, &plan.gimlet_sp) - } - SpType::Power => (&plan.psc_rot_a, &plan.psc_rot_b, &plan.psc_sp), - SpType::Switch => { - (&plan.sidecar_rot_a, &plan.sidecar_rot_b, &plan.sidecar_sp) - } - }; + let (rot_a, rot_b, sp_artifacts, rot_bootloader) = + match update_cx.sp.type_ { + SpType::Sled => ( + &plan.gimlet_rot_a, + &plan.gimlet_rot_b, + &plan.gimlet_sp, + &plan.gimlet_rot_bootloader, + ), + SpType::Power => ( + &plan.psc_rot_a, + &plan.psc_rot_b, + &plan.psc_sp, + &plan.psc_rot_bootloader, + ), + SpType::Switch => ( + &plan.sidecar_rot_a, + &plan.sidecar_rot_b, + &plan.sidecar_sp, + &plan.sidecar_rot_bootloader, + ), + }; + let rot_bootloader_registrar = + engine.for_component(UpdateComponent::RotBootloader); let rot_registrar = engine.for_component(UpdateComponent::Rot); let sp_registrar = engine.for_component(UpdateComponent::Sp); + // To update the RoT, we have to know which slot (A or B) it is + // currently executing; we must update the _other_ slot. We also want to + // know its current version (so we can skip updating if we only need to + // update the SP and/or host). + let rot_bootloader_interrogation = rot_bootloader_registrar + .new_step( + UpdateStepId::InterrogateRot, + "Checking current RoT bootloader version", + move |_cx| async move { + update_cx.interrogate_rot_bootloader(rot_bootloader).await + }, + ) + .register(); + // To update the RoT, we have to know which slot (A or B) it is // currently executing; we must update the _other_ slot. We also want to // know its current version (so we can skip updating if we only need to @@ -946,6 +977,78 @@ impl UpdateDriver { }, ) .register(); + + // Send the bootloader update to the RoT. + // XXX is this where we mutex? + let inner_cx = SpComponentUpdateContext::new( + update_cx, + UpdateComponent::RotBootloader, + ); + rot_bootloader_registrar + .new_step( + UpdateStepId::SpComponentUpdate, + "Updating RoT bootloader", + move |cx| async move { + if let Some(result) = opts.test_simulate_rot_bootloader_result { + return simulate_result(result); + } + + let rot_bootloader_interrogation = + rot_bootloader_interrogation.into_value(cx.token()).await; + + let bootloader_has_this_version = rot_bootloader_interrogation + .active_version_matches_artifact_to_apply(); + + // If this RoT already has this version, skip the rest of + // this step, UNLESS we've been told to skip this version + // check. + if bootloader_has_this_version && !opts.skip_rot_bootloader_version_check { + return StepSkipped::new( + (), + format!( + "RoT bootloader already at version {}", + rot_bootloader_interrogation.available_artifacts_version, + ), + ) + .into(); + } + + let artifact_to_apply = rot_bootloader_interrogation + .choose_artifact_to_apply( + &update_cx.mgs_client, + &update_cx.log, + ) + .await?; + + cx.with_nested_engine(|engine| { + inner_cx.register_steps( + engine, + rot_bootloader_interrogation.slot_to_update, + artifact_to_apply, + ); + Ok(()) + }) + .await?; + + // If we updated despite the RoT already having the version + // we updated to, make this step return a warning with that + // message; otherwise, this is a normal success. + if bootloader_has_this_version { + StepWarning::new( + (), + format!( + "RoT bootloader updated despite already having version {}", + rot_bootloader_interrogation.available_artifacts_version + ), + ) + .into() + } else { + StepSuccess::new(()).into() + } + }, + ) + .register(); + // Send the update to the RoT. let inner_cx = SpComponentUpdateContext::new(update_cx, UpdateComponent::Rot); @@ -1599,6 +1702,9 @@ impl RotInterrogation { /// their CMPA/CFPA pages, if we fail to fetch them _and_ /// `available_artifacts` has exactly one item, we will return that one /// item. + /// + /// This is also applicable to the RoT bootloader which follows the + /// same vaildation method async fn choose_artifact_to_apply( &self, client: &gateway_client::Client, @@ -1844,6 +1950,76 @@ impl UpdateContext { }) } + async fn interrogate_rot_bootloader( + &self, + rot_bootloader: &[ArtifactIdData], + ) -> Result, UpdateTerminalError> { + // We already validated at repo-upload time there is at least one RoT + // artifact available and that all available RoT artifacts are the same + // version, so we can unwrap the first artifact here and assume its + // version matches any subsequent artifacts. + // TODO this needs to be fixed for multi version to work! + let available_artifacts_version = rot_bootloader + .get(0) + .expect("no RoT artifacts available") + .id + .version + .clone(); + + // Read the caboose of the currently running version (always 0) + // When updating from older stage0 we may not have a caboose so an error here + // need not be fatal + // TODO make this fatal at some point + let caboose = self + .mgs_client + .sp_component_caboose_get( + self.sp.type_, + self.sp.slot, + SpComponent::STAGE0.const_as_str(), + 0, + ) + .await + .map(|v| v.into_inner()) + .ok(); + + let available_artifacts = rot_bootloader.to_vec(); + let make_result = |active_version| RotInterrogation { + // We always update slot 1 + slot_to_update: 1, + available_artifacts, + available_artifacts_version, + sp: self.sp, + active_version, + }; + + match caboose { + Some(c) => { + let message = format!( + "RoT bootloader version {} (git commit {})", + c.version, c.git_commit + ); + + match c.version.parse::() { + Ok(version) => StepSuccess::new(make_result(Some(version))) + .with_message(message) + .into(), + Err(err) => StepWarning::new( + make_result(None), + format!( + "{message} (failed to parse RoT bootloader version: {err})" + ), + ) + .into(), + } + } + None => StepSuccess::new(make_result(None)) + .with_message( + "Stage0 has no caboose, proceeding with update anyway", + ) + .into(), + } + } + async fn interrogate_rot( &self, rot_a: &[ArtifactIdData], @@ -1923,6 +2099,52 @@ impl UpdateContext { } } + /// Poll the RoT asking for its boot information. This is used to check + /// state after RoT bootloader updates + async fn wait_for_rot_boot_info( + &self, + timeout: Duration, + ) -> anyhow::Result<(Option, Option)> { + let mut ticker = tokio::time::interval(Duration::from_secs(1)); + + let start = Instant::now(); + loop { + ticker.tick().await; + match self.get_rot_boot_info().await { + Ok(state) => match state { + // the minimum we will ever return is 3 + RotState::V2 { .. } => unreachable!(), + RotState::V3 { stage0_error, stage0next_error, .. } => { + return Ok((stage0_error, stage0next_error)) + } + // ugh + RotState::CommunicationFailed { message } => { + if start.elapsed() < timeout { + warn!( + self.log, + "failed getting RoT boot info (will retry)"; + "error" => %message, + ); + } else { + return Err(anyhow!(message)); + } + } + }, + Err(error) => { + if start.elapsed() < timeout { + warn!( + self.log, + "failed getting RoT boot info (will retry)"; + "error" => %error, + ); + } else { + return Err(error); + } + } + } + } + } + /// Poll the RoT asking for its currently active slot, allowing failures up /// to a fixed timeout to give time for it to boot. /// @@ -1930,16 +2152,14 @@ impl UpdateContext { async fn wait_for_rot_reboot( &self, timeout: Duration, + component: &str, ) -> anyhow::Result { let mut ticker = tokio::time::interval(Duration::from_secs(1)); let start = Instant::now(); loop { ticker.tick().await; - match self - .get_component_active_slot(SpComponent::ROT.const_as_str()) - .await - { + match self.get_component_active_slot(component).await { Ok(slot) => return Ok(slot), Err(error) => { if start.elapsed() < timeout { @@ -2083,6 +2303,22 @@ impl UpdateContext { StepSuccess::new(()).into() } + async fn get_rot_boot_info(&self) -> anyhow::Result { + self.mgs_client + .sp_rot_boot_info( + self.sp.type_, + self.sp.slot, + SpComponent::ROT.const_as_str(), + &GetRotBootInfoParams { + version: + gateway_messages::RotBootInfo::HIGHEST_KNOWN_VERSION, + }, + ) + .await + .context("failed to get RoT boot info") + .map(|res| res.into_inner()) + } + async fn get_component_active_slot( &self, component: &str, @@ -2325,6 +2561,9 @@ impl<'a> SpComponentUpdateContext<'a> { let update_cx = self.update_cx; let component_name = match self.component { + UpdateComponent::RotBootloader => { + SpComponent::STAGE0.const_as_str() + } UpdateComponent::Rot => SpComponent::ROT.const_as_str(), UpdateComponent::Sp => SpComponent::SP_ITSELF.const_as_str(), UpdateComponent::Host => { @@ -2434,13 +2673,132 @@ impl<'a> SpComponentUpdateContext<'a> { // to stage updates for example, but for wicketd-driven recovery it's // fine to do this immediately.) match component { + UpdateComponent::RotBootloader => { + // We need to reset the RoT in order to check the signature on what we just + // updated + registrar + .new_step( + SpComponentUpdateStepId::Resetting, + "Resetting the RoT to check the bootloader signature", + move |_cx| async move { + update_cx + .reset_sp_component(SpComponent::ROT.const_as_str()) + .await + .map_err(|error| { + SpComponentUpdateTerminalError::RotResetFailed { + error, + } + })?; + StepSuccess::new(()).into() + }, + ) + .register(); + + registrar + .new_step( + SpComponentUpdateStepId::Resetting, + "Waiting for RoT to boot".to_string(), + move |_cx| async move { + const WAIT_FOR_BOOT_TIMEOUT: Duration = + Duration::from_secs(30); + let (_, stage0next_error) = update_cx + .wait_for_rot_boot_info(WAIT_FOR_BOOT_TIMEOUT) + .await + .map_err(|error| { + SpComponentUpdateTerminalError::GetRotBootInfoFailed { error } + })?; + + // check that stage0next is valid before we try to set the component + if let Some(error) = stage0next_error { + return Err(SpComponentUpdateTerminalError::RotBootloaderError { + error: anyhow!(format!("{error:?}")) + }); + } + StepSuccess::new(()).into() + }, + ) + .register(); + + // Actually set stage0 to use the new firmware + registrar + .new_step( + SpComponentUpdateStepId::SettingActiveBootSlot, + format!("Setting {component_name} active slot to {firmware_slot}"), + move |_cx| async move { + update_cx + .set_component_active_slot( + component_name, + firmware_slot, + true, + ) + .await + .map_err(|error| { + SpComponentUpdateTerminalError::SetRotActiveSlotFailed { + error, + } + })?; + StepSuccess::new(()).into() + }, + ) + .register(); + + // Now reset (again) to boot into the new stage0 + registrar + .new_step( + SpComponentUpdateStepId::Resetting, + "Resetting the RoT to boot into the new bootloader", + move |_cx| async move { + update_cx + .reset_sp_component(SpComponent::ROT.const_as_str()) + .await + .map_err(|error| { + SpComponentUpdateTerminalError::RotResetFailed { + error, + } + })?; + StepSuccess::new(()).into() + }, + ) + .register(); + + registrar + .new_step( + SpComponentUpdateStepId::Resetting, + "Checking the new RoT bootloader".to_string(), + move |_cx| async move { + const WAIT_FOR_BOOT_TIMEOUT: Duration = + Duration::from_secs(30); + let (stage0_error, stage0next_error) = update_cx + .wait_for_rot_boot_info(WAIT_FOR_BOOT_TIMEOUT) + .await + .map_err(|error| { + SpComponentUpdateTerminalError::GetRotActiveSlotFailed { error } + })?; + + // Both the active and pending slots should be valid after this spot + if let Some(error) = stage0_error { + return Err(SpComponentUpdateTerminalError::RotBootloaderError { + error: anyhow!(format!("{error:?}")) + }); + } + if let Some(error) = stage0next_error { + return Err(SpComponentUpdateTerminalError::RotBootloaderError { + error: anyhow!(format!("{error:?}")) + }); + } + + StepSuccess::new(()).into() + }, + ) + .register(); + } UpdateComponent::Rot => { // Prior to rebooting the RoT, we have to tell it to boot into // the firmware slot we just updated. registrar .new_step( SpComponentUpdateStepId::SettingActiveBootSlot, - format!("Setting RoT active slot to {firmware_slot}"), + format!("Setting {component_name} active slot to {firmware_slot}"), move |_cx| async move { update_cx .set_component_active_slot( @@ -2463,7 +2821,7 @@ impl<'a> SpComponentUpdateContext<'a> { registrar .new_step( SpComponentUpdateStepId::Resetting, - "Resetting RoT", + "Resetting {component name}", move |_cx| async move { update_cx .reset_sp_component(component_name) @@ -2502,7 +2860,7 @@ impl<'a> SpComponentUpdateContext<'a> { const WAIT_FOR_BOOT_TIMEOUT: Duration = Duration::from_secs(30); let active_slot = update_cx - .wait_for_rot_reboot(WAIT_FOR_BOOT_TIMEOUT) + .wait_for_rot_reboot(WAIT_FOR_BOOT_TIMEOUT, component_name) .await .map_err(|error| { SpComponentUpdateTerminalError::GetRotActiveSlotFailed { error } @@ -2518,6 +2876,7 @@ impl<'a> SpComponentUpdateContext<'a> { } UpdateComponent::Sp => { // Nothing special to do on the SP - just reset it. + // TODO fixup the SP to also set the active slot registrar .new_step( SpComponentUpdateStepId::Resetting,