diff --git a/update-engine/src/buffer.rs b/update-engine/src/buffer.rs index d1028ff8cc..3e7db63cb9 100644 --- a/update-engine/src/buffer.rs +++ b/update-engine/src/buffer.rs @@ -240,8 +240,13 @@ impl EventStore { // index. let root_event_index = RootEventIndex(event.event_index); - let actions = - self.recurse_for_step_event(&event, 0, None, root_event_index); + let actions = self.recurse_for_step_event( + &event, + 0, + None, + root_event_index, + event.total_elapsed, + ); if let Some(new_execution) = actions.new_execution { if new_execution.nest_level == 0 { self.root_execution_id = Some(new_execution.execution_id); @@ -312,6 +317,7 @@ impl EventStore { nest_level: usize, parent_sort_key: Option<&StepSortKey>, root_event_index: RootEventIndex, + root_total_elapsed: Duration, ) -> RecurseActions { let mut new_execution = None; let (step_key, progress_key) = match &event.kind { @@ -365,6 +371,8 @@ impl EventStore { let info = CompletionInfo { attempt: *attempt, outcome, + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -405,6 +413,8 @@ impl EventStore { let info = CompletionInfo { attempt: *last_attempt, outcome, + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -432,6 +442,8 @@ impl EventStore { total_attempts: *total_attempts, message: message.clone(), causes: causes.clone(), + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -456,6 +468,8 @@ impl EventStore { let info = AbortInfo { attempt: *attempt, message: message.clone(), + root_total_elapsed, + leaf_total_elapsed: event.total_elapsed, step_elapsed: *step_elapsed, attempt_elapsed: *attempt_elapsed, }; @@ -481,6 +495,7 @@ impl EventStore { nest_level + 1, parent_sort_key.as_ref(), root_event_index, + root_total_elapsed, ); if let Some(nested_new_execution) = &actions.new_execution { // Add an edge from the parent node to the new execution's root node. @@ -1164,6 +1179,8 @@ impl StepStatus { pub struct CompletionInfo { pub attempt: usize, pub outcome: StepOutcome, + pub root_total_elapsed: Duration, + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1179,11 +1196,23 @@ pub enum FailureReason { }, } +impl FailureReason { + /// Returns the [`FailureInfo`] if present. + pub fn info(&self) -> Option<&FailureInfo> { + match self { + Self::StepFailed(info) => Some(info), + Self::ParentFailed { .. } => None, + } + } +} + #[derive(Clone, Debug)] pub struct FailureInfo { pub total_attempts: usize, pub message: String, pub causes: Vec, + pub root_total_elapsed: Duration, + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1199,6 +1228,16 @@ pub enum AbortReason { }, } +impl AbortReason { + /// Returns the [`AbortInfo`] if present. + pub fn info(&self) -> Option<&AbortInfo> { + match self { + Self::StepAborted(info) => Some(info), + Self::ParentAborted { .. } => None, + } + } +} + #[derive(Clone, Debug)] pub enum WillNotBeRunReason { /// A preceding step failed. @@ -1230,6 +1269,13 @@ pub enum WillNotBeRunReason { pub struct AbortInfo { pub attempt: usize, pub message: String, + + /// The total elapsed time as reported by the root event. + pub root_total_elapsed: Duration, + + /// The total elapsed time as reported by the leaf execution event, for + /// nested events. + pub leaf_total_elapsed: Duration, pub step_elapsed: Duration, pub attempt_elapsed: Duration, } @@ -1261,14 +1307,61 @@ impl ExecutionSummary { StepStatus::Running { .. } => { execution_status = ExecutionStatus::Running { step_key }; } - StepStatus::Completed { .. } => { - execution_status = ExecutionStatus::Completed { step_key }; + StepStatus::Completed { info } => { + let (root_total_elapsed, leaf_total_elapsed) = match info { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Completed, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } - StepStatus::Failed { .. } => { - execution_status = ExecutionStatus::Failed { step_key }; + StepStatus::Failed { reason } => { + let (root_total_elapsed, leaf_total_elapsed) = + match reason.info() { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Failed, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } - StepStatus::Aborted { .. } => { - execution_status = ExecutionStatus::Aborted { step_key }; + StepStatus::Aborted { reason, .. } => { + let (root_total_elapsed, leaf_total_elapsed) = + match reason.info() { + Some(info) => ( + Some(info.root_total_elapsed), + Some(info.leaf_total_elapsed), + ), + None => (None, None), + }; + + let terminal_status = ExecutionTerminalInfo { + kind: TerminalKind::Aborted, + root_total_elapsed, + leaf_total_elapsed, + step_key, + }; + execution_status = + ExecutionStatus::Terminal(terminal_status); } StepStatus::WillNotBeRun { .. } => { // Ignore steps that will not be run -- a prior step failed. @@ -1306,7 +1399,7 @@ impl StepSortKey { /// Status about a single execution ID. /// /// Part of [`ExecutionSummary`]. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum ExecutionStatus { /// This execution has not been started yet. NotStarted, @@ -1319,27 +1412,50 @@ pub enum ExecutionStatus { step_key: StepKey, }, - /// This execution completed running. - Completed { - /// The last step that completed. - step_key: StepKey, - }, + /// Execution has finished. + Terminal(ExecutionTerminalInfo), +} - /// This execution failed. - Failed { - /// The step key that failed. - /// - /// Use [`EventBuffer::get`] to get more information about this step. - step_key: StepKey, - }, +/// Terminal status about a single execution ID. +/// +/// Part of [`ExecutionStatus`]. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ExecutionTerminalInfo { + /// The way in which this execution reached a terminal state. + pub kind: TerminalKind, + + /// Total elapsed time (root) for this execution. + /// + /// The total elapsed time may not be available if execution was interrupted + /// and we inferred that it was terminated. + pub root_total_elapsed: Option, + + /// Total elapsed time (leaf) for this execution. + /// + /// The total elapsed time may not be available if execution was interrupted + /// and we inferred that it was terminated. + pub leaf_total_elapsed: Option, + /// The step key that was running when this execution was terminated. + /// + /// * For completed executions, this is the last step that completed. + /// * For failed or aborted executions, this is the step that failed. + /// * For aborted executions, this is the step that was running when the + /// abort happened. + pub step_key: StepKey, +} + +/// The way in which an execution was terminated. +/// +/// Part of [`ExecutionStatus`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TerminalKind { + /// This execution completed running. + Completed, + /// This execution failed. + Failed, /// This execution was aborted. - Aborted { - /// The step that was running when the abort happened. - /// - /// Use [`EventBuffer::get`] to get more information about this step. - step_key: StepKey, - }, + Aborted, } /// Keys for the event tree. @@ -1925,8 +2041,9 @@ mod tests { if is_last_event { ensure!( matches!( - summary[&root_execution_id].execution_status, - ExecutionStatus::Completed { .. }, + &summary[&root_execution_id].execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Completed ), "this is the last event so ExecutionStatus must be completed" ); @@ -1941,8 +2058,9 @@ mod tests { .expect("this is the first nested engine"); ensure!( matches!( - nested_summary.execution_status, - ExecutionStatus::Failed { .. }, + &nested_summary.execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Failed ), "for this engine, the ExecutionStatus must be failed" ); @@ -1952,8 +2070,9 @@ mod tests { .expect("this is the second nested engine"); ensure!( matches!( - nested_summary.execution_status, - ExecutionStatus::Completed { .. }, + &nested_summary.execution_status, + ExecutionStatus::Terminal(info) + if info.kind == TerminalKind::Completed ), "for this engine, the ExecutionStatus must be succeeded" ); diff --git a/wicket/src/state/inventory.rs b/wicket/src/state/inventory.rs index 02019898e8..3a561167b1 100644 --- a/wicket/src/state/inventory.rs +++ b/wicket/src/state/inventory.rs @@ -6,7 +6,6 @@ use anyhow::anyhow; use once_cell::sync::Lazy; -use ratatui::text::Text; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fmt::Display; @@ -185,7 +184,7 @@ impl Component { } } -// The component type and its slot. +/// The component type and its slot. #[derive( Debug, Clone, @@ -205,27 +204,24 @@ pub enum ComponentId { } impl ComponentId { - pub fn name(&self) -> String { - self.to_string() + pub fn to_string_uppercase(&self) -> String { + let mut s = self.to_string(); + s.make_ascii_uppercase(); + s } } +/// Prints the component type in standard case. impl Display for ComponentId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - ComponentId::Sled(i) => write!(f, "SLED {}", i), - ComponentId::Switch(i) => write!(f, "SWITCH {}", i), + ComponentId::Sled(i) => write!(f, "sled {}", i), + ComponentId::Switch(i) => write!(f, "switch {}", i), ComponentId::Psc(i) => write!(f, "PSC {}", i), } } } -impl From for Text<'_> { - fn from(value: ComponentId) -> Self { - value.to_string().into() - } -} - pub struct ParsableComponentId<'a> { pub sp_type: &'a str, pub i: &'a str, @@ -269,3 +265,15 @@ impl PowerState { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn component_id_display() { + assert_eq!(ComponentId::Sled(0).to_string(), "sled 0"); + assert_eq!(ComponentId::Switch(1).to_string(), "switch 1"); + assert_eq!(ComponentId::Psc(2).to_string(), "PSC 2"); + } +} diff --git a/wicket/src/ui/panes/overview.rs b/wicket/src/ui/panes/overview.rs index b7a04c055d..7de0171e41 100644 --- a/wicket/src/ui/panes/overview.rs +++ b/wicket/src/ui/panes/overview.rs @@ -343,7 +343,7 @@ impl Control for InventoryView { let title_bar = Paragraph::new(Line::from(vec![ Span::styled("OXIDE RACK / ", border_style), Span::styled( - state.rack_state.selected.to_string(), + state.rack_state.selected.to_string_uppercase(), component_style, ), ])) diff --git a/wicket/src/ui/panes/update.rs b/wicket/src/ui/panes/update.rs index da6f10cf88..2819b3ddda 100644 --- a/wicket/src/ui/panes/update.rs +++ b/wicket/src/ui/panes/update.rs @@ -29,7 +29,8 @@ use ratatui::widgets::{ use slog::{info, o, Logger}; use tui_tree_widget::{Tree, TreeItem, TreeState}; use update_engine::{ - AbortReason, ExecutionStatus, FailureReason, StepKey, WillNotBeRunReason, + AbortReason, ExecutionStatus, FailureReason, StepKey, TerminalKind, + WillNotBeRunReason, }; use wicket_common::update_events::{ EventBuffer, EventReport, ProgressEvent, StepOutcome, StepStatus, @@ -180,7 +181,7 @@ impl UpdatePane { tree_state, items: ALL_COMPONENT_IDS .iter() - .map(|id| TreeItem::new(*id, vec![])) + .map(|id| TreeItem::new(id.to_string_uppercase(), vec![])) .collect(), help: vec![ ("Expand", ""), @@ -531,7 +532,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE: {}", state.rack_state.selected), + format!( + "START UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -561,7 +565,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE: {}", state.rack_state.selected), + format!( + "START UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -594,7 +601,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("START UPDATE FAILED: {}", state.rack_state.selected), + format!( + "START UPDATE FAILED: {}", + state.rack_state.selected.to_string_uppercase() + ), style::failed_update(), )]), body, @@ -635,7 +645,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE: {}", state.rack_state.selected), + format!( + "ABORT UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body, @@ -662,7 +675,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE: {}", state.rack_state.selected), + format!( + "ABORT UPDATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -695,7 +711,10 @@ impl UpdatePane { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("ABORT UPDATE FAILED: {}", state.rack_state.selected), + format!( + "ABORT UPDATE FAILED: {}", + state.rack_state.selected.to_string_uppercase() + ), style::failed_update(), )]), body, @@ -721,7 +740,10 @@ impl UpdatePane { ) { let popup_builder = PopupBuilder { header: Line::from(vec![Span::styled( - format!("CLEAR UPDATE STATE: {}", state.rack_state.selected), + format!( + "CLEAR UPDATE STATE: {}", + state.rack_state.selected.to_string_uppercase() + ), style::header(true), )]), body: Text::from(vec![Line::from(vec![Span::styled( @@ -756,7 +778,7 @@ impl UpdatePane { header: Line::from(vec![Span::styled( format!( "CLEAR UPDATE STATE FAILED: {}", - state.rack_state.selected + state.rack_state.selected.to_string_uppercase() ), style::failed_update(), )]), @@ -830,7 +852,7 @@ impl UpdatePane { }) }) .collect(); - TreeItem::new(*id, children) + TreeItem::new(id.to_string_uppercase(), children) }) .collect(); } @@ -988,9 +1010,7 @@ impl UpdatePane { } ExecutionStatus::NotStarted - | ExecutionStatus::Completed { .. } - | ExecutionStatus::Failed { .. } - | ExecutionStatus::Aborted { .. } => None, + | ExecutionStatus::Terminal(_) => None, } } else { None @@ -1020,9 +1040,7 @@ impl UpdatePane { associated with it", ); match summary.execution_status { - ExecutionStatus::Completed { .. } - | ExecutionStatus::Failed { .. } - | ExecutionStatus::Aborted { .. } => { + ExecutionStatus::Terminal(_) => { // If execution has reached a terminal // state, we can clear it. self.popup = @@ -1107,7 +1125,11 @@ impl UpdatePane { // `overview` pane. let command = self.ignition.selected_command(); let selected = state.rack_state.selected; - info!(self.log, "Sending {command:?} to {selected}"); + info!( + self.log, + "Sending {command:?} to {}", + selected.to_string_uppercase() + ); self.popup = None; Some(Action::Ignition(selected, command)) } @@ -1378,7 +1400,10 @@ impl UpdatePane { // Draw the title/tab bar let title_bar = Paragraph::new(Line::from(vec![ Span::styled("UPDATE STATUS / ", border_style), - Span::styled(state.rack_state.selected.to_string(), header_style), + Span::styled( + state.rack_state.selected.to_string_uppercase(), + header_style, + ), ])) .block(block.clone()); frame.render_widget(title_bar, self.title_rect); @@ -1860,7 +1885,7 @@ impl ComponentUpdateListState { "root execution ID should have a summary associated with it", ); - match summary.execution_status { + match &summary.execution_status { ExecutionStatus::NotStarted => { status_text.push(Span::styled( "Update not started", @@ -1885,47 +1910,63 @@ impl ComponentUpdateListState { )); Some(ComponentUpdateShowHelp::Running) } - ExecutionStatus::Completed { .. } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "completed", - style::successful_update_bold(), - )); - Some(ComponentUpdateShowHelp::Completed) - } - ExecutionStatus::Failed { step_key } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "failed", - style::failed_update_bold(), - )); - status_text.push(Span::styled( - format!( - " at step {}/{}", - step_key.index + 1, - summary.total_steps, - ), - style::plain_text(), - )); - Some(ComponentUpdateShowHelp::Completed) - } - ExecutionStatus::Aborted { step_key } => { - status_text - .push(Span::styled("Update ", style::plain_text())); - status_text.push(Span::styled( - "aborted", - style::failed_update_bold(), - )); - status_text.push(Span::styled( - format!( - " at step {}/{}", - step_key.index + 1, - summary.total_steps, - ), - style::plain_text(), - )); + ExecutionStatus::Terminal(info) => { + match info.kind { + TerminalKind::Completed => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "completed", + style::successful_update_bold(), + )); + } + TerminalKind::Failed => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "failed", + style::failed_update_bold(), + )); + status_text.push(Span::styled( + format!( + " at step {}/{}", + info.step_key.index + 1, + summary.total_steps, + ), + style::plain_text(), + )); + } + TerminalKind::Aborted => { + status_text.push(Span::styled( + "Update ", + style::plain_text(), + )); + status_text.push(Span::styled( + "aborted", + style::failed_update_bold(), + )); + status_text.push(Span::styled( + format!( + " at step {}/{}", + info.step_key.index + 1, + summary.total_steps, + ), + style::plain_text(), + )); + } + } + + if let Some(total_elapsed) = info.root_total_elapsed { + status_text.push(Span::styled( + format!(" after {:.2?}", total_elapsed), + style::plain_text(), + )); + } + Some(ComponentUpdateShowHelp::Completed) } } diff --git a/wicket/src/ui/widgets/ignition.rs b/wicket/src/ui/widgets/ignition.rs index af0818e52a..cef942d2c7 100644 --- a/wicket/src/ui/widgets/ignition.rs +++ b/wicket/src/ui/widgets/ignition.rs @@ -58,7 +58,7 @@ impl IgnitionPopup { ) -> PopupBuilder<'static> { PopupBuilder { header: Line::from(vec![Span::styled( - format!("IGNITION: {}", component), + format!("IGNITION: {}", component.to_string_uppercase()), style::header(true), )]), body: Text {