diff --git a/Cargo.lock b/Cargo.lock index ce31b18823..75d5bef792 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1397,6 +1397,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -2020,9 +2041,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] name = "ecdsa" @@ -4180,6 +4201,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +dependencies = [ + "serde", +] + [[package]] name = "nanorand" version = "0.7.0" @@ -5080,9 +5110,12 @@ dependencies = [ "async-bb8-diesel", "chrono", "clap 4.4.3", + "crossterm", "crucible-agent-client", + "csv", "diesel", "dropshot", + "dyn-clone", "expectorate", "futures", "gateway-client", @@ -5091,6 +5124,7 @@ dependencies = [ "humantime", "internal-dns", "ipnetwork", + "multimap", "nexus-client", "nexus-db-model", "nexus-db-queries", @@ -5104,6 +5138,7 @@ dependencies = [ "omicron-workspace-hack", "oximeter-client", "pq-sys", + "ratatui", "regex", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index b8379adbd1..aee7161db5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,6 +183,7 @@ crossterm = { version = "0.27.0", features = ["event-stream"] } crucible-agent-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } crucible-pantry-client = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } crucible-smf = { git = "https://github.com/oxidecomputer/crucible", rev = "2d4bc11232d53f177c286383926fa5f8c1b2a938" } +csv = "1.3.0" curve25519-dalek = "4" datatest-stable = "0.2.3" display-error-chain = "0.2.0" @@ -197,6 +198,7 @@ dns-server = { path = "dns-server" } dns-service-client = { path = "clients/dns-service-client" } dpd-client = { path = "clients/dpd-client" } dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } +dyn-clone = "1.0.16" either = "1.9.0" expectorate = "1.1.0" fatfs = "0.3.6" @@ -248,6 +250,7 @@ mime_guess = "2.0.4" mockall = "0.12" newtype_derive = "0.1.6" mg-admin-client = { path = "clients/mg-admin-client" } +multimap = "0.8.1" nexus-blueprint-execution = { path = "nexus/blueprint-execution" } nexus-client = { path = "clients/nexus-client" } nexus-db-model = { path = "nexus/db-model" } diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index e08d5f9477..3f566f55ee 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -12,9 +12,12 @@ anyhow.workspace = true async-bb8-diesel.workspace = true chrono.workspace = true clap.workspace = true +crossterm.workspace = true crucible-agent-client.workspace = true +csv.workspace = true diesel.workspace = true dropshot.workspace = true +dyn-clone.workspace = true futures.workspace = true gateway-client.workspace = true gateway-messages.workspace = true @@ -29,6 +32,7 @@ omicron-common.workspace = true oximeter-client.workspace = true # See omicron-rpaths for more about the "pq-sys" dependency. pq-sys = "*" +ratatui.workspace = true serde.workspace = true serde_json.workspace = true sled-agent-client.workspace = true @@ -43,6 +47,7 @@ uuid.workspace = true ipnetwork.workspace = true omicron-workspace-hack.workspace = true nexus-test-utils.workspace = true +multimap.workspace = true [dev-dependencies] expectorate.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/mgs.rs b/dev-tools/omdb/src/bin/omdb/mgs.rs index 770cba9f62..ece4c4f109 100644 --- a/dev-tools/omdb/src/bin/omdb/mgs.rs +++ b/dev-tools/omdb/src/bin/omdb/mgs.rs @@ -22,6 +22,12 @@ use gateway_client::types::SpState; use gateway_client::types::SpType; use tabled::Tabled; +mod dashboard; +mod sensors; + +use dashboard::DashboardArgs; +use sensors::SensorsArgs; + /// Arguments to the "omdb mgs" subcommand #[derive(Debug, Args)] pub struct MgsArgs { @@ -35,19 +41,25 @@ pub struct MgsArgs { #[derive(Debug, Subcommand)] enum MgsCommands { + /// Dashboard of SPs + Dashboard(DashboardArgs), + /// Show information about devices and components visible to MGS Inventory(InventoryArgs), + + /// Show information about sensors, as gleaned by MGS + Sensors(SensorsArgs), } #[derive(Debug, Args)] struct InventoryArgs {} impl MgsArgs { - pub(crate) async fn run_cmd( + async fn mgs_client( &self, omdb: &Omdb, log: &slog::Logger, - ) -> Result<(), anyhow::Error> { + ) -> Result { let mgs_url = match &self.mgs_url { Some(cli_or_env_url) => cli_or_env_url.clone(), None => { @@ -68,11 +80,24 @@ impl MgsArgs { } }; eprintln!("note: using MGS URL {}", &mgs_url); - let mgs_client = gateway_client::Client::new(&mgs_url, log.clone()); + Ok(gateway_client::Client::new(&mgs_url, log.clone())) + } + pub(crate) async fn run_cmd( + &self, + omdb: &Omdb, + log: &slog::Logger, + ) -> Result<(), anyhow::Error> { match &self.command { - MgsCommands::Inventory(inventory_args) => { - cmd_mgs_inventory(&mgs_client, inventory_args).await + MgsCommands::Dashboard(args) => { + dashboard::cmd_mgs_dashboard(omdb, log, self, args).await + } + MgsCommands::Inventory(args) => { + let mgs_client = self.mgs_client(omdb, log).await?; + cmd_mgs_inventory(&mgs_client, args).await + } + MgsCommands::Sensors(args) => { + sensors::cmd_mgs_sensors(omdb, log, self, args).await } } } @@ -156,6 +181,10 @@ fn sp_type_to_str(s: &SpType) -> &'static str { } } +fn sp_to_string(s: &SpIdentifier) -> String { + format!("{} {}", sp_type_to_str(&s.type_), s.slot) +} + fn show_sp_ids(sp_ids: &[SpIdentifier]) -> Result<(), anyhow::Error> { #[derive(Tabled)] #[tabled(rename_all = "SCREAMING_SNAKE_CASE")] diff --git a/dev-tools/omdb/src/bin/omdb/mgs/dashboard.rs b/dev-tools/omdb/src/bin/omdb/mgs/dashboard.rs new file mode 100644 index 0000000000..20d651bfdf --- /dev/null +++ b/dev-tools/omdb/src/bin/omdb/mgs/dashboard.rs @@ -0,0 +1,1113 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Code for the MGS dashboard subcommand + +use anyhow::{Context, Result}; +use chrono::{Local, Offset, TimeZone}; +use crossterm::{ + event::{ + self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode, + KeyModifiers, + }, + execute, + terminal::{ + disable_raw_mode, enable_raw_mode, EnterAlternateScreen, + LeaveAlternateScreen, + }, +}; +use dyn_clone::DynClone; +use ratatui::{ + backend::{Backend, CrosstermBackend}, + layout::{Alignment, Constraint, Direction, Layout, Rect}, + style::{Color, Modifier, Style}, + symbols, + text::{Line, Span}, + widgets::{ + Axis, Block, Borders, Chart, Dataset, List, ListItem, ListState, + Paragraph, + }, + Frame, Terminal, +}; + +use crate::mgs::sensors::{ + sensor_data, sensor_metadata, SensorId, SensorInput, SensorMetadata, + SensorValues, SensorsArgs, +}; +use crate::mgs::sp_to_string; +use clap::Args; +use gateway_client::types::MeasurementKind; +use gateway_client::types::SpIdentifier; +use multimap::MultiMap; +use std::collections::HashMap; +use std::fs::File; +use std::io; +use std::time::{Duration, Instant, SystemTime}; + +#[derive(Debug, Args)] +pub(crate) struct DashboardArgs { + #[clap(flatten)] + sensors_args: SensorsArgs, + + /// simulate real-time with input + #[clap(long)] + simulate_realtime: bool, +} + +struct StatefulList { + state: ListState, + n: usize, +} + +impl StatefulList { + fn next(&mut self) { + self.state.select(match self.state.selected() { + Some(ndx) => Some((ndx + 1) % self.n), + None => Some(0), + }); + } + + fn previous(&mut self) { + self.state.select(match self.state.selected() { + Some(0) => Some(self.n - 1), + Some(ndx) => Some(ndx - 1), + None => Some(0), + }); + } + + fn unselect(&mut self) { + self.state.select(None); + } + + fn selected(&self) -> Option { + self.state.selected() + } +} + +struct Series { + name: String, + color: Color, + data: Vec<(f64, f64)>, + raw: Vec>, +} + +trait Attributes: DynClone { + fn label(&self) -> String; + fn legend_label(&self) -> String; + fn x_axis_label(&self) -> String { + "Time".to_string() + } + fn y_axis_label(&self) -> String; + fn axis_value(&self, val: f64) -> String; + fn legend_value(&self, val: f64) -> String; + + fn increase(&mut self, _ndx: usize) -> Option { + None + } + + fn decrease(&mut self, _ndx: usize) -> Option { + None + } + + fn clear(&mut self) {} +} + +dyn_clone::clone_trait_object!(Attributes); + +#[derive(Clone)] +struct TempGraph; + +impl Attributes for TempGraph { + fn label(&self) -> String { + "Temperature".to_string() + } + fn legend_label(&self) -> String { + "Sensors".to_string() + } + + fn y_axis_label(&self) -> String { + "Degrees Celsius".to_string() + } + + fn axis_value(&self, val: f64) -> String { + format!("{:2.0}°", val) + } + + fn legend_value(&self, val: f64) -> String { + format!("{:4.2}°", val) + } +} + +#[derive(Clone)] +struct FanGraph; + +impl Attributes for FanGraph { + fn label(&self) -> String { + "Fan speed".to_string() + } + fn legend_label(&self) -> String { + "Fans".to_string() + } + + fn y_axis_label(&self) -> String { + "RPM".to_string() + } + + fn axis_value(&self, val: f64) -> String { + format!("{:3.1}K", val / 1000.0) + } + + fn legend_value(&self, val: f64) -> String { + format!("{:.0}", val) + } +} + +#[derive(Clone)] +struct CurrentGraph; + +impl Attributes for CurrentGraph { + fn label(&self) -> String { + "Output current".to_string() + } + + fn legend_label(&self) -> String { + "Regulators".to_string() + } + + fn y_axis_label(&self) -> String { + "Rails".to_string() + } + + fn axis_value(&self, val: f64) -> String { + format!("{:2.2}A", val) + } + + fn legend_value(&self, val: f64) -> String { + format!("{:3.2}A", val) + } +} + +#[derive(Clone)] +struct VoltageGraph; + +impl Attributes for VoltageGraph { + fn label(&self) -> String { + "Voltage".to_string() + } + + fn legend_label(&self) -> String { + "Rails".to_string() + } + + fn y_axis_label(&self) -> String { + "Volts".to_string() + } + + fn axis_value(&self, val: f64) -> String { + format!("{:2.2}V", val) + } + + fn legend_value(&self, val: f64) -> String { + format!("{:3.2}V", val) + } +} + +#[derive(Clone)] +struct SensorGraph; + +impl Attributes for SensorGraph { + fn label(&self) -> String { + "Sensor output".to_string() + } + + fn legend_label(&self) -> String { + "Sensors".to_string() + } + + fn y_axis_label(&self) -> String { + "Units".to_string() + } + + fn axis_value(&self, val: f64) -> String { + format!("{:2.2}", val) + } + + fn legend_value(&self, val: f64) -> String { + format!("{:3.2}", val) + } +} + +struct Graph { + series: Vec, + legend: StatefulList, + time: usize, + width: usize, + offs: usize, + interpolate: usize, + bounds: [f64; 2], + attributes: Box, +} + +impl Graph { + fn new(all: &[String], attr: Box) -> Result { + let mut series = vec![]; + + let colors = [ + Color::Yellow, + Color::Green, + Color::Magenta, + Color::White, + Color::Red, + Color::LightRed, + Color::Blue, + Color::LightMagenta, + Color::LightYellow, + Color::LightCyan, + Color::LightGreen, + Color::LightBlue, + Color::LightRed, + ]; + + for (ndx, s) in all.iter().enumerate() { + series.push(Series { + name: s.to_string(), + color: colors[ndx % colors.len()], + data: Vec::new(), + raw: Vec::new(), + }) + } + + Ok(Graph { + series, + legend: StatefulList { state: ListState::default(), n: all.len() }, + time: 0, + width: 600, + offs: 0, + interpolate: 0, + bounds: [20.0, 120.0], + attributes: attr, + }) + } + + fn flip(from: &[(&Self, String)], series_ndx: usize) -> Self { + let rep = from[0].0; + let mut series = vec![]; + + let colors = [ + Color::Yellow, + Color::Green, + Color::Magenta, + Color::White, + Color::Red, + Color::LightRed, + Color::Blue, + Color::LightMagenta, + Color::LightYellow, + Color::LightCyan, + Color::LightGreen, + Color::LightBlue, + Color::LightRed, + ]; + + for (ndx, (graph, name)) in from.iter().enumerate() { + series.push(Series { + name: name.clone(), + color: colors[ndx % colors.len()], + data: graph.series[series_ndx].data.clone(), + raw: graph.series[series_ndx].raw.clone(), + }); + } + + Graph { + series, + legend: StatefulList { state: ListState::default(), n: from.len() }, + time: rep.time, + width: rep.width, + offs: rep.offs, + interpolate: rep.interpolate, + bounds: rep.bounds, + attributes: rep.attributes.clone(), + } + } + + fn data(&mut self, data: &[Option]) { + for (ndx, s) in self.series.iter_mut().enumerate() { + s.raw.push(data[ndx]); + } + + self.time += 1; + + if self.offs > 0 { + self.offs += 1; + } + } + + fn update_data(&mut self) { + for s in &mut self.series { + s.data = Vec::new(); + } + + for i in 0..self.width { + if self.time < (self.width - i) + self.offs { + continue; + } + + let offs = self.time - (self.width - i) - self.offs; + + for (_ndx, s) in &mut self.series.iter_mut().enumerate() { + if let Some(datum) = s.raw[offs] { + let point = (i as f64, datum as f64); + + if self.interpolate != 0 { + if let Some(last) = s.data.last() { + let x_delta = point.0 - last.0; + let slope = (point.1 - last.1) / x_delta; + let x_inc = x_delta / self.interpolate as f64; + + for x in 0..self.interpolate { + s.data.push(( + point.0 + x as f64 * x_inc, + point.1 + (slope * x_inc), + )); + } + } + } + + s.data.push((i as f64, datum as f64)); + } + } + } + + self.update_bounds(); + } + + fn update_bounds(&mut self) { + let selected = self.legend.state.selected(); + let mut min = None; + let mut max = None; + + for (ndx, s) in self.series.iter().enumerate() { + if let Some(selected) = selected { + if ndx != selected { + continue; + } + } + + for (_, datum) in &s.data { + min = match min { + Some(min) if datum < min => Some(datum), + None => Some(datum), + _ => min, + }; + + max = match max { + Some(max) if datum > max => Some(datum), + None => Some(datum), + _ => max, + }; + } + } + + if let Some(min) = min { + self.bounds[0] = ((min * 0.85) / 2.0) * 2.0; + } + + if self.bounds[0] < 0.0 { + self.bounds[0] = 0.0; + } + + if let Some(max) = max { + self.bounds[1] = ((max * 1.15) / 2.0) * 2.0; + } + } + + fn previous(&mut self) { + self.legend.previous(); + } + + fn next(&mut self) { + self.legend.next(); + } + + fn unselect(&mut self) { + self.legend.unselect(); + } + + fn selected(&self) -> Option { + self.legend.selected() + } + + fn set_interpolate(&mut self) { + let interpolate = (1000.0 - self.width as f64) / self.width as f64; + + if interpolate >= 1.0 { + self.interpolate = interpolate as usize; + } else { + self.interpolate = 0; + } + } + + fn zoom_in(&mut self) { + self.width = (self.width as f64 * 0.8) as usize; + self.set_interpolate(); + } + + fn zoom_out(&mut self) { + self.width = (self.width as f64 * 1.25) as usize; + self.set_interpolate(); + } + + fn time_right(&mut self) { + let delta = (self.width as f64 * 0.25) as usize; + + if delta > self.offs { + self.offs = 0; + } else { + self.offs -= delta; + } + } + + fn time_left(&mut self) { + self.offs += (self.width as f64 * 0.25) as usize; + } +} + +struct Dashboard { + graphs: HashMap<(SpIdentifier, MeasurementKind), Graph>, + flipped: HashMap, + sids: HashMap<(SpIdentifier, MeasurementKind), Vec>, + kinds: Vec, + selected_kind: usize, + sps: Vec, + selected_sp: usize, + status: String, + time: u64, +} + +impl Dashboard { + fn new(metadata: &SensorMetadata) -> Result { + let mut sps = + metadata.sensors_by_sp.keys().copied().collect::>(); + let mut graphs = HashMap::new(); + let mut sids = HashMap::new(); + sps.sort(); + + let kinds = vec![ + MeasurementKind::Temperature, + MeasurementKind::Speed, + MeasurementKind::Current, + ]; + + for &sp in sps.iter() { + let sensors = metadata.sensors_by_sp.get_vec(&sp).unwrap(); + let mut by_kind = MultiMap::new(); + + for sid in sensors { + let (_, s, _) = metadata.sensors_by_id.get(sid).unwrap(); + by_kind.insert(s.kind, (s.name.clone(), *sid)); + } + + let keys = by_kind.keys().copied().collect::>(); + + for k in keys { + let mut v = by_kind.remove(&k).unwrap(); + v.sort(); + + let labels = + v.iter().map(|(n, _)| n.clone()).collect::>(); + + graphs.insert( + (sp, k), + Graph::new( + labels.as_slice(), + match k { + MeasurementKind::Temperature => Box::new(TempGraph), + MeasurementKind::Current => Box::new(CurrentGraph), + MeasurementKind::Speed => Box::new(FanGraph), + MeasurementKind::Voltage => Box::new(VoltageGraph), + _ => Box::new(SensorGraph), + }, + )?, + ); + + sids.insert( + (sp, k), + v.iter().map(|(_, sid)| *sid).collect::>(), + ); + } + } + + let status = sp_to_string(&sps[0]); + + Ok(Dashboard { + graphs, + flipped: HashMap::new(), + sids, + kinds, + selected_kind: 0, + sps, + selected_sp: 0, + status, + time: secs()?, + }) + } + + fn status(&self) -> Vec<(&str, &str)> { + vec![("Status", &self.status)] + } + + fn update_data(&mut self) { + for graph in self.graphs.values_mut() { + graph.update_data(); + } + + for graph in self.flipped.values_mut() { + graph.update_data(); + } + } + + fn up(&mut self) { + let selected_kind = self.kinds[self.selected_kind]; + let type_ = self.sps[self.selected_sp].type_; + + if let Some(flipped) = self.flipped.get_mut(&selected_kind) { + flipped.previous(); + return; + } + + for sp in self.sps.iter().filter(|&s| s.type_ == type_) { + self.graphs.get_mut(&(*sp, selected_kind)).unwrap().previous(); + } + } + + fn down(&mut self) { + let selected_kind = self.kinds[self.selected_kind]; + let type_ = self.sps[self.selected_sp].type_; + + if let Some(flipped) = self.flipped.get_mut(&selected_kind) { + flipped.next(); + return; + } + + for sp in self.sps.iter().filter(|&s| s.type_ == type_) { + self.graphs.get_mut(&(*sp, selected_kind)).unwrap().next(); + } + } + + fn esc(&mut self) { + let selected_kind = self.kinds[self.selected_kind]; + let type_ = self.sps[self.selected_sp].type_; + + if let Some(flipped) = self.flipped.get_mut(&selected_kind) { + flipped.unselect(); + return; + } + + for sp in self.sps.iter().filter(|&s| s.type_ == type_) { + self.graphs.get_mut(&(*sp, selected_kind)).unwrap().unselect(); + } + } + + fn left(&mut self) { + if self.selected_sp == 0 { + self.selected_sp = self.sps.len() - 1; + } else { + self.selected_sp -= 1; + } + + self.status = sp_to_string(&self.sps[self.selected_sp]); + } + + fn right(&mut self) { + self.selected_sp = (self.selected_sp + 1) % self.sps.len(); + self.status = sp_to_string(&self.sps[self.selected_sp]); + } + + fn time_left(&mut self) { + for graph in self.graphs.values_mut() { + graph.time_left(); + } + + for graph in self.flipped.values_mut() { + graph.time_left(); + } + } + + fn time_right(&mut self) { + for graph in self.graphs.values_mut() { + graph.time_right(); + } + + for graph in self.flipped.values_mut() { + graph.time_right(); + } + } + + fn flip(&mut self) { + let selected_kind = self.kinds[self.selected_kind]; + let type_ = self.sps[self.selected_sp].type_; + + if self.flipped.remove(&selected_kind).is_some() { + return; + } + + let sp = self.sps[self.selected_sp]; + + let graph = self.graphs.get(&(sp, selected_kind)).unwrap(); + + if let Some(ndx) = graph.selected() { + let mut from = vec![]; + + for sp in self.sps.iter().filter(|&s| s.type_ == type_) { + from.push(( + self.graphs.get(&(*sp, selected_kind)).unwrap(), + sp_to_string(sp), + )); + } + + self.flipped + .insert(selected_kind, Graph::flip(from.as_slice(), ndx)); + } + } + + fn tab(&mut self) { + self.selected_kind = (self.selected_kind + 1) % self.kinds.len(); + } + + fn zoom_in(&mut self) { + for graph in self.graphs.values_mut() { + graph.zoom_in(); + } + + for graph in self.flipped.values_mut() { + graph.zoom_in(); + } + } + + fn zoom_out(&mut self) { + for graph in self.graphs.values_mut() { + graph.zoom_out(); + } + + for graph in self.flipped.values_mut() { + graph.zoom_out(); + } + } + + fn gap(&mut self, length: u64) { + let mut gap: Vec> = vec![]; + + for (graph, sids) in &self.sids { + while gap.len() < sids.len() { + gap.push(None); + } + + let graph = self.graphs.get_mut(graph).unwrap(); + + for _ in 0..length { + graph.data(&gap[0..sids.len()]); + } + } + } + + fn values(&mut self, values: &SensorValues) { + for (graph, sids) in &self.sids { + let mut data = vec![]; + + for sid in sids { + if let Some(value) = values.values.get(sid) { + data.push(*value); + } else { + data.push(None); + } + } + + let graph = self.graphs.get_mut(graph).unwrap(); + graph.data(data.as_slice()); + } + + self.time = values.time; + } +} + +fn run_dashboard( + terminal: &mut Terminal, + dashboard: &mut Dashboard, + force_update: bool, +) -> Result { + let update = if crossterm::event::poll(Duration::from_secs(0))? { + if let Event::Key(key) = event::read()? { + match key.code { + KeyCode::Char('q') => return Ok(true), + KeyCode::Char('+') => dashboard.zoom_in(), + KeyCode::Char('-') => dashboard.zoom_out(), + KeyCode::Char('<') => dashboard.time_left(), + KeyCode::Char('>') => dashboard.time_right(), + KeyCode::Char('!') => dashboard.flip(), + KeyCode::Char('l') => { + // + // ^L -- form feed -- is historically used to clear and + // redraw the screen. And, notably, it is what dtach(1) + // will send when attaching to a dashboard. If we + // see ^L, clear the terminal to force a total redraw. + // + if key.modifiers == KeyModifiers::CONTROL { + terminal.clear()?; + } + } + KeyCode::Up => dashboard.up(), + KeyCode::Down => dashboard.down(), + KeyCode::Right => dashboard.right(), + KeyCode::Left => dashboard.left(), + KeyCode::Esc => dashboard.esc(), + KeyCode::Tab => dashboard.tab(), + _ => {} + } + } + true + } else { + force_update + }; + + if update { + dashboard.update_data(); + terminal.draw(|f| draw(f, dashboard))?; + } + + Ok(false) +} + +fn secs() -> Result { + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; + Ok(now.as_secs()) +} + +/// +/// Runs `omdb mgs dashboard` +/// +pub(crate) async fn cmd_mgs_dashboard( + omdb: &crate::Omdb, + log: &slog::Logger, + mgs_args: &crate::mgs::MgsArgs, + args: &DashboardArgs, +) -> Result<(), anyhow::Error> { + let mut input = if let Some(ref input) = args.sensors_args.input { + let file = File::open(input) + .with_context(|| format!("failed to open {input}"))?; + SensorInput::CsvReader( + csv::Reader::from_reader(file), + csv::Position::new(), + ) + } else { + SensorInput::MgsClient(mgs_args.mgs_client(omdb, log).await?) + }; + + let (metadata, values) = + sensor_metadata(&mut input, &args.sensors_args).await?; + + let mut dashboard = Dashboard::new(&metadata)?; + let mut last = values.time; + let mut force = true; + let mut update = true; + + dashboard.values(&values); + + if args.sensors_args.input.is_some() && !args.simulate_realtime { + loop { + let values = sensor_data(&mut input, &metadata).await?; + + if values.time == 0 { + break; + } + + if values.time != last + 1 { + dashboard.gap(values.time - last - 1); + } + + last = values.time; + dashboard.values(&values); + } + + update = false; + } + + // setup terminal + enable_raw_mode()?; + let mut stdout = io::stdout(); + execute!(stdout, EnterAlternateScreen, EnableMouseCapture)?; + let backend = CrosstermBackend::new(stdout); + let mut terminal = Terminal::new(backend)?; + + let res = 'outer: loop { + match run_dashboard(&mut terminal, &mut dashboard, force) { + Err(err) => break Err(err), + Ok(true) => break Ok(()), + _ => {} + } + + force = false; + + let now = match secs() { + Err(err) => break Err(err), + Ok(now) => now, + }; + + if update && now != last { + let kicked = Instant::now(); + let f = sensor_data(&mut input, &metadata); + last = now; + + while Instant::now().duration_since(kicked).as_millis() < 800 { + tokio::time::sleep(Duration::from_millis(10)).await; + + match run_dashboard(&mut terminal, &mut dashboard, force) { + Err(err) => break 'outer Err(err), + Ok(true) => break 'outer Ok(()), + _ => {} + } + } + + let values = match f.await { + Err(err) => break Err(err), + Ok(v) => v, + }; + + dashboard.values(&values); + force = true; + continue; + } + + tokio::time::sleep(Duration::from_millis(10)).await; + }; + + // restore terminal + disable_raw_mode()?; + execute!( + terminal.backend_mut(), + LeaveAlternateScreen, + DisableMouseCapture + )?; + terminal.show_cursor()?; + + if let Err(err) = res { + println!("{err:?}"); + } + + Ok(()) +} + +fn draw_graph(f: &mut Frame, parent: Rect, graph: &mut Graph, now: u64) { + // + // We want the right panel to be 31 characters wide (a left-justified 20 + // and a right justified 8 + margins), but we don't want it to consume + // more than 80%; calculate accordingly. + // + let r = std::cmp::min((31 * 100) / parent.width, 80); + + let chunks = Layout::default() + .direction(Direction::Horizontal) + .constraints( + [Constraint::Percentage(100 - r), Constraint::Percentage(r)] + .as_ref(), + ) + .split(parent); + + let latest = now as i64 - graph.offs as i64; + let earliest = Local.timestamp_opt(latest - graph.width as i64, 0).unwrap(); + let latest = Local.timestamp_opt(latest, 0).unwrap(); + + // + // We want a format that preserves horizontal real estate just a tad more + // than .to_rfc3339_opts()... + // + let fmt = "%Y-%m-%d %H:%M:%S"; + + let tz_offset = earliest.offset().fix().local_minus_utc(); + let tz = if tz_offset != 0 { + let hours = tz_offset / 3600; + let minutes = (tz_offset % 3600) / 60; + + if minutes != 0 { + format!("Z{:+}:{:02}", hours, minutes.abs()) + } else { + format!("Z{:+}", hours) + } + } else { + "Z".to_string() + }; + + let x_labels = vec![ + Span::styled( + format!("{}{}", earliest.format(fmt), tz), + Style::default().add_modifier(Modifier::BOLD), + ), + Span::styled( + format!("{}{}", latest.format(fmt), tz), + Style::default().add_modifier(Modifier::BOLD), + ), + ]; + + let mut datasets = vec![]; + let selected = graph.legend.state.selected(); + + for (ndx, s) in graph.series.iter().enumerate() { + if let Some(selected) = selected { + if ndx != selected { + continue; + } + } + + datasets.push( + Dataset::default() + .name(&s.name) + .marker(symbols::Marker::Braille) + .style(Style::default().fg(s.color)) + .data(&s.data), + ); + } + + let chart = Chart::new(datasets) + .block( + Block::default() + .title(Span::styled( + graph.attributes.label(), + Style::default() + .fg(Color::Cyan) + .add_modifier(Modifier::BOLD), + )) + .borders(Borders::ALL), + ) + .x_axis( + Axis::default() + .title(graph.attributes.x_axis_label()) + .style(Style::default().fg(Color::Gray)) + .labels(x_labels) + .bounds([0.0, graph.width as f64]) + .labels_alignment(Alignment::Right), + ) + .y_axis( + Axis::default() + .title(graph.attributes.y_axis_label()) + .style(Style::default().fg(Color::Gray)) + .labels(vec![ + Span::styled( + graph.attributes.axis_value(graph.bounds[0]), + Style::default().add_modifier(Modifier::BOLD), + ), + Span::styled( + graph.attributes.axis_value(graph.bounds[1]), + Style::default().add_modifier(Modifier::BOLD), + ), + ]) + .bounds(graph.bounds), + ); + + f.render_widget(chart, chunks[0]); + + let mut rows = vec![]; + + for s in &graph.series { + let val = match s.raw.last() { + None | Some(None) => "-".to_string(), + Some(Some(val)) => graph.attributes.legend_value((*val).into()), + }; + + rows.push(ListItem::new(Line::from(vec![ + Span::styled( + format!("{:<20}", s.name), + Style::default().fg(s.color), + ), + Span::styled(format!("{:>8}", val), Style::default().fg(s.color)), + ]))); + } + + let list = List::new(rows) + .block( + Block::default() + .borders(Borders::ALL) + .title(graph.attributes.legend_label()), + ) + .highlight_style( + Style::default() + .bg(Color::LightGreen) + .fg(Color::Black) + .add_modifier(Modifier::BOLD), + ); + + // We can now render the item list + f.render_stateful_widget(list, chunks[1], &mut graph.legend.state); +} + +fn draw_graphs(f: &mut Frame, parent: Rect, dashboard: &mut Dashboard) { + let screen = Layout::default() + .direction(Direction::Vertical) + .constraints( + [ + Constraint::Ratio(1, 2), + Constraint::Ratio(1, 4), + Constraint::Ratio(1, 4), + ] + .as_ref(), + ) + .split(parent); + + let sp = dashboard.sps[dashboard.selected_sp]; + + for (i, k) in dashboard.kinds.iter().enumerate() { + if let Some(graph) = dashboard.flipped.get_mut(k) { + draw_graph(f, screen[i], graph, dashboard.time); + } else { + draw_graph( + f, + screen[i], + dashboard.graphs.get_mut(&(sp, *k)).unwrap(), + dashboard.time, + ); + } + } +} + +fn draw_status(f: &mut Frame, parent: Rect, status: &[(&str, &str)]) { + let mut bar = vec![]; + + for i in 0..status.len() { + let s = &status[i]; + + bar.push(Span::styled( + s.0, + Style::default().add_modifier(Modifier::BOLD), + )); + + bar.push(Span::styled( + ": ", + Style::default().add_modifier(Modifier::BOLD), + )); + + bar.push(Span::raw(s.1)); + + if i < status.len() - 1 { + bar.push(Span::raw(" | ")); + } + } + + let text = vec![Line::from(bar)]; + + let para = Paragraph::new(text) + .alignment(Alignment::Right) + .style(Style::default().fg(Color::White).bg(Color::Black)); + + f.render_widget(para, parent); +} + +fn draw(f: &mut Frame, dashboard: &mut Dashboard) { + let size = f.size(); + + let screen = Layout::default() + .direction(Direction::Vertical) + .constraints([Constraint::Min(1), Constraint::Length(1)].as_ref()) + .split(size); + + draw_graphs(f, screen[0], dashboard); + draw_status(f, screen[1], &dashboard.status()); +} diff --git a/dev-tools/omdb/src/bin/omdb/mgs/sensors.rs b/dev-tools/omdb/src/bin/omdb/mgs/sensors.rs new file mode 100644 index 0000000000..d00bebd96c --- /dev/null +++ b/dev-tools/omdb/src/bin/omdb/mgs/sensors.rs @@ -0,0 +1,950 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Implementation of the "mgs sensors" subcommand + +use anyhow::{bail, Context}; +use clap::Args; +use gateway_client::types::MeasurementErrorCode; +use gateway_client::types::MeasurementKind; +use gateway_client::types::SpComponentDetails; +use gateway_client::types::SpIdentifier; +use gateway_client::types::SpIgnition; +use gateway_client::types::SpType; +use multimap::MultiMap; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +#[derive(Debug, Args)] +pub(crate) struct SensorsArgs { + /// verbose messages + #[clap(long, short)] + pub verbose: bool, + + /// restrict to specified sled(s) + #[clap(long, use_value_delimiter = true)] + pub sled: Vec, + + /// exclude sleds rather than include them + #[clap(long, short)] + pub exclude: bool, + + /// include switches + #[clap(long)] + pub switches: bool, + + /// include PSC + #[clap(long)] + pub psc: bool, + + /// print sensors every second + #[clap(long, short)] + pub sleep: bool, + + /// parseable output + #[clap(long, short)] + pub parseable: bool, + + /// show latencies + #[clap(long)] + pub show_latencies: bool, + + /// restrict sensors by type of sensor + #[clap( + long, + short, + value_name = "sensor type", + use_value_delimiter = true + )] + pub types: Option>, + + /// restrict sensors by name + #[clap( + long, + short, + value_name = "sensor name", + use_value_delimiter = true + )] + pub named: Option>, + + /// simulate using specified file as input + #[clap(long, short)] + pub input: Option, + + /// start time, if using an input file + #[clap(long, value_name = "time", requires = "input")] + pub start: Option, + + /// end time, if using an input file + #[clap(long, value_name = "time", requires = "input")] + pub end: Option, + + /// duration, if using an input file + #[clap( + long, + value_name = "seconds", + requires = "input", + conflicts_with = "end" + )] + pub duration: Option, +} + +impl SensorsArgs { + fn matches_sp(&self, sp: &SpIdentifier) -> bool { + match sp.type_ { + SpType::Sled => { + let matched = if !self.sled.is_empty() { + self.sled.contains(&sp.slot) + } else { + true + }; + + matched != self.exclude + } + SpType::Switch => self.switches, + SpType::Power => self.psc, + } + } +} + +#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct Sensor { + pub name: String, + pub kind: MeasurementKind, +} + +impl Sensor { + fn units(&self) -> &str { + match self.kind { + MeasurementKind::Temperature => "°C", + MeasurementKind::Current | MeasurementKind::InputCurrent => "A", + MeasurementKind::Voltage | MeasurementKind::InputVoltage => "V", + MeasurementKind::Speed => "RPM", + MeasurementKind::Power => "W", + } + } + + fn format(&self, value: f32, parseable: bool) -> String { + if parseable { + format!("{value}") + } else { + match self.kind { + MeasurementKind::Speed => { + // + // This space is deliberate: other units (°C, V, A) look + // more natural when directly attached to their value -- + // but RPM looks decidedly unnatural without a space. + // + format!("{value:0} RPM") + } + _ => { + format!("{value:.2}{}", self.units()) + } + } + } + } + + fn to_kind_string(&self) -> &str { + match self.kind { + MeasurementKind::Temperature => "temp", + MeasurementKind::Power => "power", + MeasurementKind::Current => "current", + MeasurementKind::Voltage => "voltage", + MeasurementKind::InputCurrent => "input-current", + MeasurementKind::InputVoltage => "input-voltage", + MeasurementKind::Speed => "speed", + } + } + + fn from_string(name: &str, kind: &str) -> Option { + let k = match kind { + "temp" | "temperature" => Some(MeasurementKind::Temperature), + "power" => Some(MeasurementKind::Power), + "current" => Some(MeasurementKind::Current), + "voltage" => Some(MeasurementKind::Voltage), + "input-current" => Some(MeasurementKind::InputCurrent), + "input-voltage" => Some(MeasurementKind::InputVoltage), + "speed" => Some(MeasurementKind::Speed), + _ => None, + }; + + k.map(|kind| Sensor { name: name.to_string(), kind }) + } +} + +pub(crate) enum SensorInput { + MgsClient(gateway_client::Client), + CsvReader(csv::Reader, csv::Position), +} + +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct SensorId(u32); + +#[derive(Debug)] +pub(crate) struct SensorMetadata { + pub sensors_by_sensor: MultiMap, + pub sensors_by_sensor_and_sp: + HashMap>, + pub sensors_by_id: + HashMap, + pub sensors_by_sp: MultiMap, + pub work_by_sp: + HashMap)>>, + #[allow(dead_code)] + pub start_time: Option, + pub end_time: Option, +} + +struct SensorSpInfo { + info: Vec<(SpIdentifier, SpInfo)>, + time: u64, + latencies: Option>, +} + +pub(crate) struct SensorValues { + pub values: HashMap>, + pub latencies: Option>, + pub time: u64, +} + +/// +/// We identify a device as either a physical device (i.e., when connecting +/// to MGS), or as a field in the CSV header (i.e., when processing data +/// postmortem. It's handy to have this as enum to allow most of the code +/// to be agnostic to the underlying source, but callers of ['device'] and +/// ['field'] are expected to know which of these they're dealing with. +/// +#[derive(Clone, Debug, Hash, PartialEq, Eq)] +pub(crate) enum DeviceIdentifier { + Field(usize), + Device(String), +} + +impl DeviceIdentifier { + fn device(&self) -> &String { + match self { + Self::Device(ref device) => device, + _ => panic!(), + } + } + + fn field(&self) -> usize { + match self { + Self::Field(field) => *field, + _ => panic!(), + } + } +} + +struct SpInfo { + devices: MultiMap)>, + timestamps: Vec, +} + +async fn sp_info( + mgs_client: gateway_client::Client, + type_: SpType, + slot: u32, +) -> Result { + let mut devices = MultiMap::new(); + let mut timestamps = vec![]; + + timestamps.push(std::time::Instant::now()); + + // + // First, get a component list. + // + let components = mgs_client.sp_component_list(type_, slot).await?; + timestamps.push(std::time::Instant::now()); + + // + // Now, for every component, we're going to get its details: for those + // that are sensors (and contain measurements), we will store the name + // of the sensor as well as the retrieved value. + // + for c in &components.components { + for s in mgs_client + .sp_component_get(type_, slot, &c.component) + .await? + .iter() + .filter_map(|detail| match detail { + SpComponentDetails::Measurement { kind, name, value } => Some( + (Sensor { name: name.clone(), kind: *kind }, Some(*value)), + ), + SpComponentDetails::MeasurementError { kind, name, error } => { + match error { + MeasurementErrorCode::NoReading + | MeasurementErrorCode::NotPresent => None, + _ => Some(( + Sensor { name: name.clone(), kind: *kind }, + None, + )), + } + } + _ => None, + }) + { + devices.insert(DeviceIdentifier::Device(c.component.clone()), s); + } + } + + timestamps.push(std::time::Instant::now()); + + Ok(SpInfo { devices, timestamps }) +} + +async fn sp_info_mgs( + mgs_client: &gateway_client::Client, + args: &SensorsArgs, +) -> Result { + let mut rval = vec![]; + let mut latencies = HashMap::new(); + + // + // First, get all of the SPs that we can see via Ignition + // + let all_sp_list = + mgs_client.ignition_list().await.context("listing ignition")?; + + let mut sp_list = all_sp_list + .iter() + .filter_map(|ignition| { + if matches!(ignition.details, SpIgnition::Yes { .. }) + && ignition.id.type_ == SpType::Sled + { + if args.matches_sp(&ignition.id) { + return Some(ignition.id); + } + } + None + }) + .collect::>(); + + if args.switches { + sp_list.push(SpIdentifier { type_: SpType::Switch, slot: 0 }); + sp_list.push(SpIdentifier { type_: SpType::Switch, slot: 1 }); + } + + if args.psc { + sp_list.push(SpIdentifier { type_: SpType::Power, slot: 0 }); + } + + sp_list.sort(); + + let now = std::time::Instant::now(); + + let mut handles = vec![]; + for sp_id in sp_list { + let handle = + tokio::spawn(sp_info(mgs_client.clone(), sp_id.type_, sp_id.slot)); + + handles.push((sp_id, handle)); + } + + for (sp_id, handle) in handles { + match handle.await.unwrap() { + Ok(info) => { + let l0 = info.timestamps[1].duration_since(info.timestamps[0]); + let l1 = info.timestamps[2].duration_since(info.timestamps[1]); + + if args.verbose { + eprintln!( + "mgs: latencies for {sp_id:?}: {l1:.1?} {l0:.1?}", + ); + } + + latencies.insert(sp_id, l0 + l1); + rval.push((sp_id, info)); + } + + Err(err) => { + eprintln!("failed to read devices for {:?}: {:?}", sp_id, err); + } + } + } + + if args.verbose { + eprintln!("total discovery time {:?}", now.elapsed()); + } + + Ok(SensorSpInfo { + info: rval, + time: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(), + latencies: Some(latencies), + }) +} + +fn sp_info_csv( + reader: &mut csv::Reader, + position: &mut csv::Position, + args: &SensorsArgs, +) -> Result { + let mut sps = vec![]; + let headers = reader.headers()?; + + let expected = ["TIME", "SENSOR", "KIND"]; + let len = expected.len(); + let hlen = headers.len(); + + if hlen < len { + bail!("expected as least {len} fields (found {headers:?})"); + } + + for ndx in 0..len { + if &headers[ndx] != expected[ndx] { + bail!( + "malformed headers: expected {}, found {} ({headers:?})", + &expected[ndx], + &headers[ndx] + ); + } + } + + for ndx in len..hlen { + let field = &headers[ndx]; + let parts: Vec<&str> = field.splitn(2, '-').collect(); + + if parts.len() != 2 { + bail!("malformed field \"{field}\""); + } + + let type_ = match parts[0] { + "SLED" => SpType::Sled, + "SWITCH" => SpType::Switch, + "POWER" => SpType::Power, + _ => { + bail!("unknown type {}", parts[0]); + } + }; + + let slot = parts[1].parse::().or_else(|_| { + bail!("invalid slot in \"{field}\""); + })?; + + let sp = SpIdentifier { type_, slot }; + + if args.matches_sp(&sp) { + sps.push(Some(sp)); + } else { + sps.push(None); + } + } + + let mut iter = reader.records(); + let mut sensors = HashSet::new(); + let mut by_sp = MultiMap::new(); + let mut time = None; + + loop { + *position = iter.reader().position().clone(); + + if let Some(record) = iter.next() { + let record = record?; + + if record.len() != hlen { + bail!("bad record length at line {}", position.line()); + } + + if time.is_none() { + let t = record[0].parse::().or_else(|_| { + bail!("bad time at line {}", position.line()); + })?; + + if let Some(start) = args.start { + if t < start { + continue; + } + } + + if let Some(end) = args.end { + if let Some(start) = args.start { + if start > end { + bail!( + "specified start time is later than end time" + ); + } + } + + if t > end { + bail!( + "specified end time ({end}) is earlier \ + than time of earliest record ({t})" + ); + } + } + + time = Some(t); + } + + if let Some(sensor) = Sensor::from_string(&record[1], &record[2]) { + if sensors.get(&sensor).is_some() { + break; + } + + sensors.insert(sensor.clone()); + + for (ndx, sp) in sps.iter().enumerate() { + if let Some(sp) = sp { + let value = match record[ndx + len].parse::() { + Ok(value) => Some(value), + _ => { + // + // We want to distinguish between the device + // having an error ("X") and it being absent + // ("-"); if it's absent, we don't want to add + // it at all. + // + match &record[ndx + len] { + "X" => {} + "-" => continue, + _ => { + bail!( + "line {}: unrecognized value \ + \"{}\" in field {}", + position.line(), + record[ndx + len].to_string(), + ndx + len + ); + } + } + + None + } + }; + + by_sp.insert(sp, (sensor.clone(), value)); + } + } + } + } else { + break; + } + } + + if time.is_none() { + bail!("no data found"); + } + + let mut rval = vec![]; + + for (field, sp) in sps.iter().enumerate() { + let mut devices = MultiMap::new(); + + if let Some(sp) = sp { + if let Some(v) = by_sp.remove(sp) { + devices.insert_many(DeviceIdentifier::Field(field + len), v); + } + + rval.push((*sp, SpInfo { devices, timestamps: vec![] })); + } + } + + Ok(SensorSpInfo { info: rval, time: time.unwrap(), latencies: None }) +} + +pub(crate) async fn sensor_metadata( + input: &mut SensorInput, + args: &SensorsArgs, +) -> Result<(Arc, SensorValues), anyhow::Error> { + let by_kind = if let Some(types) = &args.types { + let mut h = HashSet::new(); + + for t in types { + h.insert(match Sensor::from_string("", t) { + None => bail!("invalid sensor kind {t}"), + Some(s) => s.kind, + }); + } + + Some(h) + } else { + None + }; + + let by_name = args + .named + .as_ref() + .map(|named| named.into_iter().collect::>()); + + let info = match input { + SensorInput::MgsClient(ref mgs_client) => { + sp_info_mgs(mgs_client, args).await? + } + SensorInput::CsvReader(reader, position) => { + sp_info_csv(reader, position, args)? + } + }; + + let mut sensors_by_sensor = MultiMap::new(); + let mut sensors_by_sensor_and_sp = HashMap::new(); + let mut sensors_by_id = HashMap::new(); + let mut sensors_by_sp = MultiMap::new(); + let mut values = HashMap::new(); + let mut work_by_sp = HashMap::new(); + + let mut current = 0; + let time = info.time; + + for (sp_id, info) in info.info { + let mut sp_work = vec![]; + + for (device, sensors) in info.devices { + let mut device_work = vec![]; + + for (sensor, value) in sensors { + if let Some(ref by_kind) = by_kind { + if by_kind.get(&sensor.kind).is_none() { + continue; + } + } + + if let Some(ref by_name) = by_name { + if by_name.get(&sensor.name).is_none() { + continue; + } + } + + let id = SensorId(current); + current += 1; + + sensors_by_id + .insert(id, (sp_id, sensor.clone(), device.clone())); + + if value.is_none() && args.verbose { + eprintln!( + "mgs: error for {sp_id:?} on {sensor:?} ({device:?})" + ); + } + + sensors_by_sensor.insert(sensor.clone(), id); + + let by_sp = sensors_by_sensor_and_sp + .entry(sensor) + .or_insert_with(|| HashMap::new()); + by_sp.insert(sp_id, id); + sensors_by_sp.insert(sp_id, id); + values.insert(id, value); + + device_work.push(id); + } + + sp_work.push((device, device_work)); + } + + work_by_sp.insert(sp_id, sp_work); + } + + Ok(( + Arc::new(SensorMetadata { + sensors_by_sensor, + sensors_by_sensor_and_sp, + sensors_by_id, + sensors_by_sp, + work_by_sp, + start_time: args.start, + end_time: match args.end { + Some(end) => Some(end), + None => args.duration.map(|duration| time + duration), + }, + }), + SensorValues { values, time, latencies: info.latencies }, + )) +} + +async fn sp_read_sensors( + mgs_client: &gateway_client::Client, + id: &SpIdentifier, + metadata: &SensorMetadata, +) -> Result<(Vec<(SensorId, Option)>, Duration), anyhow::Error> { + let work = metadata.work_by_sp.get(id).unwrap(); + let mut rval = vec![]; + + let start = std::time::Instant::now(); + + for (component, ids) in work.iter() { + for (value, id) in mgs_client + .sp_component_get(id.type_, id.slot, component.device()) + .await? + .iter() + .filter_map(|detail| match detail { + SpComponentDetails::Measurement { kind: _, name: _, value } => { + Some(Some(*value)) + } + SpComponentDetails::MeasurementError { error, .. } => { + match error { + MeasurementErrorCode::NoReading + | MeasurementErrorCode::NotPresent => None, + _ => Some(None), + } + } + _ => None, + }) + .zip(ids.iter()) + { + rval.push((*id, value)); + } + } + + Ok((rval, start.elapsed())) +} + +async fn sp_data_mgs( + mgs_client: &gateway_client::Client, + metadata: &Arc, +) -> Result { + let mut values = HashMap::new(); + let mut latencies = HashMap::new(); + let mut handles = vec![]; + + for sp_id in metadata.sensors_by_sp.keys() { + let mgs_client = mgs_client.clone(); + let id = *sp_id; + let metadata = Arc::clone(&metadata); + + let handle = tokio::spawn(async move { + sp_read_sensors(&mgs_client, &id, &metadata).await + }); + + handles.push((id, handle)); + } + + for (id, handle) in handles { + let (rval, latency) = handle.await.unwrap()?; + + latencies.insert(id, latency); + + for (id, value) in rval { + values.insert(id, value); + } + } + + Ok(SensorValues { + values, + latencies: Some(latencies), + time: SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(), + }) +} + +fn sp_data_csv( + reader: &mut csv::Reader, + position: &mut csv::Position, + metadata: &SensorMetadata, +) -> Result { + let headers = reader.headers()?; + let hlen = headers.len(); + let mut values = HashMap::new(); + + reader.seek(position.clone())?; + let mut iter = reader.records(); + + let mut time = None; + + loop { + *position = iter.reader().position().clone(); + + if let Some(record) = iter.next() { + let record = record?; + + if record.len() != hlen { + bail!("bad record length at line {}", position.line()); + } + + let now = record[0].parse::().or_else(|_| { + bail!("bad time at line {}", position.line()); + })?; + + if let Some(time) = time { + if now != time { + break; + } + } else { + if let Some(end) = metadata.end_time { + if now > end { + time = Some(0); + break; + } + } + + time = Some(now); + } + + if let Some(sensor) = Sensor::from_string(&record[1], &record[2]) { + if let Some(ids) = metadata.sensors_by_sensor.get_vec(&sensor) { + for id in ids { + let (_, _, d) = metadata.sensors_by_id.get(id).unwrap(); + let value = match record[d.field()].parse::() { + Ok(value) => Some(value), + _ => None, + }; + + values.insert(*id, value); + } + } + } else { + bail!("bad sensor at line {}", position.line()); + } + } else { + time = Some(0); + break; + } + } + + Ok(SensorValues { values, latencies: None, time: time.unwrap() }) +} + +pub(crate) async fn sensor_data( + input: &mut SensorInput, + metadata: &Arc, +) -> Result { + match input { + SensorInput::MgsClient(ref mgs_client) => { + sp_data_mgs(mgs_client, metadata).await + } + SensorInput::CsvReader(reader, position) => { + sp_data_csv(reader, position, &metadata) + } + } +} + +/// +/// Runs `omdb mgs sensors` +/// +pub(crate) async fn cmd_mgs_sensors( + omdb: &crate::Omdb, + log: &slog::Logger, + mgs_args: &crate::mgs::MgsArgs, + args: &SensorsArgs, +) -> Result<(), anyhow::Error> { + let mut input = if let Some(ref input) = args.input { + let file = File::open(input) + .with_context(|| format!("failed to open {input}"))?; + SensorInput::CsvReader( + csv::Reader::from_reader(file), + csv::Position::new(), + ) + } else { + SensorInput::MgsClient(mgs_args.mgs_client(omdb, log).await?) + }; + + let (metadata, mut values) = sensor_metadata(&mut input, args).await?; + + let mut sensors = metadata.sensors_by_sensor.keys().collect::>(); + sensors.sort(); + + let mut sps = metadata.sensors_by_sp.keys().collect::>(); + sps.sort(); + + let print_value = |v| { + if args.parseable { + print!(",{v}"); + } else { + print!(" {v:>8}"); + } + }; + + let print_header = || { + if !args.parseable { + print!("{:20} ", "NAME"); + } else { + print!("TIME,SENSOR,KIND"); + } + + for sp in &sps { + print_value(format!( + "{}-{}", + crate::mgs::sp_type_to_str(&sp.type_).to_uppercase(), + sp.slot + )); + } + + println!(); + }; + + let print_name = |sensor: &Sensor, now: u64| { + if !args.parseable { + print!("{:20} ", sensor.name); + } else { + print!("{now},{},{}", sensor.name, sensor.to_kind_string()); + } + }; + + let print_latency = |now: u64| { + if !args.parseable { + print!("{:20} ", "LATENCY"); + } else { + print!("{now},{},{}", "LATENCY", "latency"); + } + }; + + let mut wakeup = + tokio::time::Instant::now() + tokio::time::Duration::from_millis(1000); + + print_header(); + + loop { + for sensor in &sensors { + print_name(sensor, values.time); + + let by_sp = metadata.sensors_by_sensor_and_sp.get(sensor).unwrap(); + + for sp in &sps { + print_value(if let Some(id) = by_sp.get(sp) { + if let Some(value) = values.values.get(id) { + match value { + Some(value) => { + sensor.format(*value, args.parseable) + } + None => "X".to_string(), + } + } else { + "?".to_string() + } + } else { + "-".to_string() + }); + } + + println!(); + } + + if args.show_latencies { + if let Some(latencies) = values.latencies { + print_latency(values.time); + + for sp in &sps { + print_value(if let Some(latency) = latencies.get(sp) { + format!("{}ms", latency.as_millis()) + } else { + "?".to_string() + }); + } + } + + println!(); + } + + if !args.sleep { + if args.input.is_none() { + break; + } + } else { + tokio::time::sleep_until(wakeup).await; + wakeup += tokio::time::Duration::from_millis(1000); + } + + values = sensor_data(&mut input, &metadata).await?; + + if args.input.is_some() && values.time == 0 { + break; + } + + if !args.parseable { + print_header(); + } + } + + Ok(()) +} diff --git a/dev-tools/omdb/tests/usage_errors.out b/dev-tools/omdb/tests/usage_errors.out index 2790b0ef83..7688372984 100644 --- a/dev-tools/omdb/tests/usage_errors.out +++ b/dev-tools/omdb/tests/usage_errors.out @@ -270,7 +270,9 @@ Debug a specific Management Gateway Service instance Usage: omdb mgs [OPTIONS] Commands: + dashboard Dashboard of SPs inventory Show information about devices and components visible to MGS + sensors Show information about sensors, as gleaned by MGS help Print this message or the help of the given subcommand(s) Options: