From a900cdd12ebe57a2790e44127321f7b052175783 Mon Sep 17 00:00:00 2001 From: Benjamin Naecker Date: Sat, 20 Jan 2024 01:21:32 +0000 Subject: [PATCH] Publish instance vCPU usage statistics to `oximeter` - Adds the silo and project IDs to the instance-ensure request from Nexus to the sled-agent. These are used as fields on the instance-related statistics. - Defines a `VirtualMachine` oximeter target and `VcpuUsage` metric. The latter has a `state` field which corresponds to the named kstats published by the hypervisor that accumulate the time spent in a number of vCPU microstates. The combination of these should allow us to aggregate or break down vCPU usage by silo, project, instance, vCPU ID, and CPU state. - Adds APIs to the `MetricsManager` for starting / stopping tracking instance-related metrics, and plumbs the type through the `InstanceManager` and `Instance` (and their internal friends), so that new instances can control when data is produced from them. Currently, we'll start producing as soon as we get a non-terminate response from Propolis in the `instance_state_monitor()` task, and stop when the instance is terminated. --- nexus/src/app/instance.rs | 16 ++ openapi/sled-agent.json | 27 +++ oximeter/instruments/src/kstat/mod.rs | 1 + .../instruments/src/kstat/virtual_machine.rs | 185 ++++++++++++++++++ sled-agent/src/http_entrypoints.rs | 1 + sled-agent/src/instance.rs | 46 ++++- sled-agent/src/instance_manager.rs | 12 ++ sled-agent/src/metrics.rs | 165 ++++++++++++---- sled-agent/src/params.rs | 10 + sled-agent/src/sim/http_entrypoints.rs | 1 + sled-agent/src/sim/sled_agent.rs | 8 +- sled-agent/src/sled_agent.rs | 103 +++++----- 12 files changed, 482 insertions(+), 93 deletions(-) create mode 100644 oximeter/instruments/src/kstat/virtual_machine.rs diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index 778c5e2fe18..45eab1de918 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -1135,6 +1135,21 @@ impl super::Nexus { .map(|ssh_key| ssh_key.public_key) .collect::>(); + // Construct instance metadata used to track its statistics. + // + // This current means fetching the current silo ID, since we have all + // the other metadata already. + let silo_id = self + .current_silo_lookup(opctx)? + .lookup_for(authz::Action::Read) + .await? + .0 + .id(); + let metadata = sled_agent_client::types::InstanceMetadata { + silo_id, + project_id: db_instance.project_id, + }; + // Ask the sled agent to begin the state change. Then update the // database to reflect the new intermediate state. If this update is // not the newest one, that's fine. That might just mean the sled agent @@ -1178,6 +1193,7 @@ impl super::Nexus { PROPOLIS_PORT, ) .to_string(), + metadata, }, ) .await diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index b5b9d3fd5b2..2667be5356f 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -4515,6 +4515,14 @@ } ] }, + "metadata": { + "description": "Metadata used to track instance statistics.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceMetadata" + } + ] + }, "propolis_addr": { "description": "The address at which this VMM should serve a Propolis server API.", "type": "string" @@ -4536,6 +4544,7 @@ "required": [ "hardware", "instance_runtime", + "metadata", "propolis_addr", "propolis_id", "vmm_runtime" @@ -4624,6 +4633,24 @@ "snapshot_id" ] }, + "InstanceMetadata": { + "description": "Metadata used to track statistics about an instance.", + "type": "object", + "properties": { + "project_id": { + "type": "string", + "format": "uuid" + }, + "silo_id": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "project_id", + "silo_id" + ] + }, "InstanceMigrationSourceParams": { "description": "Instance runtime state to update for a migration.", "type": "object", diff --git a/oximeter/instruments/src/kstat/mod.rs b/oximeter/instruments/src/kstat/mod.rs index 90f34acae85..d8b0c23887e 100644 --- a/oximeter/instruments/src/kstat/mod.rs +++ b/oximeter/instruments/src/kstat/mod.rs @@ -89,6 +89,7 @@ use std::time::Duration; pub mod link; mod sampler; +pub mod virtual_machine; pub use sampler::CollectionDetails; pub use sampler::ExpirationBehavior; diff --git a/oximeter/instruments/src/kstat/virtual_machine.rs b/oximeter/instruments/src/kstat/virtual_machine.rs new file mode 100644 index 00000000000..1e1637cd562 --- /dev/null +++ b/oximeter/instruments/src/kstat/virtual_machine.rs @@ -0,0 +1,185 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Copyright 2023 Oxide Computer Company + +//! Types for tracking statistics about virtual machine instances. + +use crate::kstat::hrtime_to_utc; +use crate::kstat::ConvertNamedData; +use crate::kstat::Error; +use crate::kstat::KstatList; +use crate::kstat::KstatTarget; +use chrono::DateTime; +use chrono::Utc; +use kstat_rs::Data; +use kstat_rs::Kstat; +use kstat_rs::Named; +use kstat_rs::NamedData; +use oximeter::types::Cumulative; +use oximeter::Metric; +use oximeter::Sample; +use oximeter::Target; +use uuid::Uuid; + +/// A single virtual machine +#[derive(Clone, Debug, Target)] +pub struct VirtualMachine { + /// The silo to which the instance belongs. + pub silo_id: Uuid, + /// The project to which the instance belongs. + pub project_id: Uuid, + /// The ID of the instance. + pub instance_id: Uuid, +} + +/// Metric tracking vCPU usage by state. +#[derive(Clone, Debug, Metric)] +pub struct VcpuUsage { + /// The vCPU ID. + pub vcpu_id: u32, + /// The state of the vCPU. + pub state: String, + /// The cumulative time spent in this state, in nanoseconds. + pub datum: Cumulative, +} + +// The name of the kstat module containing virtual machine kstats. +const VMM_KSTAT_MODULE_NAME: &str = "vmm"; + +// The name of the kstat with virtual machine metadata (VM name currently). +const VM_KSTAT_NAME: &str = "vm"; + +// The named kstat holding the virtual machine's name. This is currently the +// UUID assigned by the control plane to the virtual machine instance. +const VM_NAME_KSTAT: &str = "vm_name"; + +// The name of kstat containing vCPU usage data. +const VCPU_KSTAT_PREFIX: &str = "vcpu"; + +// Prefix for all named data with a valid vCPU microstate that we track. +const VCPU_MICROSTATE_PREFIX: &str = "time_"; + +// The number of expected vCPU microstates we track. This isn't load-bearing, +// and only used to help preallocate an array holding the `VcpuUsage` samples. +const N_VCPU_MICROSTATES: usize = 6; + +impl KstatTarget for VirtualMachine { + // The VMM kstats are organized like so: + // + // - module: vmm + // - instance: a kernel-assigned integer + // - name: vm -> generic VM info, vcpuX -> info for each vCPU + // + // At this part of the code, we don't have that kstat instance, only the + // virtual machine instance's control plane UUID. However, the VM's "name" + // is assigned to be that control plane UUID in the hypervisor. See + // https://github.com/oxidecomputer/propolis/blob/759bf4a19990404c135e608afbe0d38b70bfa370/bin/propolis-server/src/lib/vm/mod.rs#L420 + // for the current code which does that. + // + // So we need to indicate interest in any VMM-related kstat here, and we are + // forced to filter to the right instance by looking up the VM name inside + // the `to_samples()` method below. + fn interested(&self, kstat: &Kstat<'_>) -> bool { + kstat.ks_module == VMM_KSTAT_MODULE_NAME + } + + fn to_samples( + &self, + kstats: KstatList<'_, '_>, + ) -> Result, Error> { + // First, we need to map the instance's control plane UUID to the + // instance ID. We'll find this through the `vmm::vm:vm_name` + // kstat, which lists the instance's UUID as its name. + let instance_id = self.instance_id.to_string(); + let instance = kstats + .iter() + .find_map(|(_, kstat, data)| { + kstat_instance_from_instance_id(kstat, data, &instance_id) + }) + .ok_or_else(|| Error::NoSuchKstat)?; + + // Armed with the kstat instance, find all relevant metrics related to + // this particular VM. For now, we produce only vCPU usage metrics, but + // others may be chained in the future. + let vcpu_stats = kstats.iter().filter(|(_, kstat, _)| { + kstat.ks_instance == instance + && kstat.ks_name.starts_with(VCPU_KSTAT_PREFIX) + }); + produce_vcpu_usage(self, vcpu_stats) + } +} + +// Given a kstat and an instance's ID, return the kstat instance if it matches. +pub fn kstat_instance_from_instance_id( + kstat: &Kstat<'_>, + data: &Data<'_>, + instance_id: &str, +) -> Option { + if kstat.ks_module != VMM_KSTAT_MODULE_NAME { + return None; + } + if kstat.ks_name != VM_KSTAT_NAME { + return None; + } + let Data::Named(named) = data else { + return None; + }; + if named.iter().any(|nd| { + if nd.name != VM_NAME_KSTAT { + return false; + } + let NamedData::String(name) = &nd.value else { + return false; + }; + instance_id == *name + }) { + return Some(kstat.ks_instance); + } + None +} + +// Produce `Sample`s for the `VcpuUsage` metric from the relevant kstats. +pub fn produce_vcpu_usage<'a>( + vm: &'a VirtualMachine, + vcpu_stats: impl Iterator, Kstat<'a>, Data<'a>)> + 'a, +) -> Result, Error> { + let mut out = Vec::with_capacity(N_VCPU_MICROSTATES); + for (creation_time, kstat, data) in vcpu_stats { + let Data::Named(named) = data else { + return Err(Error::ExpectedNamedKstat); + }; + let snapshot_time = hrtime_to_utc(kstat.ks_snaptime)?; + + // Find the vCPU ID, from the relevant named data item. + let vcpu_id = named + .iter() + .find_map(|named| { + if named.name == VCPU_KSTAT_PREFIX { + named.value.as_u32().ok() + } else { + None + } + }) + .ok_or_else(|| Error::NoSuchKstat)?; + + // We'll track all statistics starting with `time_` as the microstate. + for Named { name, value } in named + .iter() + .filter(|nv| nv.name.starts_with(VCPU_MICROSTATE_PREFIX)) + { + // Safety: We're filtering in the loop on this prefix, so it must + // exist. + let state = + name.strip_prefix(VCPU_MICROSTATE_PREFIX).unwrap().to_string(); + let datum = + Cumulative::with_start_time(*creation_time, value.as_u64()?); + let metric = VcpuUsage { vcpu_id, state, datum }; + let sample = + Sample::new_with_timestamp(snapshot_time, vm, &metric)?; + out.push(sample); + } + } + Ok(out) +} diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 39d1ae26a05..886b683f49d 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -410,6 +410,7 @@ async fn instance_register( body_args.instance_runtime, body_args.vmm_runtime, body_args.propolis_addr, + body_args.metadata, ) .await?, )) diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 057402c57a9..cffdbf9e979 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -9,11 +9,14 @@ use crate::common::instance::{ PublishedVmmState, }; use crate::instance_manager::{InstanceManagerServices, InstanceTicket}; +use crate::metrics::Error as MetricsError; +use crate::metrics::MetricsManager; +use crate::metrics::INSTANCE_SAMPLE_INTERVAL; use crate::nexus::NexusClientWithResolver; use crate::params::ZoneBundleCause; use crate::params::ZoneBundleMetadata; use crate::params::{ - InstanceHardware, InstanceMigrationSourceParams, + InstanceHardware, InstanceMetadata, InstanceMigrationSourceParams, InstanceMigrationTargetParams, InstanceStateRequested, VpcFirewallRule, }; use crate::profile::*; @@ -108,6 +111,9 @@ pub enum Error { #[error("I/O error")] Io(#[from] std::io::Error), + + #[error("Failed to track instance metrics")] + Metrics(#[source] MetricsError), } // Issues read-only, idempotent HTTP requests at propolis until it responds with @@ -233,8 +239,14 @@ struct InstanceInner { // Object used to collect zone bundles from this instance when terminated. zone_bundler: ZoneBundler, + // Object used to start / stop collection of instance-related metrics. + metrics_manager: MetricsManager, + // Object representing membership in the "instance manager". instance_ticket: InstanceTicket, + + // Metadata used to track statistics for this instance. + metadata: InstanceMetadata, } impl InstanceInner { @@ -367,6 +379,10 @@ impl InstanceInner { // state to Nexus. This ensures that the instance is actually gone from // the sled when Nexus receives the state update saying it's actually // destroyed. + // + // In addition, we'll start or stop collecting metrics soley on the + // basis of whether the instance is terminated. All other states imply + // we start (or continue) to collect instance metrics. match action { Some(InstanceAction::Destroy) => { info!(self.log, "terminating VMM that has exited"; @@ -375,7 +391,17 @@ impl InstanceInner { self.terminate().await?; Ok(Reaction::Terminate) } - None => Ok(Reaction::Continue), + None => { + self.metrics_manager + .track_instance( + &self.id(), + &self.metadata, + INSTANCE_SAMPLE_INTERVAL, + ) + .await + .map_err(Error::Metrics)?; + Ok(Reaction::Continue) + } } } @@ -537,6 +563,18 @@ impl InstanceInner { ); } + // Stop tracking instance-related metrics. + if let Err(e) = + self.metrics_manager.stop_tracking_instance(self.id()).await + { + error!( + self.log, + "Failed to stop tracking instance metrics"; + "instance_id" => %self.id(), + "error" => ?e, + ); + } + // Ensure that no zone exists. This succeeds even if no zone was ever // created. // NOTE: we call`Zones::halt_and_remove_logged` directly instead of @@ -596,6 +634,7 @@ impl Instance { ticket: InstanceTicket, state: InstanceInitialState, services: InstanceManagerServices, + metadata: InstanceMetadata, ) -> Result { info!(log, "initializing new Instance"; "instance_id" => %id, @@ -615,6 +654,7 @@ impl Instance { port_manager, storage, zone_bundler, + metrics_manager, zone_builder_factory, } = services; @@ -686,7 +726,9 @@ impl Instance { storage, zone_builder_factory, zone_bundler, + metrics_manager, instance_ticket: ticket, + metadata, }; let inner = Arc::new(Mutex::new(instance)); diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index c1b7e402a49..61acf6d15f6 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -6,7 +6,9 @@ use crate::instance::propolis_zone_name; use crate::instance::Instance; +use crate::metrics::MetricsManager; use crate::nexus::NexusClientWithResolver; +use crate::params::InstanceMetadata; use crate::params::ZoneBundleMetadata; use crate::params::{ InstanceHardware, InstanceMigrationSourceParams, InstancePutStateResponse, @@ -77,6 +79,7 @@ struct InstanceManagerInternal { port_manager: PortManager, storage: StorageHandle, zone_bundler: ZoneBundler, + metrics_manager: MetricsManager, zone_builder_factory: ZoneBuilderFactory, } @@ -86,6 +89,7 @@ pub(crate) struct InstanceManagerServices { pub port_manager: PortManager, pub storage: StorageHandle, pub zone_bundler: ZoneBundler, + pub metrics_manager: MetricsManager, pub zone_builder_factory: ZoneBuilderFactory, } @@ -96,6 +100,7 @@ pub struct InstanceManager { impl InstanceManager { /// Initializes a new [`InstanceManager`] object. + #[allow(clippy::too_many_arguments)] pub fn new( log: Logger, nexus_client: NexusClientWithResolver, @@ -103,6 +108,7 @@ impl InstanceManager { port_manager: PortManager, storage: StorageHandle, zone_bundler: ZoneBundler, + metrics_manager: MetricsManager, zone_builder_factory: ZoneBuilderFactory, ) -> Result { Ok(InstanceManager { @@ -117,6 +123,7 @@ impl InstanceManager { port_manager, storage, zone_bundler, + metrics_manager, zone_builder_factory, }), }) @@ -215,6 +222,7 @@ impl InstanceManager { /// (instance ID, Propolis ID) pair multiple times, but will fail if the /// instance is registered with a Propolis ID different from the one the /// caller supplied. + #[allow(clippy::too_many_arguments)] pub async fn ensure_registered( &self, instance_id: Uuid, @@ -223,6 +231,7 @@ impl InstanceManager { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, + metadata: InstanceMetadata, ) -> Result { info!( &self.inner.log, @@ -233,6 +242,7 @@ impl InstanceManager { "instance_runtime" => ?instance_runtime, "vmm_runtime" => ?vmm_runtime, "propolis_addr" => ?propolis_addr, + "metadata" => ?metadata, ); let instance = { @@ -271,6 +281,7 @@ impl InstanceManager { port_manager: self.inner.port_manager.clone(), storage: self.inner.storage.clone(), zone_bundler: self.inner.zone_bundler.clone(), + metrics_manager: self.inner.metrics_manager.clone(), zone_builder_factory: self .inner .zone_builder_factory @@ -291,6 +302,7 @@ impl InstanceManager { ticket, state, services, + metadata, )?; let instance_clone = instance.clone(); let _old = diff --git a/sled-agent/src/metrics.rs b/sled-agent/src/metrics.rs index 6c3383c88f0..cf87d690094 100644 --- a/sled-agent/src/metrics.rs +++ b/sled-agent/src/metrics.rs @@ -11,14 +11,19 @@ use slog::Logger; use std::time::Duration; use uuid::Uuid; +use crate::params::InstanceMetadata; + cfg_if::cfg_if! { if #[cfg(target_os = "illumos")] { use oximeter_instruments::kstat::link; + use oximeter_instruments::kstat::virtual_machine; use oximeter_instruments::kstat::CollectionDetails; use oximeter_instruments::kstat::Error as KstatError; use oximeter_instruments::kstat::KstatSampler; use oximeter_instruments::kstat::TargetId; use std::collections::BTreeMap; + use std::sync::Arc; + use std::sync::Mutex; } else { use anyhow::anyhow; } @@ -30,6 +35,9 @@ pub(crate) const METRIC_COLLECTION_INTERVAL: Duration = Duration::from_secs(30); /// The interval on which we sample link metrics. pub(crate) const LINK_SAMPLE_INTERVAL: Duration = Duration::from_secs(10); +/// The interval on which we sample instance-related metrics, e.g., vCPU usage. +pub(crate) const INSTANCE_SAMPLE_INTERVAL: Duration = Duration::from_secs(10); + /// An error during sled-agent metric production. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -46,6 +54,17 @@ pub enum Error { #[error("Failed to fetch hostname")] Hostname(#[source] std::io::Error), + + #[error("Non-UTF8 hostname")] + NonUtf8Hostname, +} + +// Basic metadata about the sled agent used when publishing metrics. +#[derive(Clone, Debug)] +struct Metadata { + sled_id: Uuid, + rack_id: Uuid, + baseboard: Baseboard, } /// Type managing all oximeter metrics produced by the sled-agent. @@ -61,10 +80,7 @@ pub enum Error { // the name of fields that are not yet used. #[cfg_attr(not(target_os = "illumos"), allow(dead_code))] pub struct MetricsManager { - sled_id: Uuid, - rack_id: Uuid, - baseboard: Baseboard, - hostname: Option, + metadata: Arc, _log: Logger, #[cfg(target_os = "illumos")] kstat_sampler: KstatSampler, @@ -78,7 +94,9 @@ pub struct MetricsManager { // namespace them internally, e.g., `"datalink:{link_name}"` would be the // real key. #[cfg(target_os = "illumos")] - tracked_links: BTreeMap, + tracked_links: Arc>>, + #[cfg(target_os = "illumos")] + tracked_instances: Arc>>, registry: ProducerRegistry, } @@ -101,19 +119,19 @@ impl MetricsManager { registry .register_producer(kstat_sampler.clone()) .map_err(Error::Registry)?; - let tracked_links = BTreeMap::new(); + let tracked_links = Arc::new(Mutex::new(BTreeMap::new())); + let tracked_instances = Arc::new(Mutex::new(BTreeMap::new())); } } Ok(Self { - sled_id, - rack_id, - baseboard, - hostname: None, + metadata: Arc::new(Metadata { sled_id, rack_id, baseboard }), _log: log, #[cfg(target_os = "illumos")] kstat_sampler, #[cfg(target_os = "illumos")] tracked_links, + #[cfg(target_os = "illumos")] + tracked_instances, registry, }) } @@ -128,14 +146,14 @@ impl MetricsManager { impl MetricsManager { /// Track metrics for a physical datalink. pub async fn track_physical_link( - &mut self, + &self, link_name: impl AsRef, interval: Duration, ) -> Result<(), Error> { - let hostname = self.hostname().await?; + let hostname = hostname()?; let link = link::PhysicalDataLink { - rack_id: self.rack_id, - sled_id: self.sled_id, + rack_id: self.metadata.rack_id, + sled_id: self.metadata.sled_id, serial: self.serial_number(), hostname, link_name: link_name.as_ref().to_string(), @@ -146,7 +164,10 @@ impl MetricsManager { .add_target(link, details) .await .map_err(Error::Kstat)?; - self.tracked_links.insert(link_name.as_ref().to_string(), id); + self.tracked_links + .lock() + .unwrap() + .insert(link_name.as_ref().to_string(), id); Ok(()) } @@ -155,10 +176,12 @@ impl MetricsManager { /// This works for both physical and virtual links. #[allow(dead_code)] pub async fn stop_tracking_link( - &mut self, + &self, link_name: impl AsRef, ) -> Result<(), Error> { - if let Some(id) = self.tracked_links.remove(link_name.as_ref()) { + let maybe_id = + self.tracked_links.lock().unwrap().remove(link_name.as_ref()); + if let Some(id) = maybe_id { self.kstat_sampler.remove_target(id).await.map_err(Error::Kstat) } else { Ok(()) @@ -174,8 +197,8 @@ impl MetricsManager { interval: Duration, ) -> Result<(), Error> { let link = link::VirtualDataLink { - rack_id: self.rack_id, - sled_id: self.sled_id, + rack_id: self.metadata.rack_id, + sled_id: self.metadata.sled_id, serial: self.serial_number(), hostname: hostname.as_ref().to_string(), link_name: link_name.as_ref().to_string(), @@ -188,42 +211,57 @@ impl MetricsManager { .map_err(Error::Kstat) } + /// Start tracking instance-related metrics. + pub async fn track_instance( + &self, + instance_id: &Uuid, + metadata: &InstanceMetadata, + interval: Duration, + ) -> Result<(), Error> { + let vm = virtual_machine::VirtualMachine { + silo_id: metadata.silo_id, + project_id: metadata.project_id, + instance_id: *instance_id, + }; + let details = CollectionDetails::never(interval); + let id = self + .kstat_sampler + .add_target(vm, details) + .await + .map_err(Error::Kstat)?; + self.tracked_instances.lock().unwrap().insert(*instance_id, id); + Ok(()) + } + + /// Stop tracking metrics associated with this instance. + pub async fn stop_tracking_instance( + &self, + instance_id: &Uuid, + ) -> Result<(), Error> { + let maybe_id = + self.tracked_instances.lock().unwrap().remove(instance_id); + if let Some(id) = maybe_id { + self.kstat_sampler.remove_target(id).await.map_err(Error::Kstat) + } else { + Ok(()) + } + } + // Return the serial number out of the baseboard, if one exists. fn serial_number(&self) -> String { - match &self.baseboard { + match &self.metadata.baseboard { Baseboard::Gimlet { identifier, .. } => identifier.clone(), Baseboard::Unknown => String::from("unknown"), Baseboard::Pc { identifier, .. } => identifier.clone(), } } - - // Return the system's hostname. - // - // If we've failed to get it previously, we try again. If _that_ fails, - // return an error. - // - // TODO-cleanup: This will become much simpler once - // `OnceCell::get_or_try_init` is stabilized. - async fn hostname(&mut self) -> Result { - if let Some(hn) = &self.hostname { - return Ok(hn.clone()); - } - let hn = tokio::process::Command::new("hostname") - .env_clear() - .output() - .await - .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string()) - .map_err(Error::Hostname)?; - self.hostname.replace(hn.clone()); - Ok(hn) - } } #[cfg(not(target_os = "illumos"))] impl MetricsManager { /// Track metrics for a physical datalink. pub async fn track_physical_link( - &mut self, + &self, _link_name: impl AsRef, _interval: Duration, ) -> Result<(), Error> { @@ -237,7 +275,7 @@ impl MetricsManager { /// This works for both physical and virtual links. #[allow(dead_code)] pub async fn stop_tracking_link( - &mut self, + &self, _link_name: impl AsRef, ) -> Result<(), Error> { Err(Error::Kstat(anyhow!( @@ -257,4 +295,45 @@ impl MetricsManager { "kstat metrics are not supported on this platform" ))) } + + /// Start tracking instance-related metrics. + #[allow(dead_code)] + pub async fn track_instance( + &self, + _instance_id: &Uuid, + _metadata: &InstanceMetadata, + _interval: Duration, + ) -> Result<(), Error> { + Err(Error::Kstat(anyhow!( + "kstat metrics are not supported on this platform" + ))) + } + + /// Stop tracking metrics associated with this instance. + #[allow(dead_code)] + pub async fn stop_tracking_instance( + &self, + _instance_id: &Uuid, + ) -> Result<(), Error> { + Err(Error::Kstat(anyhow!( + "kstat metrics are not supported on this platform" + ))) + } +} + +// Return the current hostname if possible. +fn hostname() -> Result { + // See netdb.h + const MAX_LEN: usize = 256; + let mut out = vec![0u8; MAX_LEN + 1]; + if unsafe { + libc::gethostname(out.as_mut_ptr() as *mut libc::c_char, MAX_LEN) + } == 0 + { + let s = + std::ffi::CString::new(out).map_err(|_| Error::NonUtf8Hostname)?; + s.into_string().map_err(|_| Error::NonUtf8Hostname) + } else { + Err(Error::Hostname(std::io::Error::last_os_error())) + } } diff --git a/sled-agent/src/params.rs b/sled-agent/src/params.rs index 9120bafa9ad..0d9df8723dd 100644 --- a/sled-agent/src/params.rs +++ b/sled-agent/src/params.rs @@ -81,6 +81,13 @@ pub struct InstanceHardware { pub cloud_init_bytes: Option, } +/// Metadata used to track statistics about an instance. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct InstanceMetadata { + pub silo_id: Uuid, + pub project_id: Uuid, +} + /// The body of a request to ensure that a instance and VMM are known to a sled /// agent. #[derive(Serialize, Deserialize, JsonSchema)] @@ -102,6 +109,9 @@ pub struct InstanceEnsureBody { /// The address at which this VMM should serve a Propolis server API. pub propolis_addr: SocketAddr, + + /// Metadata used to track instance statistics. + pub metadata: InstanceMetadata, } /// The body of a request to move a previously-ensured instance into a specific diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index e5d7752511c..3063247942d 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -95,6 +95,7 @@ async fn instance_register( body_args.hardware, body_args.instance_runtime, body_args.vmm_runtime, + body_args.metadata, ) .await?, )) diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 8a76bf6abc1..2b0e84a0120 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -12,9 +12,10 @@ use super::storage::CrucibleData; use super::storage::Storage; use crate::nexus::NexusClient; use crate::params::{ - DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, - InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, Inventory, OmicronZonesConfig, SledRole, + DiskStateRequested, InstanceHardware, InstanceMetadata, + InstanceMigrationSourceParams, InstancePutStateResponse, + InstanceStateRequested, InstanceUnregisterResponse, Inventory, + OmicronZonesConfig, SledRole, }; use crate::sim::simulatable::Simulatable; use crate::updates::UpdateManager; @@ -236,6 +237,7 @@ impl SledAgent { hardware: InstanceHardware, instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, + _metadata: InstanceMetadata, ) -> Result { // respond with a fake 500 level failure if asked to ensure an instance // with more than 16 CPUs. diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index d094643cf9b..418e776b253 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -16,10 +16,11 @@ use crate::long_running_tasks::LongRunningTaskHandles; use crate::metrics::MetricsManager; use crate::nexus::{ConvertInto, NexusClientWithResolver, NexusRequestQueue}; use crate::params::{ - DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, - InstancePutStateResponse, InstanceStateRequested, - InstanceUnregisterResponse, Inventory, OmicronZonesConfig, SledRole, - TimeSync, VpcFirewallRule, ZoneBundleMetadata, Zpool, + DiskStateRequested, InstanceHardware, InstanceMetadata, + InstanceMigrationSourceParams, InstancePutStateResponse, + InstanceStateRequested, InstanceUnregisterResponse, Inventory, + OmicronZonesConfig, SledRole, TimeSync, VpcFirewallRule, + ZoneBundleMetadata, Zpool, }; use crate::services::{self, ServiceManager}; use crate::storage_monitor::UnderlayAccess; @@ -393,6 +394,55 @@ impl SledAgent { let underlay_nics = underlay::find_nics(&config.data_links)?; illumos_utils::opte::initialize_xde_driver(&log, &underlay_nics)?; + // Start collecting metric data. + // + // First, we're creating a shareable type for managing the metrics + // themselves early on, so that we can pass it to other components of + // the sled agent that need it. + // + // Then we'll start tracking physical links and register as a producer + // with Nexus in the background. + let metrics_manager = MetricsManager::new( + request.body.id, + request.body.rack_id, + long_running_task_handles.hardware_manager.baseboard(), + log.new(o!("component" => "MetricsManager")), + )?; + + // Start tracking the underlay physical links. + for nic in underlay::find_nics(&config.data_links)? { + let link_name = nic.interface(); + if let Err(e) = metrics_manager + .track_physical_link( + link_name, + crate::metrics::LINK_SAMPLE_INTERVAL, + ) + .await + { + error!( + log, + "failed to start tracking physical link metrics"; + "link_name" => link_name, + "error" => ?e, + ); + } + } + + // Spawn a task in the background to register our metric producer with + // Nexus. This should not block progress here. + let endpoint = ProducerEndpoint { + id: request.body.id, + kind: ProducerKind::SledAgent, + address: sled_address.into(), + base_route: String::from("/metrics/collect"), + interval: crate::metrics::METRIC_COLLECTION_INTERVAL, + }; + tokio::task::spawn(register_metric_producer_with_nexus( + log.clone(), + nexus_client.clone(), + endpoint, + )); + // Create the PortManager to manage all the OPTE ports on the sled. let port_manager = PortManager::new( parent_log.new(o!("component" => "PortManager")), @@ -416,6 +466,7 @@ impl SledAgent { port_manager.clone(), storage_manager.clone(), long_running_task_handles.zone_bundler.clone(), + metrics_manager.clone(), ZoneBuilderFactory::default(), )?; @@ -513,47 +564,6 @@ impl SledAgent { rack_network_config.clone(), )?; - let mut metrics_manager = MetricsManager::new( - request.body.id, - request.body.rack_id, - long_running_task_handles.hardware_manager.baseboard(), - log.new(o!("component" => "MetricsManager")), - )?; - - // Start tracking the underlay physical links. - for nic in underlay::find_nics(&config.data_links)? { - let link_name = nic.interface(); - if let Err(e) = metrics_manager - .track_physical_link( - link_name, - crate::metrics::LINK_SAMPLE_INTERVAL, - ) - .await - { - error!( - log, - "failed to start tracking physical link metrics"; - "link_name" => link_name, - "error" => ?e, - ); - } - } - - // Spawn a task in the background to register our metric producer with - // Nexus. This should not block progress here. - let endpoint = ProducerEndpoint { - id: request.body.id, - kind: ProducerKind::SledAgent, - address: sled_address.into(), - base_route: String::from("/metrics/collect"), - interval: crate::metrics::METRIC_COLLECTION_INTERVAL, - }; - tokio::task::spawn(register_metric_producer_with_nexus( - log.clone(), - nexus_client.clone(), - endpoint, - )); - let sled_agent = SledAgent { inner: Arc::new(SledAgentInner { id: request.body.id, @@ -909,6 +919,7 @@ impl SledAgent { /// Idempotently ensures that a given instance is registered with this sled, /// i.e., that it can be addressed by future calls to /// [`Self::instance_ensure_state`]. + #[allow(clippy::too_many_arguments)] pub async fn instance_ensure_registered( &self, instance_id: Uuid, @@ -917,6 +928,7 @@ impl SledAgent { instance_runtime: InstanceRuntimeState, vmm_runtime: VmmRuntimeState, propolis_addr: SocketAddr, + metadata: InstanceMetadata, ) -> Result { self.inner .instances @@ -927,6 +939,7 @@ impl SledAgent { instance_runtime, vmm_runtime, propolis_addr, + metadata, ) .await .map_err(|e| Error::Instance(e))