diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 2473c14566..ce6143bc91 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -51,6 +51,9 @@ pub struct Config { pub sidecar_revision: SidecarRevision, /// Optional percentage of DRAM to reserve for guest memory pub vmm_reservoir_percentage: Option, + /// Optional DRAM to reserve for guest memory in MiB (cannot be used with + /// vmm_reservoir_percentage). + pub vmm_reservoir_size_mb: Option, /// Optional swap device size in GiB pub swap_device_size_gb: Option, /// Optional VLAN ID to be used for tagging guest VNICs. diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index bdd29e4d1f..c2e7ed2662 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -40,6 +40,9 @@ pub enum Error { #[error("Failed to create reservoir: {0}")] Reservoir(#[from] vmm_reservoir::Error), + #[error("Invalid reservoir configuration: {0}")] + ReservoirConfig(String), + #[error("Cannot find data link: {0}")] Underlay(#[from] sled_hardware::underlay::Error), @@ -47,6 +50,12 @@ pub enum Error { ZoneBundle(#[from] BundleError), } +pub enum ReservoirMode { + None, + Size(u32), + Percentage(u8), +} + struct InstanceManagerInternal { log: Logger, nexus_client: NexusClientWithResolver, @@ -97,44 +106,68 @@ impl InstanceManager { }) } - /// Sets the VMM reservoir size to the requested (nonzero) percentage of - /// usable physical RAM, rounded down to nearest aligned size required by - /// the control plane. + /// Sets the VMM reservoir to the requested percentage of usable physical + /// RAM or to a size in MiB. Either mode will round down to the nearest + /// aligned size required by the control plane. pub fn set_reservoir_size( &self, hardware: &sled_hardware::HardwareManager, - target_percent: u8, + mode: ReservoirMode, ) -> Result<(), Error> { - assert!( - target_percent > 0 && target_percent < 100, - "target_percent {} must be nonzero and < 100", - target_percent - ); + let hardware_physical_ram_bytes = hardware.usable_physical_ram_bytes(); + let req_bytes = match mode { + ReservoirMode::None => return Ok(()), + ReservoirMode::Size(mb) => { + let bytes = ByteCount::from_mebibytes_u32(mb).to_bytes(); + if bytes > hardware_physical_ram_bytes { + return Err(Error::ReservoirConfig(format!( + "cannot specify a reservoir of {bytes} bytes when \ + physical memory is {hardware_physical_ram_bytes} bytes", + ))); + } + bytes + } + ReservoirMode::Percentage(percent) => { + if !matches!(percent, 1..=99) { + return Err(Error::ReservoirConfig(format!( + "reservoir percentage of {} must be between 0 and 100", + percent + ))); + }; + (hardware_physical_ram_bytes as f64 * (percent as f64 / 100.0)) + .floor() as u64 + } + }; - let req_bytes = (hardware.usable_physical_ram_bytes() as f64 - * (target_percent as f64 / 100.0)) - .floor() as u64; let req_bytes_aligned = vmm_reservoir::align_reservoir_size(req_bytes); if req_bytes_aligned == 0 { warn!( self.inner.log, - "Requested reservoir size of {} bytes < minimum aligned size of {} bytes", - req_bytes, vmm_reservoir::RESERVOIR_SZ_ALIGN); + "Requested reservoir size of {} bytes < minimum aligned size \ + of {} bytes", + req_bytes, + vmm_reservoir::RESERVOIR_SZ_ALIGN + ); return Ok(()); } - // The max ByteCount value is i64::MAX, which is ~8 million TiB. As this - // value is a percentage of DRAM, constructing this should always work. + // The max ByteCount value is i64::MAX, which is ~8 million TiB. + // As this value is either a percentage of DRAM or a size in MiB + // represented as a u32, constructing this should always work. let reservoir_size = ByteCount::try_from(req_bytes_aligned).unwrap(); + if let ReservoirMode::Percentage(percent) = mode { + info!( + self.inner.log, + "{}% of {} physical ram = {} bytes)", + percent, + hardware_physical_ram_bytes, + req_bytes, + ); + } info!( self.inner.log, - "Setting reservoir size to {} bytes \ - ({}% of {} total = {} bytes requested)", - reservoir_size, - target_percent, - hardware.usable_physical_ram_bytes(), - req_bytes, + "Setting reservoir size to {reservoir_size} bytes" ); vmm_reservoir::ReservoirControl::set(reservoir_size)?; diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 397778a663..6762d1d209 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -9,7 +9,7 @@ use crate::bootstrap::early_networking::{ }; use crate::bootstrap::params::StartSledAgentRequest; use crate::config::Config; -use crate::instance_manager::InstanceManager; +use crate::instance_manager::{InstanceManager, ReservoirMode}; use crate::nexus::{NexusClientWithResolver, NexusRequestQueue}; use crate::params::{ DiskStateRequested, InstanceHardware, InstanceMigrationSourceParams, @@ -346,21 +346,33 @@ impl SledAgent { storage.zone_bundler().clone(), )?; - match config.vmm_reservoir_percentage { - Some(sz) if sz > 0 && sz < 100 => { - instances.set_reservoir_size(&hardware, sz).map_err(|e| { - error!(log, "Failed to set VMM reservoir size: {e}"); - e - })?; - } - Some(sz) if sz == 0 => { - warn!(log, "Not using VMM reservoir (size 0 bytes requested)"); - } - None => { - warn!(log, "Not using VMM reservoir"); + // Configure the VMM reservoir as either a percentage of DRAM or as an + // exact size in MiB. + let reservoir_mode = match ( + config.vmm_reservoir_percentage, + config.vmm_reservoir_size_mb, + ) { + (None, None) => ReservoirMode::None, + (Some(p), None) => ReservoirMode::Percentage(p), + (None, Some(mb)) => ReservoirMode::Size(mb), + (Some(_), Some(_)) => panic!( + "cannot specify vmm_reservoir_percentage and \ + vmm_reservoir_size_mb at the same time" + ), + }; + + match reservoir_mode { + ReservoirMode::None => warn!(log, "Not using VMM reservoir"), + ReservoirMode::Size(0) | ReservoirMode::Percentage(0) => { + warn!(log, "Not using VMM reservoir (size 0 bytes requested)") } - Some(sz) => { - panic!("invalid requested VMM reservoir percentage: {}", sz); + _ => { + instances + .set_reservoir_size(&hardware, reservoir_mode) + .map_err(|e| { + error!(log, "Failed to setup VMM reservoir: {e}"); + e + })?; } } diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index b4cb7e6cff..73cd31bdd3 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -45,6 +45,11 @@ zpools = [ # guest memory is pulled from. vmm_reservoir_percentage = 50 +# Optionally you can specify the size of the VMM reservoir in MiB. +# Note vmm_reservoir_percentage and vmm_reservoir_size_mb cannot be specified +# at the same time. +#vmm_reservoir_size_mb = 2048 + # Swap device size for the system. The device is a sparsely allocated zvol on # the internal zpool of the M.2 that we booted from. #