diff --git a/Cargo.toml b/Cargo.toml index 94d73a2..1eb9522 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/api", "src/vmm", "src/cli"] +members = ["src/api", "src/cli", "src/vmm"] resolver = "2" diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 134a717..e4965a5 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -23,6 +23,12 @@ openpty = "0.2.0" tracing = "0.1.40" tracing-subscriber = "0.3.18" virtio-bindings = "0.2.2" +virtio-blk = { git = "https://github.com/rust-vmm/vm-virtio.git", features = [ + "backend-stdio", +] } +virtio-device = { git = "https://github.com/rust-vmm/vm-virtio.git" } +virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio.git" } +vm-allocator = "0.1.0" vm-device = "0.1.0" vm-memory = { version = "0.14.0", features = ["backend-mmap"] } vm-superio = "0.7.0" diff --git a/src/vmm/src/core/devices/mod.rs b/src/vmm/src/core/devices/mod.rs index ee70983..ebf9e66 100644 --- a/src/vmm/src/core/devices/mod.rs +++ b/src/vmm/src/core/devices/mod.rs @@ -3,6 +3,7 @@ use std::io; pub(crate) mod serial; +pub mod virtio; #[derive(Debug)] /// Devices errors. diff --git a/src/vmm/src/core/devices/virtio/mod.rs b/src/vmm/src/core/devices/virtio/mod.rs new file mode 100644 index 0000000..77af8c1 --- /dev/null +++ b/src/vmm/src/core/devices/virtio/mod.rs @@ -0,0 +1,49 @@ +use linux_loader::cmdline::Error; +use std::result; +use vm_memory::{Address, GuestAddress, GuestUsize}; + +pub mod net; + +pub type Result = result::Result; + +pub fn register_mmio_device( + size: GuestUsize, + baseaddr: GuestAddress, + irq: u32, + id: Option, +) -> Result { + // !TODO Register to MmioManager + + // Pass to kernel command line + if size == 0 { + return Err(Error::MmioSize); + } + + let mut device_str = format!( + "virtio_mmio.device={}@0x{:x?}:{}", + guestusize_to_str(size), + baseaddr.raw_value(), + irq + ); + if let Some(id) = id { + device_str.push_str(format!(":{}", id).as_str()); + } + Ok(device_str) +} + +fn guestusize_to_str(size: GuestUsize) -> String { + const KB_MULT: u64 = 1 << 10; + const MB_MULT: u64 = KB_MULT << 10; + const GB_MULT: u64 = MB_MULT << 10; + + if size % GB_MULT == 0 { + return format!("{}G", size / GB_MULT); + } + if size % MB_MULT == 0 { + return format!("{}M", size / MB_MULT); + } + if size % KB_MULT == 0 { + return format!("{}K", size / KB_MULT); + } + size.to_string() +} diff --git a/src/vmm/src/core/devices/virtio/net/device.rs b/src/vmm/src/core/devices/virtio/net/device.rs new file mode 100644 index 0000000..037c6a7 --- /dev/null +++ b/src/vmm/src/core/devices/virtio/net/device.rs @@ -0,0 +1,23 @@ +use vm_memory::{GuestAddress, GuestUsize}; + +use crate::core::devices::virtio::register_mmio_device; + +pub struct Net { + vmmio_parameter: String, +} + +impl Net { + pub fn new( + size: GuestUsize, + baseaddr: GuestAddress, + irq: u32, + ) -> Result { + let vmmio_parameter = register_mmio_device(size, baseaddr, irq, None).unwrap(); + + Ok(Self { vmmio_parameter }) + } + + pub fn get_vmmio_parameter(&self) -> String { + self.vmmio_parameter.clone() + } +} diff --git a/src/vmm/src/core/devices/virtio/net/mod.rs b/src/vmm/src/core/devices/virtio/net/mod.rs new file mode 100644 index 0000000..5458924 --- /dev/null +++ b/src/vmm/src/core/devices/virtio/net/mod.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/src/vmm/src/core/irq_allocator.rs b/src/vmm/src/core/irq_allocator.rs new file mode 100644 index 0000000..aff5613 --- /dev/null +++ b/src/vmm/src/core/irq_allocator.rs @@ -0,0 +1,64 @@ +// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause + +use std::fmt; + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + InvalidValue, + MaxIrq, + IRQOverflowed, +} + +pub type Result = std::result::Result; + +/// An irq allocator which gives next available irq. +/// It is mainly used for non-legacy devices. +// There are a few reserved irq's on x86_64. We just skip all the inital +// reserved irq to make the implementaion simple. This could be later extended +// to cater more complex scenario. +#[derive(Debug)] +pub struct IrqAllocator { + // Tracks the last allocated irq + last_used_irq: u32, + last_irq: u32, +} + +impl IrqAllocator { + pub fn new(last_used_irq: u32, last_irq: u32) -> Result { + if last_used_irq >= last_irq { + return Err(Error::InvalidValue); + } + Ok(IrqAllocator { + last_used_irq, + last_irq, + }) + } + + pub fn next_irq(&mut self) -> Result { + self.last_used_irq + .checked_add(1) + .ok_or(Error::IRQOverflowed) + .and_then(|irq| { + if irq > self.last_irq { + Err(Error::MaxIrq) + } else { + self.last_used_irq = irq; + Ok(irq) + } + }) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let err = match self { + Error::MaxIrq => "last_irq IRQ limit reached", + Error::IRQOverflowed => "IRQ overflowed", + Error::InvalidValue => { + "Check the value of last_used and last_irq. las_used should be less than last_irq" + } + }; + write!(f, "{}", err) // user-facing output + } +} diff --git a/src/vmm/src/core/kernel.rs b/src/vmm/src/core/kernel.rs index c9b82ea..3d74a56 100644 --- a/src/vmm/src/core/kernel.rs +++ b/src/vmm/src/core/kernel.rs @@ -42,7 +42,7 @@ const HIMEM_START: u64 = 0x0010_0000; // 1 MB /// Address where the kernel command line is written. const CMDLINE_START: u64 = 0x0002_0000; // Default command line -const CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off"; +const DEFAULT_CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off"; fn add_e820_entry( params: &mut boot_params, @@ -127,6 +127,7 @@ pub fn kernel_setup( guest_memory: &GuestMemoryMmap, kernel_path: PathBuf, initramfs_path: Option, + cmdline_extra_parameters: Vec, ) -> Result { let mut kernel_image = File::open(kernel_path).map_err(Error::IO)?; let zero_page_addr = GuestAddress(ZEROPG_START); @@ -143,13 +144,24 @@ pub fn kernel_setup( // Generate boot parameters. let mut bootparams = build_bootparams(guest_memory, GuestAddress(HIMEM_START))?; + let combined_cmdline: String = { + let mut combined = DEFAULT_CMDLINE.to_string(); + for param in &cmdline_extra_parameters { + combined.push(' '); + combined.push_str(param); + } + combined + }; + // Add the kernel command line to the boot parameters. bootparams.hdr.cmd_line_ptr = CMDLINE_START as u32; - bootparams.hdr.cmdline_size = CMDLINE.len() as u32 + 1; + bootparams.hdr.cmdline_size = combined_cmdline.len() as u32 + 1; // Load the kernel command line into guest memory. - let mut cmdline = Cmdline::new(CMDLINE.len() + 1).map_err(Error::Cmdline)?; - cmdline.insert_str(CMDLINE).map_err(Error::Cmdline)?; + let mut cmdline = Cmdline::new(combined_cmdline.len() + 1).map_err(Error::Cmdline)?; + cmdline + .insert_str(combined_cmdline) + .map_err(Error::Cmdline)?; load_cmdline( guest_memory, GuestAddress(CMDLINE_START), diff --git a/src/vmm/src/core/mod.rs b/src/vmm/src/core/mod.rs index 9fdbcbf..aa28d7b 100644 --- a/src/vmm/src/core/mod.rs +++ b/src/vmm/src/core/mod.rs @@ -13,6 +13,7 @@ use self::network::open_tap; mod cpu; mod devices; mod epoll_context; +mod irq_allocator; mod kernel; mod network; mod slip_pty; diff --git a/src/vmm/src/core/vmm.rs b/src/vmm/src/core/vmm.rs index d541148..d8be957 100644 --- a/src/vmm/src/core/vmm.rs +++ b/src/vmm/src/core/vmm.rs @@ -16,17 +16,41 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::thread; use tracing::info; +use vm_allocator::{AddressAllocator, AllocPolicy}; +use vm_device::bus::{MmioAddress, MmioRange}; use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion}; use vmm_sys_util::terminal::Terminal; +use super::devices::virtio::net::device::Net; +use super::irq_allocator::IrqAllocator; use super::network::open_tap::open_tap; use super::network::tap::Tap; use super::slip_pty::SlipPty; +#[cfg(target_arch = "x86_64")] +pub(crate) const MMIO_GAP_END: u64 = 1 << 32; +/// Size of the MMIO gap. +#[cfg(target_arch = "x86_64")] +pub(crate) const MMIO_GAP_SIZE: u64 = 768 << 20; +/// The start of the MMIO gap (memory area reserved for MMIO devices). +#[cfg(target_arch = "x86_64")] +pub(crate) const MMIO_GAP_START: u64 = MMIO_GAP_END - MMIO_GAP_SIZE; +/// Default address allocator alignment. It needs to be a power of 2. +pub const DEFAULT_ADDRESS_ALIGNEMNT: u64 = 4; +/// Default allocation policy for address allocator. +pub const DEFAULT_ALLOC_POLICY: AllocPolicy = AllocPolicy::FirstMatch; +/// IRQ line 4 is typically used for serial port 1. +// See more IRQ assignments & info: https://tldp.org/HOWTO/Serial-HOWTO-8.html +const SERIAL_IRQ: u32 = 4; +/// Last usable IRQ ID for virtio device interrupts on x86_64. +const IRQ_MAX: u8 = 23; + pub struct VMM { vm_fd: VmFd, kvm: Kvm, guest_memory: GuestMemoryMmap, + address_allocator: Option, + irq_allocator: IrqAllocator, vcpus: Vec, _tap: Tap, @@ -72,10 +96,14 @@ impl VMM { ) .map_err(Error::EpollError)?; + let irq_allocator = IrqAllocator::new(SERIAL_IRQ, IRQ_MAX.into()).unwrap(); + let vmm = VMM { vm_fd, kvm, guest_memory: GuestMemoryMmap::default(), + address_allocator: None, + irq_allocator, vcpus: vec![], _tap: tap, serial: Arc::new(Mutex::new( @@ -121,6 +149,19 @@ impl VMM { Ok(()) } + fn configure_allocators(&mut self, mem_size_mb: u32) -> Result<()> { + // Convert memory size from MBytes to bytes. + let mem_size = (mem_size_mb as u64) << 20; + + // Setup address allocator. + let start_addr = MMIO_GAP_START; + let address_allocator = AddressAllocator::new(start_addr, mem_size).unwrap(); + + self.address_allocator = Some(address_allocator); + + Ok(()) + } + fn configure_io(&mut self) -> Result<()> { // First, create the irqchip. // On `x86_64`, this _must_ be created _before_ the vCPUs. @@ -277,15 +318,43 @@ impl VMM { kernel_path: &Path, initramfs_path: &Option, ) -> Result<()> { + let mut cmdline_extra_parameters = Vec::new(); + self.configure_memory(mem_size_mb)?; + self.configure_allocators(mem_size_mb)?; + + let vmmio_parameter = self.configure_net_device().unwrap(); + cmdline_extra_parameters.push(vmmio_parameter); + let kernel_load = kernel::kernel_setup( &self.guest_memory, kernel_path.to_path_buf(), initramfs_path.clone(), + cmdline_extra_parameters, )?; self.configure_io()?; self.configure_vcpus(num_vcpus, kernel_load)?; Ok(()) } + + pub fn configure_net_device(&mut self) -> Result { + let _mem = Arc::new(self.guest_memory.clone()); + let range = if let Some(allocator) = &self.address_allocator { + allocator + .to_owned() + .allocate(0x1000, DEFAULT_ADDRESS_ALIGNEMNT, DEFAULT_ALLOC_POLICY) + .unwrap() + } else { + // Handle the case where self.address_allocator is None + panic!("Address allocator is not initialized"); + }; + let _mmio_range = MmioRange::new(MmioAddress(range.start()), range.len()).unwrap(); + let irq = self.irq_allocator.next_irq().unwrap(); + + // !TODO: MMIO Device Discovery + MMIO Device Register Layout + let net = Net::new(range.len(), GuestAddress(range.start()), irq).unwrap(); + + Ok(net.get_vmmio_parameter()) + } }