feat(virtio): configure net device

- MMIO Device Discovery by the kernel command line
- irq allocator

Signed-off-by: sylvain-pierrot <[email protected]>

Cargo.toml

@@ -1,3 +1,3 @@
 [workspace]
-members = ["src/api", "src/vmm", "src/cli"]
+members = ["src/api", "src/cli", "src/vmm"]
 resolver = "2"

src/vmm/Cargo.toml

@@ -23,6 +23,12 @@ openpty = "0.2.0"
 tracing = "0.1.40"
 tracing-subscriber = "0.3.18"
 virtio-bindings = "0.2.2"
+virtio-blk = { git = "https://github.com/rust-vmm/vm-virtio.git", features = [
+    "backend-stdio",
+] }
+virtio-device = { git = "https://github.com/rust-vmm/vm-virtio.git" }
+virtio-queue = { git = "https://github.com/rust-vmm/vm-virtio.git" }
+vm-allocator = "0.1.0"
 vm-device = "0.1.0"
 vm-memory = { version = "0.14.0", features = ["backend-mmap"] }
 vm-superio = "0.7.0"

src/vmm/src/core/devices/mod.rs

@@ -3,6 +3,7 @@
 use std::io;
 
 pub(crate) mod serial;
+pub mod virtio;
 
 #[derive(Debug)]
 /// Devices errors.

src/vmm/src/core/devices/virtio/mod.rs

@@ -0,0 +1,49 @@
+use linux_loader::cmdline::Error;
+use std::result;
+use vm_memory::{Address, GuestAddress, GuestUsize};
+
+pub mod net;
+
+pub type Result<T> = result::Result<T, Error>;
+
+pub fn register_mmio_device(
+    size: GuestUsize,
+    baseaddr: GuestAddress,
+    irq: u32,
+    id: Option<u32>,
+) -> Result<String> {
+    // !TODO Register to MmioManager
+
+    // Pass to kernel command line
+    if size == 0 {
+        return Err(Error::MmioSize);
+    }
+
+    let mut device_str = format!(
+        "virtio_mmio.device={}@0x{:x?}:{}",
+        guestusize_to_str(size),
+        baseaddr.raw_value(),
+        irq
+    );
+    if let Some(id) = id {
+        device_str.push_str(format!(":{}", id).as_str());
+    }
+    Ok(device_str)
+}
+
+fn guestusize_to_str(size: GuestUsize) -> String {
+    const KB_MULT: u64 = 1 << 10;
+    const MB_MULT: u64 = KB_MULT << 10;
+    const GB_MULT: u64 = MB_MULT << 10;
+
+    if size % GB_MULT == 0 {
+        return format!("{}G", size / GB_MULT);
+    }
+    if size % MB_MULT == 0 {
+        return format!("{}M", size / MB_MULT);
+    }
+    if size % KB_MULT == 0 {
+        return format!("{}K", size / KB_MULT);
+    }
+    size.to_string()
+}

src/vmm/src/core/devices/virtio/net/device.rs

@@ -0,0 +1,23 @@
+use vm_memory::{GuestAddress, GuestUsize};
+
+use crate::core::devices::virtio::register_mmio_device;
+
+pub struct Net {
+    vmmio_parameter: String,
+}
+
+impl Net {
+    pub fn new(
+        size: GuestUsize,
+        baseaddr: GuestAddress,
+        irq: u32,
+    ) -> Result<Self, linux_loader::cmdline::Error> {
+        let vmmio_parameter = register_mmio_device(size, baseaddr, irq, None).unwrap();
+
+        Ok(Self { vmmio_parameter })
+    }
+
+    pub fn get_vmmio_parameter(&self) -> String {
+        self.vmmio_parameter.clone()
+    }
+}

src/vmm/src/core/devices/virtio/net/mod.rs

@@ -0,0 +1 @@
+pub mod device;

src/vmm/src/core/irq_allocator.rs

@@ -0,0 +1,64 @@
+// Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR BSD-3-Clause
+
+use std::fmt;
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum Error {
+    InvalidValue,
+    MaxIrq,
+    IRQOverflowed,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+/// An irq allocator which gives next available irq.
+/// It is mainly used for non-legacy devices.
+// There are a few reserved irq's on x86_64. We just skip all the inital
+// reserved irq to make the implementaion simple. This could be later extended
+// to cater more complex scenario.
+#[derive(Debug)]
+pub struct IrqAllocator {
+    // Tracks the last allocated irq
+    last_used_irq: u32,
+    last_irq: u32,
+}
+
+impl IrqAllocator {
+    pub fn new(last_used_irq: u32, last_irq: u32) -> Result<Self> {
+        if last_used_irq >= last_irq {
+            return Err(Error::InvalidValue);
+        }
+        Ok(IrqAllocator {
+            last_used_irq,
+            last_irq,
+        })
+    }
+
+    pub fn next_irq(&mut self) -> Result<u32> {
+        self.last_used_irq
+            .checked_add(1)
+            .ok_or(Error::IRQOverflowed)
+            .and_then(|irq| {
+                if irq > self.last_irq {
+                    Err(Error::MaxIrq)
+                } else {
+                    self.last_used_irq = irq;
+                    Ok(irq)
+                }
+            })
+    }
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let err = match self {
+            Error::MaxIrq => "last_irq IRQ limit reached",
+            Error::IRQOverflowed => "IRQ overflowed",
+            Error::InvalidValue => {
+                "Check the value of last_used and last_irq. las_used should be less than last_irq"
+            }
+        };
+        write!(f, "{}", err) // user-facing output
+    }
+}

src/vmm/src/core/kernel.rs

@@ -42,7 +42,7 @@ const HIMEM_START: u64 = 0x0010_0000; // 1 MB
 /// Address where the kernel command line is written.
 const CMDLINE_START: u64 = 0x0002_0000;
 // Default command line
-const CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off";
+const DEFAULT_CMDLINE: &str = "console=ttyS0 i8042.nokbd reboot=k panic=1 pci=off";
 
 fn add_e820_entry(
     params: &mut boot_params,
@@ -127,6 +127,7 @@ pub fn kernel_setup(
     guest_memory: &GuestMemoryMmap,
     kernel_path: PathBuf,
     initramfs_path: Option<PathBuf>,
+    cmdline_extra_parameters: Vec<String>,
 ) -> Result<KernelLoaderResult> {
     let mut kernel_image = File::open(kernel_path).map_err(Error::IO)?;
     let zero_page_addr = GuestAddress(ZEROPG_START);
@@ -143,13 +144,24 @@ pub fn kernel_setup(
     // Generate boot parameters.
     let mut bootparams = build_bootparams(guest_memory, GuestAddress(HIMEM_START))?;
 
+    let combined_cmdline: String = {
+        let mut combined = DEFAULT_CMDLINE.to_string();
+        for param in &cmdline_extra_parameters {
+            combined.push(' ');
+            combined.push_str(param);
+        }
+        combined
+    };
+
     // Add the kernel command line to the boot parameters.
     bootparams.hdr.cmd_line_ptr = CMDLINE_START as u32;
-    bootparams.hdr.cmdline_size = CMDLINE.len() as u32 + 1;
+    bootparams.hdr.cmdline_size = combined_cmdline.len() as u32 + 1;
 
     // Load the kernel command line into guest memory.
-    let mut cmdline = Cmdline::new(CMDLINE.len() + 1).map_err(Error::Cmdline)?;
-    cmdline.insert_str(CMDLINE).map_err(Error::Cmdline)?;
+    let mut cmdline = Cmdline::new(combined_cmdline.len() + 1).map_err(Error::Cmdline)?;
+    cmdline
+        .insert_str(combined_cmdline)
+        .map_err(Error::Cmdline)?;
     load_cmdline(
         guest_memory,
         GuestAddress(CMDLINE_START),

src/vmm/src/core/mod.rs

@@ -13,6 +13,7 @@ use self::network::open_tap;
 mod cpu;
 mod devices;
 mod epoll_context;
+mod irq_allocator;
 mod kernel;
 mod network;
 mod slip_pty;

src/vmm/src/core/vmm.rs

@@ -16,17 +16,41 @@ use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 use std::thread;
 use tracing::info;
+use vm_allocator::{AddressAllocator, AllocPolicy};
+use vm_device::bus::{MmioAddress, MmioRange};
 use vm_memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap, GuestMemoryRegion};
 use vmm_sys_util::terminal::Terminal;
 
+use super::devices::virtio::net::device::Net;
+use super::irq_allocator::IrqAllocator;
 use super::network::open_tap::open_tap;
 use super::network::tap::Tap;
 use super::slip_pty::SlipPty;
 
+#[cfg(target_arch = "x86_64")]
+pub(crate) const MMIO_GAP_END: u64 = 1 << 32;
+/// Size of the MMIO gap.
+#[cfg(target_arch = "x86_64")]
+pub(crate) const MMIO_GAP_SIZE: u64 = 768 << 20;
+/// The start of the MMIO gap (memory area reserved for MMIO devices).
+#[cfg(target_arch = "x86_64")]
+pub(crate) const MMIO_GAP_START: u64 = MMIO_GAP_END - MMIO_GAP_SIZE;
+/// Default address allocator alignment. It needs to be a power of 2.
+pub const DEFAULT_ADDRESS_ALIGNEMNT: u64 = 4;
+/// Default allocation policy for address allocator.
+pub const DEFAULT_ALLOC_POLICY: AllocPolicy = AllocPolicy::FirstMatch;
+/// IRQ line 4 is typically used for serial port 1.
+// See more IRQ assignments & info: https://tldp.org/HOWTO/Serial-HOWTO-8.html
+const SERIAL_IRQ: u32 = 4;
+/// Last usable IRQ ID for virtio device interrupts on x86_64.
+const IRQ_MAX: u8 = 23;
+
 pub struct VMM {
     vm_fd: VmFd,
     kvm: Kvm,
     guest_memory: GuestMemoryMmap,
+    address_allocator: Option<AddressAllocator>,
+    irq_allocator: IrqAllocator,
     vcpus: Vec<Vcpu>,
     _tap: Tap,
 
@@ -72,10 +96,14 @@ impl VMM {
             )
             .map_err(Error::EpollError)?;
 
+        let irq_allocator = IrqAllocator::new(SERIAL_IRQ, IRQ_MAX.into()).unwrap();
+
         let vmm = VMM {
             vm_fd,
             kvm,
             guest_memory: GuestMemoryMmap::default(),
+            address_allocator: None,
+            irq_allocator,
             vcpus: vec![],
             _tap: tap,
             serial: Arc::new(Mutex::new(
@@ -121,6 +149,19 @@ impl VMM {
         Ok(())
     }
 
+    fn configure_allocators(&mut self, mem_size_mb: u32) -> Result<()> {
+        // Convert memory size from MBytes to bytes.
+        let mem_size = (mem_size_mb as u64) << 20;
+
+        // Setup address allocator.
+        let start_addr = MMIO_GAP_START;
+        let address_allocator = AddressAllocator::new(start_addr, mem_size).unwrap();
+
+        self.address_allocator = Some(address_allocator);
+
+        Ok(())
+    }
+
     fn configure_io(&mut self) -> Result<()> {
         // First, create the irqchip.
         // On `x86_64`, this _must_ be created _before_ the vCPUs.
@@ -277,15 +318,43 @@ impl VMM {
         kernel_path: &Path,
         initramfs_path: &Option<PathBuf>,
     ) -> Result<()> {
+        let mut cmdline_extra_parameters = Vec::new();
+
         self.configure_memory(mem_size_mb)?;
+        self.configure_allocators(mem_size_mb)?;
+
+        let vmmio_parameter = self.configure_net_device().unwrap();
+        cmdline_extra_parameters.push(vmmio_parameter);
+
         let kernel_load = kernel::kernel_setup(
             &self.guest_memory,
             kernel_path.to_path_buf(),
             initramfs_path.clone(),
+            cmdline_extra_parameters,
         )?;
         self.configure_io()?;
         self.configure_vcpus(num_vcpus, kernel_load)?;
 
         Ok(())
     }
+
+    pub fn configure_net_device(&mut self) -> Result<String> {
+        let _mem = Arc::new(self.guest_memory.clone());
+        let range = if let Some(allocator) = &self.address_allocator {
+            allocator
+                .to_owned()
+                .allocate(0x1000, DEFAULT_ADDRESS_ALIGNEMNT, DEFAULT_ALLOC_POLICY)
+                .unwrap()
+        } else {
+            // Handle the case where self.address_allocator is None
+            panic!("Address allocator is not initialized");
+        };
+        let _mmio_range = MmioRange::new(MmioAddress(range.start()), range.len()).unwrap();
+        let irq = self.irq_allocator.next_irq().unwrap();
+
+        // !TODO: MMIO Device Discovery + MMIO Device Register Layout
+        let net = Net::new(range.len(), GuestAddress(range.start()), irq).unwrap();
+
+        Ok(net.get_vmmio_parameter())
+    }
 }