From e19384d6c418b5017a6e7c6b36427bf8fd57e64c Mon Sep 17 00:00:00 2001 From: Ruoqing He Date: Thu, 29 Aug 2024 11:08:38 +0800 Subject: [PATCH 1/2] vmm: Introduce `riscv64` cpu fdt Introduce logic needed to create cpu node for `riscv64` architecture. Signed-off-by: Ruoqing He --- src/vmm/src/arch/mod.rs | 11 ++ src/vmm/src/arch/riscv64/fdt.rs | 173 ++++++++++++++++++++++++++++++++ src/vmm/src/arch/riscv64/mod.rs | 145 ++++++++++++++++++++++++++ 3 files changed, 329 insertions(+) create mode 100644 src/vmm/src/arch/riscv64/fdt.rs create mode 100644 src/vmm/src/arch/riscv64/mod.rs diff --git a/src/vmm/src/arch/mod.rs b/src/vmm/src/arch/mod.rs index f5a2f98cb7c..8e7f02c92cf 100644 --- a/src/vmm/src/arch/mod.rs +++ b/src/vmm/src/arch/mod.rs @@ -16,6 +16,17 @@ pub use aarch64::{ layout::SYSTEM_MEM_START, ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, }; +/// Module for riscv64 related functionality. +#[cfg(target_arch = "riscv64")] +pub mod riscv64; + +#[cfg(target_arch = "riscv64")] +pub use riscv64::{ + arch_memory_regions, configure_system, get_kernel_start, initrd_load_addr, + layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE, + layout::SYSTEM_MEM_START, ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START, +}; + /// Module for x86_64 related functionality. #[cfg(target_arch = "x86_64")] pub mod x86_64; diff --git a/src/vmm/src/arch/riscv64/fdt.rs b/src/vmm/src/arch/riscv64/fdt.rs new file mode 100644 index 00000000000..b973beda92a --- /dev/null +++ b/src/vmm/src/arch/riscv64/fdt.rs @@ -0,0 +1,173 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +use std::collections::HashMap; +use std::ffi::CString; +use std::fmt::Debug; + +use vm_fdt::{Error as VmFdtError, FdtWriter, FdtWriterNode}; +use vm_memory::GuestMemoryError; + +use super::super::{DeviceType, InitrdConfig}; +use super::get_fdt_addr; +use super::aia::AIADevice; +use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; + +const CPU_BASE_PHANDLE: u32 = 0x100; + +const AIA_APLIC_PHANDLE: u32 = 2; +const AIA_IMSIC_PHANDLE: u32 = 3; +const CPU_INTC_BASE_PHANDLE: u32 = 4; +// Read the documentation specified when appending the root node to the FDT. +const ADDRESS_CELLS: u32 = 0x2; +const SIZE_CELLS: u32 = 0x2; + +// From https://elixir.bootlin.com/linux/v6.10/source/include/dt-bindings/interrupt-controller/irq.h#L14 +const IRQ_TYPE_EDGE_RISING: u32 = 1; +const IRQ_TYPE_LEVEL_HI: u32 = 4; + +/// Trait for devices to be added to the Flattened Device Tree. +pub trait DeviceInfoForFDT { + /// Returns the address where this device will be loaded. + fn addr(&self) -> u64; + /// Returns the associated interrupt for this device. + fn irq(&self) -> u32; + /// Returns the amount of memory that needs to be reserved for this device. + fn length(&self) -> u64; +} + +/// Errors thrown while configuring the Flattened Device Tree for riscv64. +#[derive(Debug, thiserror::Error, displaydoc::Display)] +pub enum FdtError { + /// Create FDT error: {0} + CreateFdt(#[from] VmFdtError), + /// Read cache info error: {0} + ReadCacheInfo(String), + /// Failure in writing FDT in memory. + WriteFdtToMemory(#[from] GuestMemoryError), +} + +/// Creates the flattened device tree for this riscv64 microVM. +pub fn create_fdt( + guest_mem: &GuestMemoryMmap, + num_cpus: u32, + cmdline: CString, + device_info: &HashMap<(DeviceType, String), T, S>, + aia_device: &AIADevice, + initrd: &Option, +) -> Result, FdtError> { + // Allocate stuff necessary for storing the blob. + let mut fdt_writer = FdtWriter::new()?; + + // For an explanation why these nodes were introduced in the blob take a look at + // https://github.com/devicetree-org/devicetree-specification/releases/tag/v0.4 + // In chapter 3. + + // Header or the root node as per above mentioned documentation. + let root = fdt_writer.begin_node("")?; + fdt_writer.property_string("compatible", "linux,dummy-virt")?; + // For info on #address-cells and size-cells resort to Table 3.1 Root Node + // Properties + fdt_writer.property_u32("#address-cells", ADDRESS_CELLS)?; + fdt_writer.property_u32("#size-cells", SIZE_CELLS)?; + create_cpu_nodes(&mut fdt_writer, num_cpus)?; + create_memory_node(&mut fdt_writer, guest_mem)?; + create_chosen_node(&mut fdt_writer, cmdline, initrd)?; + create_aia_node(&mut fdt_writer, aia_device)?; + create_devices_node(&mut fdt_writer, device_info)?; + + // End Header node. + fdt_writer.end_node(root)?; + + // Allocate another buffer so we can format and then write fdt to guest. + let fdt_final = fdt_writer.finish()?; + + // Write FDT to memory. + let fdt_address = GuestAddress(get_fdt_addr(guest_mem)); + guest_mem.write_slice(fdt_final.as_slice(), fdt_address)?; + Ok(fdt_final) +} + +// Following are the auxiliary function for creating the different nodes that we append to our FDT. +fn create_cpu_nodes(fdt: &mut FdtWriter, num_cpus: u32) -> Result<(), FdtError> { + // See https://elixir.bootlin.com/linux/v6.10/source/Documentation/devicetree/bindings/riscv/cpus.yaml + let cpus = fdt.begin_node("cpus")?; + // As per documentation, on RISC-V 64-bit systems value should be set to 1. + fdt.property_u32("#address-cells", 0x01)?; + fdt.property_u32("#size-cells", 0x0)?; + // Retrieve CPU frequency from cpu timer regs + let timebase_frequency: u32 = 369999; + fdt.property_u32("timebase-frequency", timebase_frequency); + + for cpu_index in 0..num_cpus { + let cpu = fdt.begin_node(&format!("cpu@{:x}", cpu_index))?; + fdt.property_string("device_type", "cpu")?; + fdt.property_string("compatible", "riscv")?; + fdt.property_string("mmy-type", "sv48")?; + fdt.property_string("riscv,isa", "rv64iafdcsu_smaia_ssaia")?; + fdt.property_string("status", "okay")?; + fdt.property_u64("reg", cpu_index as u64)?; + fdt.property_u32("phandle", CPU_BASE_PHANDLE + cpu_index)?; + fdt.end_node(cpu)?; + + // interrupt controller node + let intc_node = fdt.begin_node("interrupt-controller")?; + fdt.property_string("compatible", "riscv,cpu-intc")?; + fdt.property_u32("#interrupt-cells", 1u32)?; + fdt.property_array_u32("interrupt-controller", &Vec::new())?; + fdt.property_u32("phandle", CPU_INTC_BASE_PHANDLE + cpu_index)?; + fdt.end_node(intc_node)?; + } + fdt.end_node(cpus)?; + + Ok(()) +} + +fn create_memory_node(fdt: &mut FdtWriter, guest_mem: &GuestMemoryMmap) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_chosen_node( + fdt: &mut FdtWriter, + cmdline: CString, + initrd: &Option, +) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_aia_node(fdt: &mut FdtWriter, aia_device: &AIADevice) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_virtio_node( + fdt: &mut FdtWriter, + dev_info: &T, +) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_serial_node( + fdt: &mut FdtWriter, + dev_info: &T, +) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_rtc_node( + fdt: &mut FdtWriter, + dev_info: &T, +) -> Result<(), FdtError> { + unimplemented!() +} + +fn create_devices_node( + fdt: &mut FdtWriter, + dev_info: &HashMap<(DeviceType, String), T, S>, +) -> Result<(), FdtError> { + unimplemented!() +} + +#[cfg(test)] +mod tests { +} diff --git a/src/vmm/src/arch/riscv64/mod.rs b/src/vmm/src/arch/riscv64/mod.rs new file mode 100644 index 00000000000..3375c72fdbd --- /dev/null +++ b/src/vmm/src/arch/riscv64/mod.rs @@ -0,0 +1,145 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod fdt; +// /// Module for the global interrupt controller configuration. +// pub mod aia; +// /// Layout for this riscv64 system. +// pub mod layout; +// /// Logic for configuring riscv64 registers. +// pub mod regs; +// /// Helper methods for VcpuFd. +// pub mod vcpu; + +use std::cmp::min; +use std::collections::HashMap; +use std::ffi::CString; +use std::fmt::Debug; + +pub use self::fdt::DeviceInfoForFDT; +use self::aia::AIADevice; +use crate::arch::DeviceType; +use crate::vstate::memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap}; + +/// Errors thrown while configuring riscv64 system. +#[derive(Debug, thiserror::Error, displaydoc::Display)] +pub enum ConfigurationError { + /// Failed to create a Flattened Device Tree for this riscv64 microVM. + SetupFDT(#[from] fdt::FdtError), + /// Failed to compute the initrd address. + InitrdAddress, +} + +/// The start of the memory area reserved for MMIO devices. +pub const MMIO_MEM_START: u64 = layout::MAPPED_IO_START; +/// The size of the memory area reserved for MMIO devices. +pub const MMIO_MEM_SIZE: u64 = layout::DRAM_MEM_START - layout::MAPPED_IO_START; //>> 1GB + +/// Returns a Vec of the valid memory addresses for riscv64. +/// See [`layout`](layout) module for a drawing of the specific memory model for this platform. +pub fn arch_memory_regions(size: usize) -> Vec<(GuestAddress, usize)> { + let dram_size = min(size, layout::DRAM_MEM_MAX_SIZE); + vec![(GuestAddress(layout::DRAM_MEM_START), dram_size)] +} + +/// Configures the system and should be called once per vm before starting vcpu threads. +/// For riscv64, we only setup the FDT. +/// +/// # Arguments +/// +/// * `guest_mem` - The memory to be used by the guest. +/// * `cmdline_cstring` - The kernel commandline. +/// * `vcpu_mpidr` - Array of MPIDR register values per vcpu. +/// * `device_info` - A hashmap containing the attached devices for building FDT device nodes. +/// * `aia_device` - The AIA device. +/// * `initrd` - Information about an optional initrd. +pub fn configure_system( + guest_mem: &GuestMemoryMmap, + cmdline_cstring: CString, + num_cpus: u32, + device_info: &HashMap<(DeviceType, String), T, S>, + aia_device: &AIADevice, + initrd: &Option, +) -> Result<(), ConfigurationError> { + fdt::create_fdt( + guest_mem, + num_cpus, + cmdline_cstring, + device_info, + aia_device, + initrd, + )?; + Ok(()) +} + +/// Returns the memory address where the kernel could be loaded. +pub fn get_kernel_start() -> u64 { + layout::SYSTEM_MEM_START + layout::SYSTEM_MEM_SIZE +} + +/// Returns the memory address where the initrd could be loaded. +pub fn initrd_load_addr( + guest_mem: &GuestMemoryMmap, + initrd_size: usize, +) -> Result { + let round_to_pagesize = |size| (size + (super::PAGE_SIZE - 1)) & !(super::PAGE_SIZE - 1); + match GuestAddress(get_fdt_addr(guest_mem)).checked_sub(round_to_pagesize(initrd_size) as u64) { + Some(offset) => { + if guest_mem.address_in_range(offset) { + Ok(offset.raw_value()) + } else { + Err(ConfigurationError::InitrdAddress) + } + } + None => Err(ConfigurationError::InitrdAddress), + } +} + +// Auxiliary function to get the address where the device tree blob is loaded. +fn get_fdt_addr(mem: &GuestMemoryMmap) -> u64 { + // If the memory allocated is smaller than the size allocated for the FDT, + // we return the start of the DRAM so that + // we allow the code to try and load the FDT. + + if let Some(addr) = mem.last_addr().checked_sub(layout::FDT_MAX_SIZE as u64 - 1) { + if mem.address_in_range(addr) { + return addr.raw_value(); + } + } + + layout::DRAM_MEM_START +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::utilities::test_utils::arch_mem; + + #[test] + fn test_regions_lt_1024gb() { + let regions = arch_memory_regions(1usize << 29); + assert_eq!(1, regions.len()); + assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0); + assert_eq!(1usize << 29, regions[0].1); + } + + #[test] + fn test_regions_gt_1024gb() { + let regions = arch_memory_regions(1usize << 41); + assert_eq!(1, regions.len()); + assert_eq!(GuestAddress(super::layout::DRAM_MEM_START), regions[0].0); + assert_eq!(super::layout::DRAM_MEM_MAX_SIZE, regions[0].1); + } + + #[test] + fn test_get_fdt_addr() { + let mem = arch_mem(layout::FDT_MAX_SIZE - 0x1000); + assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START); + + let mem = arch_mem(layout::FDT_MAX_SIZE); + assert_eq!(get_fdt_addr(&mem), layout::DRAM_MEM_START); + + let mem = arch_mem(layout::FDT_MAX_SIZE + 0x1000); + assert_eq!(get_fdt_addr(&mem), 0x1000 + layout::DRAM_MEM_START); + } +} From 1dc09f7ece4651dd690894ce13a39a3bdbdc90ac Mon Sep 17 00:00:00 2001 From: Ruoqing He Date: Fri, 30 Aug 2024 18:43:15 +0800 Subject: [PATCH 2/2] vmm: Introduce `riscv64` layout Introduce a minimum vm layout with essential devices and definitions. Signed-off-by: Ruoqing He --- src/vmm/src/arch/riscv64/fdt.rs | 7 +-- src/vmm/src/arch/riscv64/layout.rs | 80 ++++++++++++++++++++++++++++++ src/vmm/src/arch/riscv64/mod.rs | 9 ++-- 3 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 src/vmm/src/arch/riscv64/layout.rs diff --git a/src/vmm/src/arch/riscv64/fdt.rs b/src/vmm/src/arch/riscv64/fdt.rs index b973beda92a..e047b2ffe55 100644 --- a/src/vmm/src/arch/riscv64/fdt.rs +++ b/src/vmm/src/arch/riscv64/fdt.rs @@ -1,4 +1,6 @@ +// Copyright © 2024, Institute of Software, CAS. All rights reserved. // Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// // SPDX-License-Identifier: Apache-2.0 // @@ -10,8 +12,8 @@ use vm_fdt::{Error as VmFdtError, FdtWriter, FdtWriterNode}; use vm_memory::GuestMemoryError; use super::super::{DeviceType, InitrdConfig}; -use super::get_fdt_addr; use super::aia::AIADevice; +use super::get_fdt_addr; use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; const CPU_BASE_PHANDLE: u32 = 0x100; @@ -169,5 +171,4 @@ fn create_devices_node