diff --git a/Cargo.lock b/Cargo.lock index 8e5072c9c4..f8a33432e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,12 @@ dependencies = [ "tock-registers 0.7.0", ] +[[package]] +name = "critical-section" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" + [[package]] name = "crossbeam-utils" version = "0.8.12" @@ -91,6 +97,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + +[[package]] +name = "exclusive_cell" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b9e9908e50b47ebbc3d6fd66ed295b997c270e8d2312a035bcc62722a160ef" + [[package]] name = "fastrand" version = "1.8.0" @@ -100,6 +122,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fdt" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784a4df722dc6267a04af36895398f59d21d07dce47232adf31ec0ff2fa45e67" + [[package]] name = "float-cmp" version = "0.9.0" @@ -173,14 +201,13 @@ dependencies = [ [[package]] name = "hermit-sync" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9b3a836d168ff283be2421bb51b8dd8197f9f44a5be14a346c879d7e62fe2b" +version = "0.1.2" dependencies = [ "aarch64-cpu", - "crossbeam-utils", + "exclusive_cell", "generic_once_cell", "lock_api", + "riscv", "tock-registers 0.8.1", "x86_64", ] @@ -221,6 +248,7 @@ dependencies = [ "async-task", "bitflags", "crossbeam-utils", + "fdt", "float-cmp", "futures-lite", "hashbrown", @@ -234,10 +262,13 @@ dependencies = [ "num-traits", "pci-ids", "qemu-exit", + "riscv", "scopeguard", "shell-words", "smoltcp", "time", + "tock-registers 0.8.1", + "trapframe", "x86", "x86_64", ] @@ -294,6 +325,21 @@ dependencies = [ "paste", ] +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.1.0", +] + +[[package]] +name = "nb" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5439c4ad607c3c23abf66de8c8bf57ba8adcd1f129e699851a6e43935d339d" + [[package]] name = "nom" version = "7.1.1" @@ -540,6 +586,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "riscv" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa3145d2fae3778b1e31ec2e827b228bdc6abd9b74bb5705ba46dcb82069bc4f" +dependencies = [ + "bit_field", + "critical-section", + "embedded-hal", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -647,6 +704,16 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" +[[package]] +name = "trapframe" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f287ee70169f5bfddba441baf901b620e3655f16fa7815f48a7e100ec6d86a8f" +dependencies = [ + "raw-cpuid", + "x86_64", +] + [[package]] name = "unicode-ident" version = "1.0.5" @@ -659,6 +726,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "volatile" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 6e06ad58e7..83ef1b0734 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,6 @@ ahash = { version = "0.8", default-features = false } bitflags = "1.3" crossbeam-utils = { version = "0.8", default-features = false } hashbrown = { version = "0.13", default-features = false } -hermit-entry = { version = "0.9", features = ["kernel"] } hermit-sync = "0.1.0" include-transformed = { version = "0.2", optional = true } log = { version = "0.4", default-features = false } @@ -85,6 +84,9 @@ qemu-exit = "3.0" futures-lite = { version = "1.11", default-features = false, optional = true } async-task = { version = "4.3", default-features = false, optional = true } +[target.'cfg(not(target_arch = "riscv64"))'.dependencies] +hermit-entry = { version = "0.9", features = ["kernel"] } + [dependencies.smoltcp] version = "0.8" optional = true @@ -117,6 +119,12 @@ x86 = { version = "0.52", default-features = false } version = "0.0.7" default-features = false +[target.'cfg(target_arch = "riscv64")'.dependencies] +riscv = "0.10" +trapframe = "0.9" +fdt = "0.1" +tock-registers = "0.8" + [workspace] members = [ "xtask", @@ -124,3 +132,6 @@ members = [ exclude = [ "hermit-builtins", ] + +[patch.crates-io] +hermit-sync = { path = "../../hermit-sync" } diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 4d724ff28f..c9a6d32095 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -2,6 +2,9 @@ #[cfg(target_arch = "aarch64")] pub mod aarch64; +#[cfg(target_arch = "riscv64")] +pub mod riscv; + #[cfg(target_arch = "x86_64")] pub mod x86_64; @@ -29,6 +32,29 @@ pub use crate::arch::aarch64::kernel::{ }; #[cfg(target_arch = "aarch64")] pub use crate::arch::aarch64::*; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::irq; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::percore; +#[cfg(target_arch = "riscv64")] +use crate::arch::riscv::kernel::percore::core_scheduler; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::processor; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::processor::{set_oneshot_timer, wakeup_core}; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::scheduler; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::switch; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::systemtime::get_boot_time; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::kernel::{ + application_processor_init, boot_application_processors, boot_processor_init, + get_processor_count, message_output_init, output_message_buf, output_message_byte, +}; +#[cfg(target_arch = "riscv64")] +pub use crate::arch::riscv::*; #[cfg(target_arch = "x86_64")] pub use crate::arch::x86_64::kernel::apic::{set_oneshot_timer, wakeup_core}; #[cfg(all(target_arch = "x86_64", target_os = "none", feature = "smp"))] diff --git a/src/arch/riscv/kernel/devicetree.rs b/src/arch/riscv/kernel/devicetree.rs new file mode 100644 index 0000000000..3b3caac93d --- /dev/null +++ b/src/arch/riscv/kernel/devicetree.rs @@ -0,0 +1,223 @@ +use alloc::vec::Vec; +use core::convert::TryFrom; + +use fdt::Fdt; +use hermit_sync::InterruptSpinMutex; + +use crate::arch::riscv::kernel::get_dtb_ptr; +use crate::arch::riscv::kernel::irq::init_plic; +use crate::arch::riscv::kernel::mmio::*; +use crate::arch::riscv::mm::{paging, PhysAddr, VirtAddr}; +use crate::drivers::net::gem; +use crate::drivers::virtio::transport::mmio as mmio_virtio; +use crate::drivers::virtio::transport::mmio::{DevId, MmioRegisterLayout, VirtioDriver}; + +pub const MMIO_MAGIC_VALUE: u32 = 0x74726976 as u32; +static mut PLATFORM_MODEL: Model = Model::UNKNOWN; + +enum Model { + FUX40, + VIRT, + UNKNOWN, +} + +/// Inits variables based on the device tree +/// This function should only be called once +pub fn init() { + debug!("Init devicetree"); + if !get_dtb_ptr().is_null() { + unsafe { + let dtb = Fdt::from_ptr(get_dtb_ptr()).expect("DTB is invalid"); + + let model = dtb + .find_node("/") + .unwrap() + .property("compatible") + .expect("compatible not found in DTB") + .as_str() + .unwrap(); + + if model.contains("riscv-virtio") { + PLATFORM_MODEL = Model::VIRT; + } else if model.contains("sifive,hifive-unmatched-a00") + || model.contains("sifive,hifive-unleashed-a00") + || model.contains("sifive,fu740") + || model.contains("sifive,fu540") + { + PLATFORM_MODEL = Model::FUX40; + } else { + warn!("Unknown platform, guessing PLIC context 1"); + PLATFORM_MODEL = Model::UNKNOWN; + } + info!("Model: {}", model); + } + } +} + +/// Inits drivers based on the device tree +/// This function should only be called once +pub fn init_drivers() { + // TODO: Implement devicetree correctly + if !get_dtb_ptr().is_null() { + unsafe { + debug!("Init drivers using devicetree"); + let dtb = Fdt::from_ptr(get_dtb_ptr()).expect("DTB is invalid"); + + // Init PLIC first + if let Some(plic_node) = dtb.find_compatible(&["sifive,plic-1.0.0"]) { + debug!("Found interrupt controller"); + let plic_region = plic_node + .reg() + .expect("Reg property for PLIC not found in DTB") + .next() + .unwrap(); + + debug!( + "Init PLIC at {:p}, size: {:x}", + plic_region.starting_address, + plic_region.size.unwrap() + ); + paging::identity_map::( + PhysAddr(plic_region.starting_address as u64), + PhysAddr( + (plic_region.starting_address as usize + plic_region.size.unwrap() - 1) + as u64, + ), + ); + + // TODO: Determine correct context via devicetree and allow more than one context + match PLATFORM_MODEL { + Model::VIRT => init_plic(plic_region.starting_address as usize, 1), + Model::UNKNOWN => init_plic(plic_region.starting_address as usize, 1), + Model::FUX40 => init_plic(plic_region.starting_address as usize, 2), + } + } + + // Init GEM + if let Some(gem_node) = dtb.find_compatible(&["sifive,fu540-c000-gem"]) { + debug!("Found Ethernet controller"); + + let gem_region = gem_node + .reg() + .expect("reg property for GEM not found in DTB") + .next() + .unwrap(); + let irq = gem_node + .interrupts() + .expect("interrupts property for GEM not found in DTB") + .next() + .unwrap(); + let mac = gem_node + .property("local-mac-address") + .expect("local-mac-address property for GEM not found in DTB") + .value; + debug!("Local MAC address: {:x?}", mac); + let mut phy_addr = u32::MAX; + + let phy_node = gem_node + .children() + .next() + .expect("GEM node has no child node (i. e. ethernet-phy)"); + if phy_node.name.contains("ethernet-phy") { + phy_addr = phy_node + .property("reg") + .expect("reg property for ethernet-phy not found in DTB") + .as_usize() + .unwrap() as u32; + } else { + warn!("Expected ethernet-phy node, found something else"); + } + + debug!( + "Init GEM at {:p}, irq: {}, phy_addr: {}", + gem_region.starting_address, irq, phy_addr + ); + paging::identity_map::( + PhysAddr(gem_region.starting_address as u64), + PhysAddr( + (gem_region.starting_address as usize + gem_region.size.unwrap() - 1) + as u64, + ), + ); + match gem::init_device( + VirtAddr(gem_region.starting_address as u64), + irq as u32, + phy_addr.into(), + <[u8; 6]>::try_from(mac).expect("MAC with invalid length"), + ) { + Ok(drv) => register_driver(MmioDriver::GEMNet(InterruptSpinMutex::new(drv))), + Err(_) => (), // could have information on error + } + } + + // Init virtio-mmio + if let Some(virtio_node) = dtb.find_compatible(&["virtio,mmio"]) { + debug!("Found virtio mmio device"); + let virtio_region = virtio_node + .reg() + .expect("reg property for virtio mmio not found in DTB") + .next() + .unwrap(); + let irq = virtio_node + .interrupts() + .expect("interrupts property for virtio mmio not found in DTB") + .next() + .unwrap(); + + debug!( + "Init virtio_mmio at {:p}, irq: {}", + virtio_region.starting_address, irq + ); + paging::identity_map::( + PhysAddr(virtio_region.starting_address as u64), + PhysAddr( + (virtio_region.starting_address as usize + virtio_region.size.unwrap() - 1) + as u64, + ), + ); + + // Verify the first register value to find out if this is really an MMIO magic-value. + let mmio = + unsafe { &mut *(virtio_region.starting_address as *mut MmioRegisterLayout) }; + + let magic = mmio.get_magic_value(); + let version = mmio.get_version(); + + if magic != MMIO_MAGIC_VALUE { + error!("It's not a MMIO-device at {:#X}", mmio as *const _ as usize); + } + + if version != 2 { + warn!("Found a leagacy device, which isn't supported"); + } else { + // We found a MMIO-device (whose 512-bit address in this structure). + trace!("Found a MMIO-device at {:#X}", mmio as *const _ as usize); + + // Verify the device-ID to find the network card + let id = mmio.get_device_id(); + + if id != DevId::VIRTIO_DEV_ID_NET { + debug!( + "It's not a network card at {:#X}", + mmio as *const _ as usize + ); + } else { + info!("Found network card at {:#X}", mmio as *const _ as usize); + + // crate::arch::mm::physicalmem::reserve( + // PhysAddr::from(align_down!(current_address, BasePageSize::SIZE as usize)), + // BasePageSize::SIZE as usize, + // ); + + match mmio_virtio::init_device(mmio, irq.try_into().unwrap()) { + Ok(VirtioDriver::Network(drv)) => { + register_driver(MmioDriver::VirtioNet(InterruptSpinMutex::new(drv))) + } + Err(_) => (), + } + } + } + } + } + } +} diff --git a/src/arch/riscv/kernel/externs_linux.rs b/src/arch/riscv/kernel/externs_linux.rs new file mode 100644 index 0000000000..02ee8f1ae4 --- /dev/null +++ b/src/arch/riscv/kernel/externs_linux.rs @@ -0,0 +1,257 @@ +// memcpy and memset derived from Linux kernel 5.15.5 /arch/riscv/lib +// memmove and memcmp from Redox (kernel/src/externs.rs) https://gitlab.redox-os.org/redox-os/kernel/-/blob/master/src/externs.rs + +use core::arch::asm; +use core::mem; + +const WORD_SIZE: usize = mem::size_of::(); + +/// Memcpy +/// +/// Copy N bytes of memory from one location to another. +#[no_mangle] +#[naked] +pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) { + asm!( + "move t6, a0", + "sltiu a3, a2, 128", + "bnez a3, 4f", + "andi a3, t6, 8-1", + "andi a4, a1, 8-1", + "bne a3, a4, 4f", + "beqz a3, 2f", + "andi a3, a1, ~(8-1)", + "addi a3, a3, 8", + "sub a4, a3, a1", + "1: lb a5, 0(a1)", + "addi a1, a1, 1", + "sb a5, 0(t6)", + "addi t6, t6, 1", + "bltu a1, a3, 1b", + "sub a2, a2, a4", + "2: andi a4, a2, ~((16*8)-1)", + "beqz a4, 4f", + "add a3, a1, a4", + "3: ld a4, 0(a1)", + "ld a5, 8(a1)", + "ld a6, 2*8(a1)", + "ld a7, 3*8(a1)", + "ld t0, 4*8(a1)", + "ld t1, 5*8(a1)", + "ld t2, 6*8(a1)", + "ld t3, 7*8(a1)", + "ld t4, 8*8(a1)", + "ld t5, 9*8(a1)", + "sd a4, 0(t6)", + "sd a5, 8(t6)", + "sd a6, 2*8(t6)", + "sd a7, 3*8(t6)", + "sd t0, 4*8(t6)", + "sd t1, 5*8(t6)", + "sd t2, 6*8(t6)", + "sd t3, 7*8(t6)", + "sd t4, 8*8(t6)", + "sd t5, 9*8(t6)", + "ld a4, 10*8(a1)", + "ld a5, 11*8(a1)", + "ld a6, 12*8(a1)", + "ld a7, 13*8(a1)", + "ld t0, 14*8(a1)", + "ld t1, 15*8(a1)", + "addi a1, a1, 16*8", + "sd a4, 10*8(t6)", + "sd a5, 11*8(t6)", + "sd a6, 12*8(t6)", + "sd a7, 13*8(t6)", + "sd t0, 14*8(t6)", + "sd t1, 15*8(t6)", + "addi t6, t6, 16*8", + "bltu a1, a3, 3b", + "andi a2, a2, (16*8)-1", + "4: beqz a2, 6f", + "add a3, a1, a2", + "or a5, a1, t6", + "or a5, a5, a3", + "andi a5, a5, 3", + "bnez a5, 5f", + "7: lw a4, 0(a1)", + "addi a1, a1, 4", + "sw a4, 0(t6)", + "addi t6, t6, 4", + "bltu a1, a3, 7b", + "ret", + "5: lb a4, 0(a1)", + "addi a1, a1, 1", + "sb a4, 0(t6)", + "addi t6, t6, 1", + "bltu a1, a3, 5b", + "6: ret", + options(noreturn), + ); +} + +/// Memmove +/// +/// Copy N bytes of memory from src to dest. The memory areas may overlap. +#[no_mangle] +pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + if src < dest as *const u8 { + let n_usize: usize = n / WORD_SIZE; // Number of word sized groups + let mut i: usize = n_usize * WORD_SIZE; + + // Copy `WORD_SIZE` bytes at a time + while i != 0 { + i -= WORD_SIZE; + *((dest as usize + i) as *mut usize) = *((src as usize + i) as *const usize); + } + + let mut i: usize = n; + + // Copy 1 byte at a time + while i != n_usize * WORD_SIZE { + i -= 1; + *((dest as usize + i) as *mut u8) = *((src as usize + i) as *const u8); + } + } else { + let n_usize: usize = n / WORD_SIZE; // Number of word sized groups + let mut i: usize = 0; + + // Copy `WORD_SIZE` bytes at a time + let n_fast = n_usize * WORD_SIZE; + while i < n_fast { + *((dest as usize + i) as *mut usize) = *((src as usize + i) as *const usize); + i += WORD_SIZE; + } + + // Copy 1 byte at a time + while i < n { + *((dest as usize + i) as *mut u8) = *((src as usize + i) as *const u8); + i += 1; + } + } + + dest +} + +/// Memset +/// +/// Fill a block of memory with a specified value. +#[no_mangle] +#[naked] +pub unsafe extern "C" fn memset(dest: *mut u8, c: i32, n: usize) -> *mut u8 { + asm!( + "move t0, a0", + "sltiu a3, a2, 16", + "bnez a3, 4f", + "addi a3, t0, 8-1", + "andi a3, a3, ~(8-1)", + "beq a3, t0, 2f", + "sub a4, a3, t0", + "1: sb a1, 0(t0)", + "addi t0, t0, 1", + "bltu t0, a3, 1b", + "sub a2, a2, a4", + "2: andi a1, a1, 0xff", + "slli a3, a1, 8", + "or a1, a3, a1", + "slli a3, a1, 16", + "or a1, a3, a1", + "slli a3, a1, 32", + "or a1, a3, a1", + "andi a4, a2, ~(8-1)", + "add a3, t0, a4", + "andi a4, a4, 31*8", + "beqz a4, 3f", + "neg a4, a4", + "addi a4, a4, 32*8", + "sub t0, t0, a4", + "la a5, 3f", + "srli a4, a4, 1", + "add a5, a5, a4", + "jr a5", + "3: sd a1, 0(t0)", + "sd a1, 8(t0)", + "sd a1, 2*8(t0)", + "sd a1, 3*8(t0)", + "sd a1, 4*8(t0)", + "sd a1, 5*8(t0)", + "sd a1, 6*8(t0)", + "sd a1, 7*8(t0)", + "sd a1, 8*8(t0)", + "sd a1, 9*8(t0)", + "sd a1, 10*8(t0)", + "sd a1, 11*8(t0)", + "sd a1, 12*8(t0)", + "sd a1, 13*8(t0)", + "sd a1, 14*8(t0)", + "sd a1, 15*8(t0)", + "sd a1, 16*8(t0)", + "sd a1, 17*8(t0)", + "sd a1, 18*8(t0)", + "sd a1, 19*8(t0)", + "sd a1, 20*8(t0)", + "sd a1, 21*8(t0)", + "sd a1, 22*8(t0)", + "sd a1, 23*8(t0)", + "sd a1, 24*8(t0)", + "sd a1, 25*8(t0)", + "sd a1, 26*8(t0)", + "sd a1, 27*8(t0)", + "sd a1, 28*8(t0)", + "sd a1, 29*8(t0)", + "sd a1, 30*8(t0)", + "sd a1, 31*8(t0)", + "addi t0, t0, 32*8", + "bltu t0, a3, 3b", + "andi a2, a2, 8-1", + "4: beqz a2, 6f", + "add a3, t0, a2", + "5: sb a1, 0(t0)", + "addi t0, t0, 1", + "bltu t0, a3, 5b", + "6: ret", + options(noreturn), + ); +} + +/// Memcmp +/// +/// Compare two blocks of memory. +/// +/// This faster implementation works by comparing bytes not one-by-one, but in +/// groups of 8 bytes (or 4 bytes in the case of 32-bit architectures). +#[no_mangle] +pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + let n_usize: usize = n / WORD_SIZE; + let mut i: usize = 0; + + let n_fast = n_usize * WORD_SIZE; + while i < n_fast { + let a = *((s1 as usize + i) as *const usize); + let b = *((s2 as usize + i) as *const usize); + if a != b { + let n: usize = i + WORD_SIZE; + // Find the one byte that is not equal + while i < n { + let a = *((s1 as usize + i) as *const u8); + let b = *((s2 as usize + i) as *const u8); + if a != b { + return a as i32 - b as i32; + } + i += 1; + } + } + i += WORD_SIZE; + } + + while i < n { + let a = *((s1 as usize + i) as *const u8); + let b = *((s2 as usize + i) as *const u8); + if a != b { + return a as i32 - b as i32; + } + i += 1; + } + + 0 +} diff --git a/src/arch/riscv/kernel/irq.rs b/src/arch/riscv/kernel/irq.rs new file mode 100644 index 0000000000..a256a8b327 --- /dev/null +++ b/src/arch/riscv/kernel/irq.rs @@ -0,0 +1,253 @@ +use alloc::vec::Vec; +use core::arch::asm; + +use hermit_sync::SpinMutex; +use riscv::asm::wfi; +use riscv::register::*; +use trapframe::TrapFrame; + +/// base address of the PLIC, only one access at the same time is allowed +static PLIC_BASE: SpinMutex = SpinMutex::new(0x0); + +/// PLIC context for new interrupt handlers +static PLIC_CONTEXT: SpinMutex = SpinMutex::new(0x0); + +/// PLIC context for new interrupt handlers +static CURRENT_INTERRUPTS: SpinMutex> = SpinMutex::new(Vec::new()); + +const PLIC_PENDING_OFFSET: usize = 0x001000; +const PLIC_ENABLE_OFFSET: usize = 0x002000; + +const MAX_IRQ: usize = 69; + +static mut IRQ_HANDLERS: [usize; MAX_IRQ] = [0; MAX_IRQ]; + +/// Init Interrupts +pub fn install() { + unsafe { + // Intstall trap handler + trapframe::init(); + // Enable external interrupts + sie::set_sext(); + } +} + +/// Init PLIC +pub fn init_plic(base: usize, context: u16) { + *PLIC_BASE.lock() = base; + *PLIC_CONTEXT.lock() = context; +} + +/// Enable Interrupts +#[inline] +pub fn enable() { + unsafe { + sstatus::set_sie(); + } +} + +/// Waits for the next interrupt (Only Supervisor-level software/timer interrupt for now) +/// and calls the specific handler +#[inline] +pub fn enable_and_wait() { + unsafe { + //Enable Supervisor-level software interrupts + sie::set_ssoft(); + //sie::set_sext(); + debug!("Wait {:x?}", sie::read()); + loop { + wfi(); + // Interrupts are disabled at this point, so a pending interrupt will + // resume the execution. We still have to check if a interrupt is pending + // because the WFI instruction could be implemented as NOP (The RISC-V Instruction Set ManualVolume II: Privileged Architecture) + + let pending_interrupts = sip::read(); + + // trace!("sip: {:x?}", pending_interrupts); + #[cfg(feature = "smp")] + if pending_interrupts.ssoft() { + //Clear Supervisor-level software interrupt + asm!( + "csrc sip, {ssoft_mask}", + ssoft_mask = in(reg) 0x2, + ); + trace!("SOFT"); + //Disable Supervisor-level software interrupt + sie::clear_ssoft(); + crate::arch::riscv::kernel::scheduler::wakeup_handler(); + break; + } + + if pending_interrupts.sext() { + trace!("EXT"); + external_handler(); + break; + } + + if pending_interrupts.stimer() { + // // Disable Supervisor-level software interrupt, wakeup not needed + // sie::clear_ssoft(); + + debug!("sip: {:x?}", pending_interrupts); + trace!("TIMER"); + crate::arch::riscv::kernel::scheduler::timer_handler(); + break; + } + } + } +} + +/// Disable Interrupts +#[inline] +pub fn disable() { + unsafe { sstatus::clear_sie() }; +} + +/// Disable IRQs (nested) +/// +/// Disable IRQs when unsure if IRQs were enabled at all. +/// This function together with nested_enable can be used +/// in situations when interrupts shouldn't be activated if they +/// were not activated before calling this function. +#[inline] +pub fn nested_disable() -> bool { + let was_enabled = sstatus::read().sie(); + + disable(); + was_enabled +} + +/// Enable IRQs (nested) +/// +/// Can be used in conjunction with nested_disable() to only enable +/// interrupts again if they were enabled before. +#[inline] +pub fn nested_enable(was_enabled: bool) { + if was_enabled { + enable(); + } +} + +/// Currently not needed because we use the trapframe crate +#[no_mangle] +pub extern "C" fn irq_install_handler(irq_number: u32, handler: usize) { + unsafe { + let base_ptr = PLIC_BASE.lock(); + let context = PLIC_CONTEXT.lock(); + debug!( + "Install handler for interrupt {}, context {}", + irq_number, *context + ); + IRQ_HANDLERS[irq_number as usize - 1] = handler; + // Set priority to 7 (highest on FU740) + let prio_address = *base_ptr + irq_number as usize * 4; + core::ptr::write_volatile(prio_address as *mut u32, 1); + // Set Threshold to 0 (lowest) + let thresh_address = *base_ptr + 0x20_0000 + 0x1000 * (*context as usize); + core::ptr::write_volatile(thresh_address as *mut u32, 0); + // Enable irq for context + let enable_address = *base_ptr + + PLIC_ENABLE_OFFSET + + 0x80 * (*context as usize) + + ((irq_number / 32) * 4) as usize; + debug!("enable_address {:x}", enable_address); + core::ptr::write_volatile(enable_address as *mut u32, 1 << (irq_number % 32)); + } +} + +// Derived from rCore: https://github.com/rcore-os/rCore +/// Dispatch and handle interrupt. +/// +/// This function is called from `trap.S` which is in the trapframe crate. +#[no_mangle] +pub extern "C" fn trap_handler(tf: &mut TrapFrame) { + use self::scause::{Exception as E, Interrupt as I, Trap}; + let scause = scause::read(); + let stval = stval::read(); + let sepc = tf.sepc; + trace!("Interrupt: {:?} ", scause.cause()); + trace!("tf: {:x?} ", tf); + trace!("stvall: {:x}", stval); + trace!("sepc: {:x}", sepc); + trace!("SSTATUS FS: {:?}", sstatus::read().fs()); + trace!("FCSR: {:x?}", fcsr::read()); + + match scause.cause() { + Trap::Interrupt(I::SupervisorExternal) => external_handler(), + #[cfg(feature = "smp")] + Trap::Interrupt(I::SupervisorSoft) => crate::arch::riscv::kernel::scheduler::wakeup_handler(), + Trap::Interrupt(I::SupervisorTimer) => { + crate::arch::riscv::kernel::scheduler::timer_handler() + } + //Trap::Exception(E::LoadPageFault) => page_fault(stval, tf), + //Trap::Exception(E::StorePageFault) => page_fault(stval, tf), + //Trap::Exception(E::InstructionPageFault) => page_fault(stval, tf), + _ => { + error!("Interrupt: {:?} ", scause.cause()); + error!("tf: {:x?} ", tf); + error!("stvall: {:x}", stval); + error!("sepc: {:x}", sepc); + error!("SSTATUS FS: {:?}", sstatus::read().fs()); + error!("FCSR: {:x?}", fcsr::read()); + panic!("unhandled trap {:?}", scause.cause()) + } + } + trace!("Interrupt end"); +} + +/// Handles external interrupts +fn external_handler() { + unsafe { + let handler: Option = { + // Claim interrupt + let base_ptr = PLIC_BASE.lock(); + let context = PLIC_CONTEXT.lock(); + //let claim_address = *base_ptr + 0x20_2004; + let claim_address = *base_ptr + 0x20_0004 + 0x1000 * (*context as usize); + let irq = core::ptr::read_volatile(claim_address as *mut u32); + if irq != 0 { + debug!("External INT: {}", irq); + let mut cur_int = CURRENT_INTERRUPTS.lock(); + cur_int.push(irq); + if cur_int.len() > 1 { + warn!("More than one external interrupt is pending!"); + } + + // Call handler + if IRQ_HANDLERS[irq as usize - 1] != 0 { + let ptr = IRQ_HANDLERS[irq as usize - 1] as *const (); + let handler: fn() = core::mem::transmute(ptr); + Some(handler) + } else { + error!("Interrupt handler not installed"); + None + } + } else { + None + } + }; + + if let Some(handler) = handler { + handler(); + } + } +} + +/// End of external interrupt +pub fn external_eoi() { + unsafe { + let base_ptr = PLIC_BASE.lock(); + let context = PLIC_CONTEXT.lock(); + let claim_address = *base_ptr + 0x20_0004 + 0x1000 * (*context as usize); + + let mut cur_int = CURRENT_INTERRUPTS.lock(); + let irq = cur_int.pop().unwrap_or(0); + if irq != 0 { + debug!("EOI INT: {}", irq); + // Complete interrupt + core::ptr::write_volatile(claim_address as *mut u32, irq); + } else { + warn!("Called EOI without active interrupt"); + } + } +} diff --git a/src/arch/riscv/kernel/mmio.rs b/src/arch/riscv/kernel/mmio.rs new file mode 100644 index 0000000000..b4dff97dac --- /dev/null +++ b/src/arch/riscv/kernel/mmio.rs @@ -0,0 +1,33 @@ +use alloc::vec::Vec; + +use hermit_sync::InterruptSpinMutex; + +use crate::drivers::net::gem::GEMDriver; +use crate::drivers::net::virtio_net::VirtioNetDriver; +use crate::drivers::net::NetworkInterface; + +static mut MMIO_DRIVERS: Vec = Vec::new(); + +pub enum MmioDriver { + GEMNet(InterruptSpinMutex), + VirtioNet(InterruptSpinMutex), +} + +impl<'a> MmioDriver { + fn get_network_driver(&self) -> Option<&InterruptSpinMutex> { + match self { + Self::VirtioNet(drv) => Some(drv), + Self::GEMNet(drv) => Some(drv), + _ => None, + } + } +} +pub fn register_driver(drv: MmioDriver) { + unsafe { + MMIO_DRIVERS.push(drv); + } +} + +pub fn get_network_driver() -> Option<&'static InterruptSpinMutex> { + unsafe { MMIO_DRIVERS.iter().find_map(|drv| drv.get_network_driver()) } +} diff --git a/src/arch/riscv/kernel/mod.rs b/src/arch/riscv/kernel/mod.rs new file mode 100644 index 0000000000..e98038a90e --- /dev/null +++ b/src/arch/riscv/kernel/mod.rs @@ -0,0 +1,334 @@ +mod devicetree; +pub mod externs_linux; +pub mod irq; +pub mod mmio; +pub mod pci; +pub mod percore; +pub mod processor; +mod sbi; +pub mod scheduler; +pub mod serial; +mod start; +pub mod switch; +pub mod systemtime; + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::{fmt, intrinsics, mem, ptr}; + +use riscv::register::{fcsr, sstatus}; + +pub use crate::arch::riscv::kernel::devicetree::init_drivers; +use crate::arch::riscv::kernel::percore::*; +use crate::arch::riscv::kernel::processor::lsb; +use crate::arch::riscv::kernel::serial::SerialPort; +pub use crate::arch::riscv::kernel::systemtime::get_boot_time; +use crate::arch::riscv::mm::{paging, physicalmem, PhysAddr, VirtAddr}; +use crate::config::*; +use crate::env; +use crate::scheduler::CoreId; + +const SERIAL_PORT_BAUDRATE: u32 = 115200; +const BOOTINFO_MAGIC_NUMBER: u32 = 0xC0DE_CAFEu32; + +static mut COM1: SerialPort = SerialPort::new(0x9000000); + +// Used to store information about available harts. The index of the hart in the vector +// represents its CpuId and does not need to match its hart_id +pub static mut HARTS_AVAILABLE: Vec = Vec::new(); + +#[repr(C)] +struct BootInfo { + pub magic_number: u32, + pub version: u32, + pub base: u64, + pub ram_start: u64, + pub limit: u64, + pub image_size: u64, + pub tls_start: u64, + pub tls_filesz: u64, + pub tls_memsz: u64, + pub tls_align: u64, + pub current_stack_address: u64, + pub current_percore_address: u64, + pub host_logical_addr: u64, + pub boot_gtod: u64, + pub cmdline: u64, + pub cmdsize: u64, + pub cpu_freq: u32, + pub boot_processor: u32, + pub cpu_online: u32, + pub possible_cpus: u32, + pub current_boot_id: u32, + pub uartport: u32, + pub single_kernel: u8, + pub uhyve: u8, + pub hcip: [u8; 4], + pub hcgateway: [u8; 4], + pub hcmask: [u8; 4], + pub dtb_ptr: u64, + pub hart_mask: u64, + pub timebase_freq: u64, +} + +impl fmt::Debug for BootInfo { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "magic_number {:#x}", self.magic_number)?; + writeln!(f, "version {:#x}", self.version)?; + writeln!(f, "base {:#x}", self.base)?; + writeln!(f, "ram address {:#x}", self.ram_start)?; + writeln!(f, "limit {:#x}", self.limit)?; + writeln!(f, "tls_start {:#x}", self.tls_start)?; + writeln!(f, "tls_filesz {:#x}", self.tls_filesz)?; + writeln!(f, "tls_memsz {:#x}", self.tls_memsz)?; + writeln!(f, "tls_align {:#x}", self.tls_align)?; + writeln!(f, "image_size {:#x}", self.image_size)?; + writeln!(f, "current_stack_address {:#x}", self.current_stack_address)?; + writeln!( + f, + "current_percore_address {:#x}", + self.current_percore_address + )?; + writeln!(f, "host_logical_addr {:#x}", self.host_logical_addr)?; + writeln!(f, "boot_gtod {:#x}", self.boot_gtod)?; + writeln!(f, "cmdline {:#x}", self.cmdline)?; + writeln!(f, "cmdsize {:#x}", self.cmdsize)?; + writeln!(f, "cpu_freq {}", self.cpu_freq)?; + writeln!(f, "boot_processor {}", self.boot_processor)?; + writeln!(f, "cpu_online {}", self.cpu_online)?; + writeln!(f, "possible_cpus {}", self.possible_cpus)?; + writeln!(f, "current_boot_id {}", self.current_boot_id)?; + writeln!(f, "uartport {:#x}", self.uartport)?; + writeln!(f, "single_kernel {}", self.single_kernel)?; + writeln!(f, "uhyve {}", self.uhyve)?; + writeln!(f, "dtb_ptr {:x}", self.dtb_ptr)?; + writeln!(f, "hart_mask {:x}", self.hart_mask)?; + writeln!(f, "timebase_freq {}", self.timebase_freq) + } +} + +/// Kernel header to announce machine features +static mut BOOT_INFO: *mut BootInfo = ptr::null_mut(); + +// FUNCTIONS + +pub fn get_ram_address() -> PhysAddr { + unsafe { PhysAddr(core::ptr::read_volatile(&(*BOOT_INFO).ram_start)) } +} + +pub fn get_image_size() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).image_size) as usize } +} + +pub fn get_limit() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).limit) as usize } +} + +#[cfg(feature = "smp")] +pub fn get_processor_count() -> u32 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).cpu_online) as u32 } +} + +#[cfg(not(feature = "smp"))] +pub fn get_processor_count() -> u32 { + 1 +} + +pub fn get_base_address() -> VirtAddr { + unsafe { VirtAddr(core::ptr::read_volatile(&(*BOOT_INFO).base)) } +} + +pub fn get_tls_start() -> VirtAddr { + unsafe { VirtAddr(core::ptr::read_volatile(&(*BOOT_INFO).tls_start)) } +} + +pub fn get_tls_filesz() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).tls_filesz) as usize } +} + +pub fn get_tls_memsz() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).tls_memsz) as usize } +} + +pub fn get_tls_align() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).tls_align) as usize } +} + +/// Whether HermitCore is running under the "uhyve" hypervisor. +pub fn is_uhyve() -> bool { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).uhyve) != 0 } +} + +/// Whether HermitCore is running alone (true) or side-by-side to Linux in Multi-Kernel mode (false). +pub fn is_single_kernel() -> bool { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).single_kernel) != 0 } +} + +pub fn get_cmdsize() -> usize { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).cmdsize) as usize } +} + +pub fn get_cmdline() -> VirtAddr { + VirtAddr(unsafe { core::ptr::read_volatile(&(*BOOT_INFO).cmdline) }) +} + +pub fn get_dtb_ptr() -> *const u8 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).dtb_ptr) as *const u8 } +} + +pub fn get_hart_mask() -> u64 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).hart_mask) } +} + +pub fn get_timebase_freq() -> u64 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).timebase_freq) as u64 } +} + +pub fn get_current_boot_id() -> u32 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).current_boot_id) } +} + +/// Earliest initialization function called by the Boot Processor. +pub fn message_output_init() { + percore::init(); + + // We can only initialize the serial port here, because VGA requires processor + // configuration first. + unsafe { + COM1.init(SERIAL_PORT_BAUDRATE); + } +} + +pub fn output_message_byte(byte: u8) { + // Output messages to the serial port and VGA screen in unikernel mode. + unsafe { + COM1.write_byte(byte); + } +} + +pub fn output_message_buf(buf: &[u8]) { + for byte in buf { + output_message_byte(*byte); + } +} + +/// Real Boot Processor initialization as soon as we have put the first Welcome message on the screen. +pub fn boot_processor_init() { + devicetree::init(); + crate::mm::init(); + crate::mm::print_information(); + env::init(); + irq::install(); + + finish_processor_init(); + irq::enable(); +} + +/// Boots all available Application Processors on bare-metal or QEMU. +/// Called after the Boot Processor has been fully initialized along with its scheduler. +pub fn boot_application_processors() { + // Nothing to do here yet. +} + +extern "C" { + fn _start(hart_id: usize, boot_info: &'static mut BootInfo) -> !; +} + +/// Application Processor initialization +pub fn application_processor_init() { + percore::init(); + paging::init_application_processor(); + irq::install(); + finish_processor_init(); + irq::enable(); +} + +fn finish_processor_init() { + unsafe { + sstatus::set_fs(sstatus::FS::Initial); + } + trace!("SSTATUS FS: {:?}", sstatus::read().fs()); + trace!("FCSR: {:x?}", fcsr::read()); + + let current_hart_id = get_current_boot_id() as usize; + + unsafe { + // Add hart to HARTS_AVAILABLE, the hart id is stored in current_boot_id + HARTS_AVAILABLE.push(current_hart_id); + info!( + "Initialized CPU with hart_id {}", + HARTS_AVAILABLE[percore::core_id() as usize] + ); + } + + crate::scheduler::add_current_core(); + + // Remove current hart from the hart_mask + let new_hart_mask = get_hart_mask() & (u64::MAX - (1 << current_hart_id)); + unsafe { + core::ptr::write_volatile(&mut (*BOOT_INFO).hart_mask, new_hart_mask); + } + + let next_hart_index = lsb(new_hart_mask); + + if let Some(next_hart_id) = next_hart_index { + // The current processor already needs to prepare the processor variables for a possible next processor. + init_next_processor_variables(core_id() + 1); + + info!( + "Starting CPU {} with hart_id {}", + core_id() + 1, + next_hart_id + ); + + // Changing cpu_online will cause uhyve to start the next processor + unsafe { + let _ = intrinsics::atomic_xadd_seqcst(&mut (*BOOT_INFO).cpu_online as *mut u32, 1); + + //When running bare-metal/QEMU we use the firmware to start the next hart + if !is_uhyve() { + let ret = sbi::sbi_hart_start( + next_hart_id as usize, + _start as *const () as usize, + BOOT_INFO as usize, + ); + debug!("sbi_hart_start: {:?}", ret); + } + } + } else { + info!("All processors are initialized"); + unsafe { + let _ = intrinsics::atomic_xadd_seqcst(&mut (*BOOT_INFO).cpu_online as *mut u32, 1); + } + } +} + +pub fn print_statistics() {} + +/// Initialize the required start.rs variables for the next CPU to be booted. +pub fn init_next_processor_variables(core_id: CoreId) { + // Allocate stack and PerCoreVariables structure for the CPU and pass the addresses. + // Keep the stack executable to possibly support dynamically generated code on the stack (see https://security.stackexchange.com/a/47825). + let stack = physicalmem::allocate(KERNEL_STACK_SIZE) + .expect("Failed to allocate boot stack for new core"); + let mut boxed_percore = Box::new(PerCoreVariables::new(core_id)); + //let boxed_irq = Box::new(IrqStatistics::new()); + //let boxed_irq_raw = Box::into_raw(boxed_irq); + + unsafe { + //IRQ_COUNTERS.insert(core_id, &(*boxed_irq_raw)); + //boxed_percore.irq_statistics = PerCoreVariable::new(boxed_irq_raw); + + core::ptr::write_volatile(&mut (*BOOT_INFO).current_stack_address, stack.as_u64()); + core::ptr::write_volatile( + &mut (*BOOT_INFO).current_percore_address, + Box::into_raw(boxed_percore) as u64, + ); + + info!( + "Initialize per core data at 0x{:x} (size {} bytes)", + core::ptr::read_volatile(&(*BOOT_INFO).current_percore_address), + mem::size_of::() + ); + } +} diff --git a/src/arch/riscv/kernel/pci.rs b/src/arch/riscv/kernel/pci.rs new file mode 100644 index 0000000000..b9dddbab92 --- /dev/null +++ b/src/arch/riscv/kernel/pci.rs @@ -0,0 +1,42 @@ +use hermit_sync::InterruptSpinMutex; + +// Currently, onbly a dummy implementation +pub struct VirtioNetDriver; + +impl VirtioNetDriver { + pub fn init_vqs(&mut self) {} + + pub fn set_polling_mode(&mut self, value: bool) { + //(self.vqueues.as_deref_mut().unwrap())[VIRTIO_NET_RX_QUEUE].set_polling_mode(value); + } + + pub fn get_mac_address(&self) -> [u8; 6] { + [0; 6] + } + + pub fn get_mtu(&self) -> u16 { + 1500 //self.device_cfg.mtu + } + + pub fn get_tx_buffer(&mut self, len: usize) -> Result<(*mut u8, usize), ()> { + Err(()) + } + + pub fn send_tx_buffer(&mut self, index: usize, len: usize) -> Result<(), ()> { + Err(()) + } + + pub fn has_packet(&self) -> bool { + false + } + + pub fn receive_rx_buffer(&self) -> Result<&'static [u8], ()> { + Err(()) + } + + pub fn rx_buffer_consumed(&mut self) {} +} + +pub fn get_network_driver() -> Option<&'static InterruptSpinMutex> { + None +} diff --git a/src/arch/riscv/kernel/percore.rs b/src/arch/riscv/kernel/percore.rs new file mode 100644 index 0000000000..274855d88a --- /dev/null +++ b/src/arch/riscv/kernel/percore.rs @@ -0,0 +1,138 @@ +use core::arch::asm; + +use crate::arch::riscv::kernel::BOOT_INFO; +use crate::scheduler::{CoreId, PerCoreScheduler}; + +#[no_mangle] +pub static mut PERCORE: PerCoreVariables = PerCoreVariables::new(0); + +#[derive(Debug)] +pub struct PerCoreVariables { + /// ID of the current Core. + core_id: PerCoreVariable, + /// Scheduler of the current Core. + scheduler: PerCoreVariable, + /// start address of the kernel stack + pub kernel_stack: PerCoreVariable, +} + +impl PerCoreVariables { + pub const fn new(core_id: CoreId) -> Self { + Self { + core_id: PerCoreVariable::new(core_id as usize), + scheduler: PerCoreVariable::new(0), + kernel_stack: PerCoreVariable::new(0), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct PerCoreVariable { + data: usize, +} + +pub trait PerCoreVariableMethods { + unsafe fn get(&self) -> usize; + unsafe fn set(&mut self, value: usize); +} + +impl PerCoreVariable { + const fn new(value: usize) -> Self { + Self { data: value } + } + + #[inline] + unsafe fn offset(&self) -> usize { + let base = &PERCORE as *const _ as usize; + let field = self as *const _ as usize; + field - base + } +} + +// Treat all per-core variables as 64-bit variables by default. This is true for u64, usize, pointers. +// Implement the PerCoreVariableMethods trait functions using 64-bit memory moves. +// The functions are implemented as default functions, which can be overriden in specialized implementations of the trait. +impl PerCoreVariableMethods for PerCoreVariable { + #[inline] + #[cfg(feature = "smp")] + default unsafe fn get(&self) -> usize { + let mut value: usize; + let mut offset = self.offset(); + asm!( + "add {offset}, {offset}, gp", + "ld {value}, 0({offset})", + value = out(reg) value, + offset = inout(reg) offset, // This has to be "inout" to work with the "release" profile? + ); + value + } + + #[inline] + #[cfg(not(feature = "smp"))] + default unsafe fn get(&self) -> usize { + self.data + } + + #[inline] + #[cfg(feature = "smp")] + default unsafe fn set(&mut self, value: usize) { + let mut offset = self.offset(); + asm!( + "add {offset}, {offset}, gp", + "sd {value}, 0({offset})", + value = in(reg) value, + offset = inout(reg) offset, // This has to be "inout" to work with the "release" profile? + ); + } + + #[inline] + #[cfg(not(feature = "smp"))] + default unsafe fn set(&mut self, value: usize) { + self.data = value; + } +} + +#[inline] +pub fn core_id() -> CoreId { + unsafe { PERCORE.core_id.get() as u32 } +} + +#[inline] +pub fn core_scheduler() -> &'static mut PerCoreScheduler { + unsafe { &mut *(PERCORE.scheduler.get() as *mut PerCoreScheduler) } +} + +#[inline] +pub fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { + unsafe { + PERCORE.scheduler.set(scheduler as usize); + } +} + +#[inline(always)] +pub fn get_kernel_stack() -> u64 { + unsafe { PERCORE.kernel_stack.get() as u64 } +} + +#[inline] +pub fn set_kernel_stack(addr: u64) { + unsafe { PERCORE.kernel_stack.set(addr as usize) } +} + +pub fn init() { + unsafe { + // Store the address to the PerCoreVariables structure allocated for this core in gp. + let mut address = core::ptr::read_volatile(&(*BOOT_INFO).current_percore_address); + if address == 0 { + address = &PERCORE as *const _ as u64; + } + + asm!( + "mv gp, {address}", + address = in(reg) address, + ); + + //println!("percore address: {:x}, {:x?}", address, *(address as *const PerCoreVariables)); + } +} diff --git a/src/arch/riscv/kernel/processor.rs b/src/arch/riscv/kernel/processor.rs new file mode 100644 index 0000000000..1e2fe4086a --- /dev/null +++ b/src/arch/riscv/kernel/processor.rs @@ -0,0 +1,313 @@ +use core::arch::asm; +use core::convert::TryInto; +use core::hint::spin_loop; + +use riscv::asm::wfi; +use riscv::register::{sie, sstatus, time}; + +use crate::arch::riscv::kernel::{get_timebase_freq, sbi, HARTS_AVAILABLE}; +use crate::scheduler::CoreId; + +/// Current FPU state. Saved at context switch when changed +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct FPUState { + /// f0 register + f0: u64, + /// f1 register + f1: u64, + /// f2 register + f2: u64, + /// f3 register + f3: u64, + /// f4 register + f4: u64, + /// f5 register + f5: u64, + /// f6 register + f6: u64, + /// f7 register + f7: u64, + /// f8 register + f8: u64, + /// f9 register + f9: u64, + /// f10 register + f10: u64, + /// f11 register + f11: u64, + /// f12 register + f12: u64, + /// f13 register + f13: u64, + /// f14 register + f14: u64, + /// f15 register + f15: u64, + /// f16 register + f16: u64, + /// f17 register + f17: u64, + /// f18 register + f18: u64, + /// f19 register + f19: u64, + /// f20 register + f20: u64, + /// f21 register + f21: u64, + /// f22 register + f22: u64, + /// f23 register + f23: u64, + /// f24 register + f24: u64, + /// f25 register + f25: u64, + /// f26 register + f26: u64, + /// f27 register + f27: u64, + /// f28 register + f28: u64, + /// f29 register + f29: u64, + /// f30 register + f30: u64, + /// f31 register + f31: u64, + /// fcsr register + fcsr: usize, +} + +impl FPUState { + pub fn new() -> Self { + Self { + f0: 0, + f1: 0, + f2: 0, + f3: 0, + f4: 0, + f5: 0, + f6: 0, + f7: 0, + f8: 0, + f9: 0, + f10: 0, + f11: 0, + f12: 0, + f13: 0, + f14: 0, + f15: 0, + f16: 0, + f17: 0, + f18: 0, + f19: 0, + f20: 0, + f21: 0, + f22: 0, + f23: 0, + f24: 0, + f25: 0, + f26: 0, + f27: 0, + f28: 0, + f29: 0, + f30: 0, + f31: 0, + fcsr: 0, + } + } + + pub fn restore(&self) { + unsafe { + trace!("Restore FPUState at {:p} with {:x?}", self, self); + asm! { + "fld f0, (8*0)({fpu_state})", + "fld f1, (8*1)({fpu_state})", + "fld f2, (8*2)({fpu_state})", + "fld f3, (8*3)({fpu_state})", + "fld f4, (8*4)({fpu_state})", + "fld f5, (8*5)({fpu_state})", + "fld f6, (8*6)({fpu_state})", + "fld f7, (8*7)({fpu_state})", + "fld f8, (8*8)({fpu_state})", + "fld f9, (8*9)({fpu_state})", + "fld f10, (8*10)({fpu_state})", + "fld f11, (8*11)({fpu_state})", + "fld f12, (8*12)({fpu_state})", + "fld f13, (8*13)({fpu_state})", + "fld f14, (8*14)({fpu_state})", + "fld f15, (8*15)({fpu_state})", + "fld f16, (8*16)({fpu_state})", + "fld f17, (8*17)({fpu_state})", + "fld f18, (8*18)({fpu_state})", + "fld f19, (8*19)({fpu_state})", + "fld f20, (8*20)({fpu_state})", + "fld f21, (8*21)({fpu_state})", + "fld f22, (8*22)({fpu_state})", + "fld f23, (8*23)({fpu_state})", + "fld f24, (8*24)({fpu_state})", + "fld f25, (8*25)({fpu_state})", + "fld f26, (8*26)({fpu_state})", + "fld f27, (8*27)({fpu_state})", + "fld f28, (8*28)({fpu_state})", + "fld f29, (8*29)({fpu_state})", + "fld f30, (8*30)({fpu_state})", + "fld f31, (8*31)({fpu_state})", + "ld t0, (8*32)({fpu_state})", + "fscsr t0", + fpu_state = in(reg) self as *const _, + out("t0") _, + } + + sstatus::set_fs(sstatus::FS::Clean); + } + } + + pub fn save(&mut self) { + unsafe { + trace!("Save FPUState at {:p}", self); + asm! { + "fsd f0, (8*0)({fpu_state})", + "fsd f1, (8*1)({fpu_state})", + "fsd f2, (8*2)({fpu_state})", + "fsd f3, (8*3)({fpu_state})", + "fsd f4, (8*4)({fpu_state})", + "fsd f5, (8*5)({fpu_state})", + "fsd f6, (8*6)({fpu_state})", + "fsd f7, (8*7)({fpu_state})", + "fsd f8, (8*8)({fpu_state})", + "fsd f9, (8*9)({fpu_state})", + "fsd f10, (8*10)({fpu_state})", + "fsd f11, (8*11)({fpu_state})", + "fsd f12, (8*12)({fpu_state})", + "fsd f13, (8*13)({fpu_state})", + "fsd f14, (8*14)({fpu_state})", + "fsd f15, (8*15)({fpu_state})", + "fsd f16, (8*16)({fpu_state})", + "fsd f17, (8*17)({fpu_state})", + "fsd f18, (8*18)({fpu_state})", + "fsd f19, (8*19)({fpu_state})", + "fsd f20, (8*20)({fpu_state})", + "fsd f21, (8*21)({fpu_state})", + "fsd f22, (8*22)({fpu_state})", + "fsd f23, (8*23)({fpu_state})", + "fsd f24, (8*24)({fpu_state})", + "fsd f25, (8*25)({fpu_state})", + "fsd f26, (8*26)({fpu_state})", + "fsd f27, (8*27)({fpu_state})", + "fsd f28, (8*28)({fpu_state})", + "fsd f29, (8*29)({fpu_state})", + "fsd f30, (8*30)({fpu_state})", + "fsd f31, (8*31)({fpu_state})", + "frcsr t0", + "sd t0, (8*32)({fpu_state})", + fpu_state = in(reg) self as *mut _, + out("t0") _, + } + } + } +} + +pub fn generate_random_number32() -> Option { + None +} + +pub fn generate_random_number64() -> Option { + None +} + +pub fn run_on_hypervisor() -> bool { + true +} + +/// Search the most significant bit, indices start at 0 +#[inline(always)] +pub fn msb(value: u64) -> Option { + if value > 0 { + let ret: u64 = 63 - value.leading_zeros() as u64; + Some(ret) + } else { + None + } +} + +/// Search the least significant bit, indices start at 0 +#[inline(always)] +pub fn lsb(value: u64) -> Option { + if value > 0 { + let ret: u64 = value.trailing_zeros() as u64; + Some(ret) + } else { + None + } +} + +/// The halt function stops the processor until the next interrupt arrives +pub fn halt() { + unsafe { + wfi(); + } +} + +/// Shutdown the system +pub fn shutdown() -> ! { + info!("Shutting down system"); + //SBI shutdown + sbi::shutdown_legacy() +} + +pub fn get_timer_ticks() -> u64 { + // We simulate a timer with a 1 microsecond resolution by taking the CPU timestamp + // and dividing it by the CPU frequency in MHz. + get_timestamp() / u64::from(get_frequency()) +} + +pub fn get_frequency() -> u16 { + (get_timebase_freq() / 1000000).try_into().unwrap() +} + +#[inline] +pub fn get_timestamp() -> u64 { + time::read64() +} + +pub fn supports_1gib_pages() -> bool { + true +} + +pub fn supports_2mib_pages() -> bool { + true +} + +/// Delay execution by the given number of microseconds using busy-waiting. +#[inline] +pub fn udelay(usecs: u64) { + let end = get_timestamp() + get_frequency() as u64 * usecs; + while get_timestamp() < end { + spin_loop(); + } +} + +pub fn set_oneshot_timer(wakeup_time: Option) { + if let Some(wt) = wakeup_time { + debug!("Starting Timer: {:x}", get_timestamp()); + unsafe { + sie::set_stimer(); + } + let next_time = wt * u64::from(get_frequency()); + + sbi::set_timer(next_time); + } else { + // Disable the Timer (and clear a pending interrupt) + debug!("Stopping Timer"); + sbi::set_timer(u64::MAX); + } +} + +pub fn wakeup_core(core_to_wakeup: CoreId) { + let hart_id = unsafe { HARTS_AVAILABLE[core_to_wakeup as usize] }; + debug!("Wakeup core: {} , hart_id: {}", core_to_wakeup, hart_id); + sbi::send_ipi(1 << hart_id); +} diff --git a/src/arch/riscv/kernel/sbi.rs b/src/arch/riscv/kernel/sbi.rs new file mode 100644 index 0000000000..1aa4b631f5 --- /dev/null +++ b/src/arch/riscv/kernel/sbi.rs @@ -0,0 +1,188 @@ +//from https://github.com/rcore-os/rCore/blob/master/kernel/src/arch/riscv/sbi.rs +#![allow(dead_code)] + +use core::arch::asm; + +#[derive(Clone, Copy, Debug)] +pub struct SBIRet { + error: isize, + value: usize, +} +#[derive(Clone, Copy, Debug)] +pub struct SBICall { + eid: usize, + fid: usize, +} +#[inline(always)] +fn sbi_call(which: SBICall, arg0: usize, arg1: usize, arg2: usize) -> SBIRet { + let ret1; + let ret2; + unsafe { + // llvm_asm!("ecall" + // : "={x10}" (ret1), "={x11}"(ret2) + // : "{x10}" (arg0), "{x11}" (arg1), "{x12}" (arg2), "{x17}" (which.eid), "{x16}" (which.fid) + // : "memory" + // : "volatile"); + asm!( + "ecall", + in("x10") arg0, + in("x11") arg1, + in("x12") arg2, + in("x17") which.eid, + in("x16") which.fid, + lateout("x10") ret1, + lateout("x11") ret2, + ); + } + SBIRet { + error: ret1, + value: ret2, + } +} + +pub fn sbi_hart_start(hartid: usize, start_addr: usize, opaque: usize) -> SBIRet { + sbi_call( + SBICall { + eid: SBI_EID_HSM, + fid: SBI_FID_HSM_START, + }, + hartid, + start_addr as usize, + opaque, + ) +} +pub fn sbi_hart_stop() -> ! { + sbi_call( + SBICall { + eid: SBI_EID_HSM, + fid: SBI_FID_HSM_STOP, + }, + 0, + 0, + 0, + ); + unreachable!(); +} +pub fn sbi_hart_get_status(hartid: usize) -> SBIRet { + sbi_call( + SBICall { + eid: SBI_EID_HSM, + fid: SBI_FID_HSM_START, + }, + hartid, + 0, + 0, + ) +} + +pub fn sbi_system_reset(reset_type: usize, reset_reason: usize) -> SBIRet { + sbi_call( + SBICall { + eid: SBI_EID_SRST, + fid: SBI_FID_SRST_RESET, + }, + reset_type, + reset_reason, + 0, + ) +} + +const SBI_SUCCESS: isize = 0; +const SBI_ERR_FAILED: isize = -1; +const SBI_ERR_NOT_SUPPORTED: isize = -2; +const SBI_ERR_INVALID_PARAM: isize = -3; +const SBI_ERR_DENIED: isize = -4; +const SBI_ERR_INVALID_ADDRESS: isize = -5; +const SBI_ERR_ALREADY_AVAILABLE: isize = -6; + +const SBI_EID_HSM: usize = 0x48534D; +const SBI_FID_HSM_START: usize = 0; +const SBI_FID_HSM_STOP: usize = 1; +const SBI_FID_HSM_STATUS: usize = 2; + +const SBI_EID_SRST: usize = 0x53525354; +const SBI_FID_SRST_RESET: usize = 0; + +/// Legacy calls. + +#[inline(always)] +fn sbi_call_legacy(which: usize, arg0: usize, arg1: usize, arg2: usize) -> usize { + let ret; + unsafe { + // llvm_asm!("ecall" + // : "={x10}" (ret) + // : "{x10}" (arg0), "{x11}" (arg1), "{x12}" (arg2), "{x17}" (which) + // : "memory" + // : "volatile"); + asm!( + "ecall", + in("x10") arg0, + in("x11") arg1, + in("x12") arg2, + in("x17") which, + lateout("x10") ret, + ); + } + ret +} + +pub fn console_putchar(ch: usize) { + sbi_call_legacy(SBI_CONSOLE_PUTCHAR, ch, 0, 0); +} + +pub fn console_getchar() -> usize { + sbi_call_legacy(SBI_CONSOLE_GETCHAR, 0, 0, 0) +} + +pub fn shutdown_legacy() -> ! { + sbi_call_legacy(SBI_SHUTDOWN, 0, 0, 0); + unreachable!() +} + +pub fn set_timer(stime_value: u64) { + #[cfg(target_pointer_width = "32")] + sbi_call_legacy( + SBI_SET_TIMER, + stime_value as usize, + (stime_value >> 32) as usize, + 0, + ); + #[cfg(target_pointer_width = "64")] + sbi_call_legacy(SBI_SET_TIMER, stime_value as usize, 0, 0); +} + +pub fn clear_ipi() { + sbi_call_legacy(SBI_CLEAR_IPI, 0, 0, 0); +} + +pub fn send_ipi(hart_mask: usize) { + sbi_call_legacy(SBI_SEND_IPI, &hart_mask as *const _ as usize, 0, 0); +} + +pub fn remote_fence_i(hart_mask: usize) { + sbi_call_legacy(SBI_REMOTE_FENCE_I, &hart_mask as *const _ as usize, 0, 0); +} + +pub fn remote_sfence_vma(hart_mask: usize, _start: usize, _size: usize) { + sbi_call_legacy(SBI_REMOTE_SFENCE_VMA, &hart_mask as *const _ as usize, 0, 0); +} + +pub fn remote_sfence_vma_asid(hart_mask: usize, _start: usize, _size: usize, _asid: usize) { + sbi_call_legacy( + SBI_REMOTE_SFENCE_VMA_ASID, + &hart_mask as *const _ as usize, + 0, + 0, + ); +} + +const SBI_SET_TIMER: usize = 0; +const SBI_CONSOLE_PUTCHAR: usize = 1; +const SBI_CONSOLE_GETCHAR: usize = 2; +const SBI_CLEAR_IPI: usize = 3; +const SBI_SEND_IPI: usize = 4; +const SBI_REMOTE_FENCE_I: usize = 5; +const SBI_REMOTE_SFENCE_VMA: usize = 6; +const SBI_REMOTE_SFENCE_VMA_ASID: usize = 7; +const SBI_SHUTDOWN: usize = 8; +// Legacy calls end. diff --git a/src/arch/riscv/kernel/scheduler.rs b/src/arch/riscv/kernel/scheduler.rs new file mode 100644 index 0000000000..c33d50cc5f --- /dev/null +++ b/src/arch/riscv/kernel/scheduler.rs @@ -0,0 +1,486 @@ +use alloc::alloc::{alloc, dealloc, Layout}; +use core::convert::TryInto; +use core::{mem, ptr}; + +use riscv::register::sie; + +use crate::arch::riscv::kernel::percore::*; +use crate::arch::riscv::kernel::processor::set_oneshot_timer; +use crate::arch::riscv::mm::paging::{BasePageSize, PageSize, PageTableEntryFlags}; +use crate::arch::riscv::mm::{PhysAddr, VirtAddr}; +use crate::scheduler::task::{Task, TaskFrame}; +use crate::{env, DEFAULT_STACK_SIZE, KERNEL_STACK_SIZE}; + +/* extern "C" { + static tls_start: u8; + static tls_end: u8; +} */ + +#[repr(C, packed)] +#[derive(Clone, Copy, Debug)] +pub struct State { + /// return address register + ra: usize, + /// stack pointer register + sp: usize, + /// global pointer register + gp: usize, + /// thread pointer register + tp: usize, + /// x5 register + x5: usize, + /// x6 register + x6: usize, + /// x7 register + x7: usize, + /// x8 register + x8: usize, + /// x9 register + x9: usize, + /// Function arguments/return values + a0: usize, + /// a1 register + a1: usize, + /// a2 register + a2: usize, + /// x13 register + x13: usize, + /// x14 register + x14: usize, + /// x15 register + x15: usize, + /// x16 register + x16: usize, + /// x17 register + x17: usize, + /// x18 register + x18: usize, + /// x19 register + x19: usize, + /// x20 register + x20: usize, + /// x21 register + x21: usize, + /// x22 register + x22: usize, + /// x23 register + x23: usize, + /// x24 register + x24: usize, + /// x25 register + x25: usize, + /// x26 register + x26: usize, + /// x27 register + x27: usize, + /// x28 register + x28: usize, + /// x29 register + x29: usize, + /// x30 register + x30: usize, + /// x31 register + x31: usize, +} + +pub struct BootStack { + /// stack for kernel tasks + stack: VirtAddr, + /// stack to handle interrupts + ist0: VirtAddr, +} + +pub struct CommonStack { + /// start address of allocated virtual memory region + virt_addr: VirtAddr, + /// start address of allocated virtual memory region + phys_addr: PhysAddr, + /// total size of all stacks + total_size: usize, +} + +pub enum TaskStacks { + Boot(BootStack), + Common(CommonStack), +} + +impl TaskStacks { + /// Size of the debug marker at the very top of each stack. + /// + /// We have a marker at the very top of the stack for debugging (`0xdeadbeef`), which should not be overridden. + pub const MARKER_SIZE: usize = 0x10; + + pub fn new(size: usize) -> Self { + let user_stack_size = if size < KERNEL_STACK_SIZE { + KERNEL_STACK_SIZE + } else { + align_up!(size, BasePageSize::SIZE as usize) + }; + let total_size = user_stack_size + DEFAULT_STACK_SIZE + KERNEL_STACK_SIZE; + let virt_addr = + crate::arch::mm::virtualmem::allocate(total_size + 4 * BasePageSize::SIZE as usize) + //let virt_addr = crate::arch::mm::virtualmem::allocate(total_size) + .expect("Failed to allocate Virtual Memory for TaskStacks"); + let phys_addr = crate::arch::mm::physicalmem::allocate(total_size) + .expect("Failed to allocate Physical Memory for TaskStacks"); + + debug!( + "Create stacks at {:#X} with a size of {} KB", + virt_addr, + total_size >> 10 + ); + + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().execute_disable(); + + // map IST0 into the address space + crate::arch::mm::paging::map::( + virt_addr + BasePageSize::SIZE as usize, + //virt_addr, + phys_addr, + KERNEL_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + + // map kernel stack into the address space + crate::arch::mm::paging::map::( + virt_addr + KERNEL_STACK_SIZE + 2 * BasePageSize::SIZE as usize, + //virt_addr + KERNEL_STACK_SIZE, + phys_addr + KERNEL_STACK_SIZE, + DEFAULT_STACK_SIZE / BasePageSize::SIZE as usize, + flags, + ); + + // map user stack into the address space + crate::arch::mm::paging::map::( + virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE + 3 * BasePageSize::SIZE as usize, + //virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE, + phys_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE, + user_stack_size / BasePageSize::SIZE as usize, + flags, + ); + + // clear user stack + debug!("Clearing user stack..."); + unsafe { + ptr::write_bytes( + (virt_addr + + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE + + 3 * BasePageSize::SIZE as usize) + //(virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE) + .as_mut_ptr::(), + 0xAC, + user_stack_size, + ); + } + + debug!("Creating stacks finished"); + + TaskStacks::Common(CommonStack { + virt_addr, + phys_addr, + total_size, + }) + } + + pub fn from_boot_stacks() -> TaskStacks { + //let tss = unsafe { &(*PERCORE.tss.get()) }; + /*let stack = VirtAddr::from_usize(tss.rsp[0] as usize + 0x10 - KERNEL_STACK_SIZE); + debug!("Using boot stack {:#X}", stack); + let ist0 = VirtAddr::from_usize(tss.ist[0] as usize + 0x10 - KERNEL_STACK_SIZE); + debug!("IST0 is located at {:#X}", ist0);*/ + let stack = VirtAddr::zero(); + let ist0 = VirtAddr::zero(); + + TaskStacks::Boot(BootStack { stack, ist0 }) + } + + pub fn get_user_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => 0, + TaskStacks::Common(stacks) => { + stacks.total_size - DEFAULT_STACK_SIZE - KERNEL_STACK_SIZE + } + } + } + + pub fn get_user_stack(&self) -> VirtAddr { + match self { + TaskStacks::Boot(_) => VirtAddr::zero(), + TaskStacks::Common(stacks) => { + stacks.virt_addr + + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE + + 3 * BasePageSize::SIZE as usize + //stacks.virt_addr + KERNEL_STACK_SIZE + DEFAULT_STACK_SIZE + } + } + } + + pub fn get_kernel_stack(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.stack, + TaskStacks::Common(stacks) => { + stacks.virt_addr + KERNEL_STACK_SIZE + 2 * BasePageSize::SIZE as usize + //stacks.virt_addr + KERNEL_STACK_SIZE + } + } + } + + pub fn get_kernel_stack_size(&self) -> usize { + match self { + TaskStacks::Boot(_) => KERNEL_STACK_SIZE, + TaskStacks::Common(_) => DEFAULT_STACK_SIZE, + } + } + + pub fn get_interupt_stack(&self) -> VirtAddr { + match self { + TaskStacks::Boot(stacks) => stacks.ist0, + TaskStacks::Common(stacks) => stacks.virt_addr + BasePageSize::SIZE as usize, + //TaskStacks::Common(stacks) => stacks.virt_addr, + } + } + + pub fn get_interupt_stack_size(&self) -> usize { + KERNEL_STACK_SIZE + } +} + +impl Clone for TaskStacks { + fn clone(&self) -> TaskStacks { + match self { + TaskStacks::Boot(_) => TaskStacks::new(0), + TaskStacks::Common(stacks) => { + TaskStacks::new(stacks.total_size - DEFAULT_STACK_SIZE - KERNEL_STACK_SIZE) + } + } + } +} + +impl Drop for TaskStacks { + fn drop(&mut self) { + // we should never deallocate a boot stack + match self { + TaskStacks::Boot(_) => {} + TaskStacks::Common(stacks) => { + debug!( + "Deallocating stacks at {:#X} with a size of {} KB", + stacks.virt_addr, + stacks.total_size >> 10, + ); + + crate::arch::mm::paging::unmap::( + stacks.virt_addr, + stacks.total_size / BasePageSize::SIZE as usize + 4, + //stacks.total_size / BasePageSize::SIZE as usize, + ); + crate::arch::mm::virtualmem::deallocate( + stacks.virt_addr, + stacks.total_size + 4 * BasePageSize::SIZE as usize, + //stacks.total_size, + ); + crate::arch::mm::physicalmem::deallocate(stacks.phys_addr, stacks.total_size); + } + } + } +} + +pub struct TaskTLS { + address: VirtAddr, + tp: VirtAddr, + layout: Layout, +} + +impl TaskTLS { + pub fn new(tls_size: usize) -> Self { + // determine the size of tdata (tls without tbss) + let tdata_size: usize = env::get_tls_filesz(); + // Yes, it does, so we have to allocate TLS memory. + // Allocate enough space for the given size and one more variable of type usize, which holds the tls_pointer. + let tls_allocation_size = align_up!(tls_size, 32); // + mem::size_of::(); + // We allocate in 128 byte granularity (= cache line size) to avoid false sharing + let memory_size = align_up!(tls_allocation_size, 128); + let layout = + Layout::from_size_align(memory_size, 128).expect("TLS has an invalid size / alignment"); + let ptr = VirtAddr(unsafe { alloc(layout) as u64 }); + + // The tls_pointer is the address to the end of the TLS area requested by the task. + let tls_pointer = ptr; // + align_up!(tls_size, 32); + + unsafe { + // Copy over TLS variables with their initial values. + ptr::copy_nonoverlapping( + env::get_tls_start().as_ptr::(), + ptr.as_mut_ptr::(), + tdata_size, + ); + + ptr::write_bytes( + ptr.as_mut_ptr::() + .offset(tdata_size.try_into().unwrap()), + 0, + align_up!(tls_size, 32) - tdata_size, + ); + + // The x86-64 TLS specification also requires that the tls_pointer can be accessed at fs:0. + // This allows TLS variable values to be accessed by "mov rax, fs:0" and a later "lea rdx, [rax+VARIABLE_OFFSET]". + // See "ELF Handling For Thread-Local Storage", version 0.20 by Ulrich Drepper, page 12 for details. + // + // fs:0 is where tls_pointer points to and we have reserved space for a usize value above. + //*(tls_pointer.as_mut_ptr::()) = tls_pointer.as_u64(); + } + + debug!( + "Set up TLS at 0x{:x}, tdata_size 0x{:x}, tls_size 0x{:x}", + tls_pointer, tdata_size, tls_size + ); + + Self { + address: ptr, + tp: tls_pointer, + layout: layout, + } + } + + #[inline] + pub fn address(&self) -> VirtAddr { + self.address + } + + #[inline] + pub fn get_tp(&self) -> VirtAddr { + self.tp + } +} + +impl Drop for TaskTLS { + fn drop(&mut self) { + debug!( + "Deallocate TLS at 0x{:x} (layout {:?})", + self.address, self.layout, + ); + + unsafe { + dealloc(self.address.as_mut_ptr::(), self.layout); + } + } +} + +impl Clone for TaskTLS { + fn clone(&self) -> Self { + TaskTLS::new(env::get_tls_memsz()) + } +} + +extern "C" fn leave_task() -> ! { + core_scheduler().exit(0); +} + +#[no_mangle] +extern "C" fn task_entry(func: extern "C" fn(usize), arg: usize) { + // Check if the task (process or thread) uses Thread-Local-Storage. + /*let tls_size = unsafe { &tls_end as *const u8 as usize - &tls_start as *const u8 as usize }; + if tls_size > 0 { + // Yes, it does, so we have to allocate TLS memory. + // Allocate enough space for the given size and one more variable of type usize, which holds the tls_pointer. + let tls_allocation_size = tls_size + mem::size_of::(); + let tls = TaskTLS::new(tls_allocation_size); + + // The tls_pointer is the address to the end of the TLS area requested by the task. + let tls_pointer = tls.address() + tls_size; + + // TODO: Implement AArch64 TLS + + // Associate the TLS memory to the current task. + let mut current_task_borrowed = core_scheduler().current_task.borrow_mut(); + debug!( + "Set up TLS for task {} at address {:#X}", + current_task_borrowed.id, + tls.address() + ); + current_task_borrowed.tls = Some(tls); + }*/ + + // Call the actual entry point of the task. + //unsafe{debug!("state: {:#X?}", *((func as usize -31*8 ) as *const crate::arch::riscv::kernel::scheduler::State));} + //panic!("Not impl"); + //println!("Task start"); + func(arg); + //println!("Task end"); + + // switch_to_kernel!(); + + // Exit task + core_scheduler().exit(0) +} + +impl TaskFrame for Task { + fn create_stack_frame(&mut self, func: extern "C" fn(usize), arg: usize) { + // Check if the task (process or thread) uses Thread-Local-Storage. + let tls_size = env::get_tls_memsz(); + // check is TLS is already allocated + if self.tls.is_none() && tls_size > 0 { + self.tls = Some(TaskTLS::new(tls_size)) + } + + unsafe { + // Set a marker for debugging at the very top. + let mut stack = + self.stacks.get_kernel_stack() + self.stacks.get_kernel_stack_size() - 0x10u64; + *stack.as_mut_ptr::() = 0xDEAD_BEEFu64; + + // Put the State structure expected by the ASM switch() function on the stack. + stack = stack - mem::size_of::(); + + let state = stack.as_mut_ptr::(); + ptr::write_bytes(stack.as_mut_ptr::(), 0, mem::size_of::()); + + if let Some(tls) = &self.tls { + (*state).tp = tls.get_tp().as_usize(); + } + (*state).ra = task_start as usize; + (*state).a0 = func as usize; + (*state).a1 = arg as usize; + + // Set the task's stack pointer entry to the stack we have just crafted. + self.last_stack_pointer = stack; + self.user_stack_pointer = + self.stacks.get_user_stack() + self.stacks.get_user_stack_size() - 0x10u64; + + (*state).sp = self.last_stack_pointer.as_usize(); + (*state).a2 = self.user_stack_pointer.as_usize() - mem::size_of::(); + // trace!("state: {:#X?}", *state); + } + } +} + +extern "C" { + fn task_start(func: extern "C" fn(usize), arg: usize, user_stack: u64); +} + +pub fn timer_handler() { + //increment_irq_counter(apic::TIMER_INTERRUPT_NUMBER.into()); + core_scheduler().handle_waiting_tasks(); + set_oneshot_timer(None); + core_scheduler().scheduler(); +} + +#[cfg(feature = "smp")] +pub fn wakeup_handler() { + debug!("Received Wakeup Interrupt"); + //increment_irq_counter(WAKEUP_INTERRUPT_NUMBER.into()); + let core_scheduler = core_scheduler(); + core_scheduler.check_input(); + unsafe { + sie::clear_ssoft(); + } + if core_scheduler.is_scheduling() { + core_scheduler.scheduler(); + } +} + +#[inline(never)] +#[no_mangle] +pub fn set_current_kernel_stack() { + core_scheduler().set_current_kernel_stack(); +} diff --git a/src/arch/riscv/kernel/serial.rs b/src/arch/riscv/kernel/serial.rs new file mode 100644 index 0000000000..d9aa15033e --- /dev/null +++ b/src/arch/riscv/kernel/serial.rs @@ -0,0 +1,23 @@ +// TODO: sifive UART +use crate::arch::riscv::kernel::sbi; + +pub struct SerialPort {} + +impl SerialPort { + pub const fn new(port_address: u32) -> Self { + Self {} + } + + pub fn write_byte(&self, byte: u8) { + // LF newline characters need to be extended to CRLF over a real serial port. + if byte == b'\n' { + sbi::console_putchar('\r' as usize); + } + + sbi::console_putchar(byte as usize); + } + + pub fn init(&self, _baudrate: u32) { + // We don't do anything here (yet). + } +} diff --git a/src/arch/riscv/kernel/start.rs b/src/arch/riscv/kernel/start.rs new file mode 100644 index 0000000000..6f4fc64caf --- /dev/null +++ b/src/arch/riscv/kernel/start.rs @@ -0,0 +1,50 @@ +use core::arch::asm; + +#[cfg(not(feature = "smp"))] +use crate::arch::riscv::kernel::processor; +use crate::arch::riscv::kernel::{BootInfo, BOOTINFO_MAGIC_NUMBER, BOOT_INFO}; +use crate::KERNEL_STACK_SIZE; + +//static mut BOOT_STACK: [u8; KERNEL_STACK_SIZE] = [0; KERNEL_STACK_SIZE]; + +/// Entrypoint - Initalize Stack pointer and Exception Table +#[no_mangle] +#[naked] +pub unsafe extern "C" fn _start() -> ! { + asm!( + "ld sp, (0x48)(a1)", + "li t0, {top_offset}", + "add sp, sp, t0", + //"mv a0, a1", + "j {pre_init}", //call or j? + //boot_stack = sym BOOT_STACK, + top_offset = const KERNEL_STACK_SIZE - 16, /*Previous version subtracted 0x10 from End, so I'm doing this too. Not sure why though */ + pre_init = sym pre_init, + options(noreturn), + ) +} + +#[inline(never)] +#[no_mangle] +unsafe fn pre_init(hart_id: usize, boot_info: &'static mut BootInfo) -> ! { + BOOT_INFO = boot_info as *mut BootInfo; + + // info!("Welcome to hermit kernel."); + assert_eq!(boot_info.magic_number, BOOTINFO_MAGIC_NUMBER); + + core::ptr::write_volatile(&mut (*BOOT_INFO).current_boot_id, hart_id as u32); + + if boot_info.cpu_online == 0 { + crate::boot_processor_main() + } else { + #[cfg(not(feature = "smp"))] + { + error!("SMP support deactivated"); + loop { + processor::halt(); + } + } + #[cfg(feature = "smp")] + crate::application_processor_main(); + } +} diff --git a/src/arch/riscv/kernel/switch.rs b/src/arch/riscv/kernel/switch.rs new file mode 100644 index 0000000000..c62ac700ae --- /dev/null +++ b/src/arch/riscv/kernel/switch.rs @@ -0,0 +1,7 @@ +use core::arch::global_asm; + +global_asm!(include_str!("switch.s")); + +extern "C" { + pub fn switch_to_task(old_stack: *mut usize, new_stack: usize); +} diff --git a/src/arch/riscv/kernel/switch.s b/src/arch/riscv/kernel/switch.s new file mode 100755 index 0000000000..65725a67c6 --- /dev/null +++ b/src/arch/riscv/kernel/switch.s @@ -0,0 +1,168 @@ +.section .text +.global switch_to_task +.global task_start +.extern task_entry +// .extern set_current_kernel_stack + + +.align 16 +// This function should only be called if the fp registers are clean +switch_to_task: + // a0 = old_stack => the address to store the old rsp + // a1 = new_stack => stack pointer of the new task + + addi sp, sp, -(31*8) + sd x31, (8*30)(sp) + sd x30, (8*29)(sp) + sd x29, (8*28)(sp) + sd x28, (8*27)(sp) + sd x27, (8*26)(sp) + sd x26, (8*25)(sp) + sd x25, (8*24)(sp) + sd x24, (8*23)(sp) + sd x23, (8*22)(sp) + sd x22, (8*21)(sp) + sd x21, (8*20)(sp) + sd x20, (8*19)(sp) + sd x19, (8*18)(sp) + sd x18, (8*17)(sp) + sd x17, (8*16)(sp) + sd x16, (8*15)(sp) + sd x15, (8*14)(sp) + sd x14, (8*13)(sp) + sd x13, (8*12)(sp) + sd x12, (8*11)(sp) + sd x11, (8*10)(sp) + sd x10, (8*9)(sp) + sd x9, (8*8)(sp) + sd x8, (8*7)(sp) + sd x7, (8*6)(sp) + sd x6, (8*5)(sp) + sd x5, (8*4)(sp) + sd x4, (8*3)(sp) + //sd x3, (8*2)(sp) + //sd x2, (8*1)(sp) + sd x1, (8*0)(sp) + + //Store floating point registers + //TODO: Save only when changed + # fsd f0, (8*31)(sp) + # fsd f1, (8*32)(sp) + # fsd f2, (8*33)(sp) + # fsd f3, (8*34)(sp) + # fsd f4, (8*35)(sp) + # fsd f5, (8*36)(sp) + # fsd f6, (8*37)(sp) + # fsd f7, (8*38)(sp) + # fsd f8, (8*39)(sp) + # fsd f9, (8*40)(sp) + # fsd f10, (8*41)(sp) + # fsd f11, (8*42)(sp) + # fsd f12, (8*43)(sp) + # fsd f13, (8*44)(sp) + # fsd f14, (8*45)(sp) + # fsd f15, (8*46)(sp) + # fsd f16, (8*47)(sp) + # fsd f17, (8*48)(sp) + # fsd f18, (8*49)(sp) + # fsd f19, (8*50)(sp) + # fsd f20, (8*51)(sp) + # fsd f21, (8*52)(sp) + # fsd f22, (8*53)(sp) + # fsd f23, (8*54)(sp) + # fsd f24, (8*55)(sp) + # fsd f25, (8*56)(sp) + # fsd f26, (8*57)(sp) + # fsd f27, (8*58)(sp) + # fsd f28, (8*59)(sp) + # fsd f29, (8*60)(sp) + # fsd f30, (8*61)(sp) + # fsd f31, (8*62)(sp) + # frcsr t0 + # sd t0, (8*63)(sp) + + // Store current stack pointer with saved context in `_dst`. + sd sp, (0)(a0) + // Set stack pointer to supplied `_src`. + mv sp, a1 + + //set current kernel stack + call set_current_kernel_stack + + # // Restore fp regs + # fld f0, (8*31)(sp) + # fld f1, (8*32)(sp) + # fld f2, (8*33)(sp) + # fld f3, (8*34)(sp) + # fld f4, (8*35)(sp) + # fld f5, (8*36)(sp) + # fld f6, (8*37)(sp) + # fld f7, (8*38)(sp) + # fld f8, (8*39)(sp) + # fld f9, (8*40)(sp) + # fld f10, (8*41)(sp) + # fld f11, (8*42)(sp) + # fld f12, (8*43)(sp) + # fld f13, (8*44)(sp) + # fld f14, (8*45)(sp) + # fld f15, (8*46)(sp) + # fld f16, (8*47)(sp) + # fld f17, (8*48)(sp) + # fld f18, (8*49)(sp) + # fld f19, (8*50)(sp) + # fld f20, (8*51)(sp) + # fld f21, (8*52)(sp) + # fld f22, (8*53)(sp) + # fld f23, (8*54)(sp) + # fld f24, (8*55)(sp) + # fld f25, (8*56)(sp) + # fld f26, (8*57)(sp) + # fld f27, (8*58)(sp) + # fld f28, (8*59)(sp) + # fld f29, (8*60)(sp) + # fld f30, (8*61)(sp) + # fld f31, (8*62)(sp) + # ld t0, (8*63)(sp) + # fscsr t0 + + // Restore context + ld x1, (8*0)(sp) + //ld x2, (8*1)(sp) + //ld x3, (8*2)(sp) + ld x4, (8*3)(sp) + ld x5, (8*4)(sp) + ld x6, (8*5)(sp) + ld x7, (8*6)(sp) + ld x8, (8*7)(sp) + ld x9, (8*8)(sp) + ld x10, (8*9)(sp) + ld x11, (8*10)(sp) + ld x12, (8*11)(sp) + ld x13, (8*12)(sp) + ld x14, (8*13)(sp) + ld x15, (8*14)(sp) + ld x16, (8*15)(sp) + ld x17, (8*16)(sp) + ld x18, (8*17)(sp) + ld x19, (8*18)(sp) + ld x20, (8*19)(sp) + ld x21, (8*20)(sp) + ld x22, (8*21)(sp) + ld x23, (8*22)(sp) + ld x24, (8*23)(sp) + ld x25, (8*24)(sp) + ld x26, (8*25)(sp) + ld x27, (8*26)(sp) + ld x28, (8*27)(sp) + ld x29, (8*28)(sp) + ld x30, (8*29)(sp) + ld x31, (8*30)(sp) + + addi sp, sp, (31*8) + + ret + +.align 16 +task_start: + mv sp, a2 + j task_entry \ No newline at end of file diff --git a/src/arch/riscv/kernel/systemtime.rs b/src/arch/riscv/kernel/systemtime.rs new file mode 100644 index 0000000000..e14786c8d1 --- /dev/null +++ b/src/arch/riscv/kernel/systemtime.rs @@ -0,0 +1,5 @@ +use crate::arch::riscv::kernel::BOOT_INFO; + +pub fn get_boot_time() -> u64 { + unsafe { core::ptr::read_volatile(&(*BOOT_INFO).boot_gtod) } +} diff --git a/src/arch/riscv/mm/addr.rs b/src/arch/riscv/mm/addr.rs new file mode 100644 index 0000000000..bc3f31d819 --- /dev/null +++ b/src/arch/riscv/mm/addr.rs @@ -0,0 +1,707 @@ +use core::convert::{From, Into}; +use core::hash::{Hash, Hasher}; +use core::{fmt, ops}; + +/// Align address downwards. +/// +/// Returns the greatest x with alignment `align` so that x <= addr. +/// The alignment must be a power of 2. +#[inline(always)] +fn align_down(addr: u64, align: u64) -> u64 { + addr & !(align - 1) +} + +/// Align address upwards. +/// +/// Returns the smallest x with alignment `align` so that x >= addr. +/// The alignment must be a power of 2. +#[inline(always)] +fn align_up(addr: u64, align: u64) -> u64 { + let align_mask = align - 1; + if addr & align_mask == 0 { + addr + } else { + (addr | align_mask) + 1 + } +} + +/// A wrapper for a physical address, which is in principle +/// derived from the crate x86. +#[repr(transparent)] +#[derive(Copy, Clone, Eq, Ord, PartialEq, PartialOrd)] +pub struct PhysAddr(pub u64); + +impl PhysAddr { + /// Convert to `u64` + pub fn as_u64(self) -> u64 { + self.0 + } + + /// Convert to `usize` + pub fn as_usize(self) -> usize { + self.0 as usize + } + + /// Physical Address zero. + pub const fn zero() -> Self { + PhysAddr(0) + } + + /// Is zero? + pub fn is_zero(self) -> bool { + self == PhysAddr::zero() + } + + fn align_up(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_up(self.0, align.into())) + } + + fn align_down(self, align: U) -> Self + where + U: Into, + { + PhysAddr(align_down(self.0, align.into())) + } + + /// Is this address aligned to `align`? + /// + /// # Note + /// `align` must be a power of two. + pub fn is_aligned(self, align: U) -> bool + where + U: Into + Copy, + { + if !align.into().is_power_of_two() { + return false; + } + + self.align_down(align) == self + } +} + +impl From for PhysAddr { + fn from(num: u64) -> Self { + PhysAddr(num) + } +} + +impl From for PhysAddr { + fn from(num: usize) -> Self { + PhysAddr(num as u64) + } +} + +impl From for PhysAddr { + fn from(num: i32) -> Self { + PhysAddr(num as u64) + } +} + +impl Into for PhysAddr { + fn into(self) -> u64 { + self.0 + } +} + +impl Into for PhysAddr { + fn into(self) -> usize { + self.0 as usize + } +} + +impl ops::Add for PhysAddr { + type Output = PhysAddr; + + fn add(self, rhs: PhysAddr) -> Self::Output { + PhysAddr(self.0 + rhs.0) + } +} + +impl ops::Add for PhysAddr { + type Output = PhysAddr; + + fn add(self, rhs: u64) -> Self::Output { + PhysAddr::from(self.0 + rhs) + } +} + +impl ops::Add for PhysAddr { + type Output = PhysAddr; + + fn add(self, rhs: usize) -> Self::Output { + PhysAddr::from(self.0 + rhs as u64) + } +} + +impl ops::AddAssign for PhysAddr { + fn add_assign(&mut self, other: PhysAddr) { + *self = PhysAddr::from(self.0 + other.0); + } +} + +impl ops::AddAssign for PhysAddr { + fn add_assign(&mut self, offset: u64) { + *self = PhysAddr::from(self.0 + offset); + } +} + +impl ops::Sub for PhysAddr { + type Output = PhysAddr; + + fn sub(self, rhs: PhysAddr) -> Self::Output { + PhysAddr::from(self.0 - rhs.0) + } +} + +impl ops::Sub for PhysAddr { + type Output = PhysAddr; + + fn sub(self, rhs: u64) -> Self::Output { + PhysAddr::from(self.0 - rhs) + } +} + +impl ops::Sub for PhysAddr { + type Output = PhysAddr; + + fn sub(self, rhs: usize) -> Self::Output { + PhysAddr::from(self.0 - rhs as u64) + } +} + +impl ops::Rem for PhysAddr { + type Output = PhysAddr; + + fn rem(self, rhs: PhysAddr) -> Self::Output { + PhysAddr(self.0 % rhs.0) + } +} + +impl ops::Rem for PhysAddr { + type Output = u64; + + fn rem(self, rhs: u64) -> Self::Output { + self.0 % rhs + } +} + +impl ops::Rem for PhysAddr { + type Output = u64; + + fn rem(self, rhs: usize) -> Self::Output { + self.0 % (rhs as u64) + } +} + +impl ops::BitAnd for PhysAddr { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self { + PhysAddr(self.0 & rhs.0) + } +} + +impl ops::BitAnd for PhysAddr { + type Output = u64; + + fn bitand(self, rhs: u64) -> Self::Output { + Into::::into(self) & rhs + } +} + +impl ops::BitOr for PhysAddr { + type Output = PhysAddr; + + fn bitor(self, rhs: PhysAddr) -> Self::Output { + PhysAddr(self.0 | rhs.0) + } +} + +impl ops::BitOr for PhysAddr { + type Output = u64; + + fn bitor(self, rhs: u64) -> Self::Output { + self.0 | rhs + } +} + +impl ops::Shr for PhysAddr { + type Output = u64; + + fn shr(self, rhs: u64) -> Self::Output { + self.0 >> rhs + } +} + +impl fmt::Binary for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Display for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Debug for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#x}", self.0) + } +} + +impl fmt::LowerHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Octal for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::UpperHex for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Pointer for PhysAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use core::fmt::LowerHex; + self.0.fmt(f) + } +} + +impl Hash for PhysAddr { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +/// A wrapper for a virtual address, which is in principle +/// derived from the crate x86. +#[repr(transparent)] +#[derive(Copy, Clone, Eq, Ord, PartialEq, PartialOrd)] +pub struct VirtAddr(pub u64); + +impl VirtAddr { + /// Convert from `u64` + pub const fn from_u64(v: u64) -> Self { + VirtAddr(v) + } + + /// Convert from `usize` + pub const fn from_usize(v: usize) -> Self { + VirtAddr(v as u64) + } + + /// Convert to `u64` + pub const fn as_u64(self) -> u64 { + self.0 + } + + /// Convert to `usize` + pub const fn as_usize(self) -> usize { + self.0 as usize + } + + /// Convert to mutable pointer. + pub fn as_mut_ptr(self) -> *mut T { + self.0 as *mut T + } + + /// Convert to pointer. + pub fn as_ptr(self) -> *const T { + self.0 as *const T + } + + /// Physical Address zero. + pub const fn zero() -> Self { + VirtAddr(0) + } + + /// Is zero? + pub fn is_zero(self) -> bool { + self == VirtAddr::zero() + } + + fn align_up(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_up(self.0, align.into())) + } + + fn align_down(self, align: U) -> Self + where + U: Into, + { + VirtAddr(align_down(self.0, align.into())) + } + + /// Offset within the 4 KiB page. + pub fn base_page_offset(self) -> u64 { + self.0 & (BASE_PAGE_SIZE as u64 - 1) + } + + /// Offset within the 2 MiB page. + pub fn large_page_offset(self) -> u64 { + self.0 & (MEGA_PAGE_SIZE as u64 - 1) + } + + /// Offset within the 1 GiB page. + pub fn giga_page_offset(self) -> u64 { + self.0 & (GIGA_PAGE_SIZE as u64 - 1) + } + + /// Offset within the 512 GiB page. + pub fn tera_page_offset(self) -> u64 { + self.0 & (TERA_PAGE_SIZE as u64 - 1) + } + + /// Return address of nearest 4 KiB page (lower or equal than self). + pub fn align_down_to_base_page(self) -> Self { + self.align_down(BASE_PAGE_SIZE as u64) + } + + /// Return address of nearest 2 MiB page (lower or equal than self). + pub fn align_down_to_large_page(self) -> Self { + self.align_down(MEGA_PAGE_SIZE as u64) + } + + /// Return address of nearest 1 GiB page (lower or equal than self). + pub fn align_down_to_giga_page(self) -> Self { + self.align_down(GIGA_PAGE_SIZE as u64) + } + + /// Return address of nearest 512 GiB page (lower or equal than self). + pub fn align_down_to_tera_page(self) -> Self { + self.align_down(TERA_PAGE_SIZE as u64) + } + + /// Return address of nearest 4 KiB page (higher or equal than self). + pub fn align_up_to_base_page(self) -> Self { + self.align_up(BASE_PAGE_SIZE as u64) + } + + /// Return address of nearest 2 MiB page (higher or equal than self). + pub fn align_up_to_large_page(self) -> Self { + self.align_up(MEGA_PAGE_SIZE as u64) + } + + /// Return address of nearest 1 GiB page (higher or equal than self). + pub fn align_up_to_giga_page(self) -> Self { + self.align_up(GIGA_PAGE_SIZE as u64) + } + + /// Return address of nearest 512 GiB page (higher or equal than self). + pub fn align_up_to_tera_page(self) -> Self { + self.align_up(TERA_PAGE_SIZE as u64) + } + + /// Is this address aligned to a 4 KiB page? + pub fn is_base_page_aligned(self) -> bool { + self.align_down(BASE_PAGE_SIZE as u64) == self + } + + /// Is this address aligned to a 2 MiB page? + pub fn is_large_page_aligned(self) -> bool { + self.align_down(MEGA_PAGE_SIZE as u64) == self + } + + /// Is this address aligned to a 1 GiB page? + pub fn is_giga_page_aligned(self) -> bool { + self.align_down(GIGA_PAGE_SIZE as u64) == self + } + + /// Is this address aligned to a 512 GiB page? + pub fn is_tera_page_aligned(self) -> bool { + self.align_down(TERA_PAGE_SIZE as u64) == self + } + + /// Is this address aligned to `align`? + /// + /// # Note + /// `align` must be a power of two. + pub fn is_aligned(self, align: U) -> bool + where + U: Into + Copy, + { + if !align.into().is_power_of_two() { + return false; + } + + self.align_down(align) == self + } +} + +impl From for VirtAddr { + fn from(num: u64) -> Self { + VirtAddr(num) + } +} + +impl From for VirtAddr { + fn from(num: i32) -> Self { + VirtAddr(num as u64) + } +} + +impl Into for VirtAddr { + fn into(self) -> u64 { + self.0 + } +} + +impl From for VirtAddr { + fn from(num: usize) -> Self { + VirtAddr(num as u64) + } +} + +impl Into for VirtAddr { + fn into(self) -> usize { + self.0 as usize + } +} + +impl ops::Add for VirtAddr { + type Output = VirtAddr; + + fn add(self, rhs: VirtAddr) -> Self::Output { + VirtAddr(self.0 + rhs.0) + } +} + +impl ops::Add for VirtAddr { + type Output = VirtAddr; + + fn add(self, rhs: u64) -> Self::Output { + VirtAddr(self.0 + rhs) + } +} + +impl ops::Add for VirtAddr { + type Output = VirtAddr; + + fn add(self, rhs: usize) -> Self::Output { + VirtAddr::from(self.0 + rhs as u64) + } +} + +impl ops::AddAssign for VirtAddr { + fn add_assign(&mut self, other: VirtAddr) { + *self = VirtAddr::from(self.0 + other.0); + } +} + +impl ops::AddAssign for VirtAddr { + fn add_assign(&mut self, offset: u64) { + *self = VirtAddr::from(self.0 + offset); + } +} + +impl ops::AddAssign for VirtAddr { + fn add_assign(&mut self, offset: usize) { + *self = VirtAddr::from(self.0 + offset as u64); + } +} + +impl ops::Sub for VirtAddr { + type Output = VirtAddr; + + fn sub(self, rhs: VirtAddr) -> Self::Output { + VirtAddr::from(self.0 - rhs.0) + } +} + +impl ops::Sub for VirtAddr { + type Output = VirtAddr; + + fn sub(self, rhs: u64) -> Self::Output { + VirtAddr::from(self.0 - rhs) + } +} + +impl ops::Sub for VirtAddr { + type Output = VirtAddr; + + fn sub(self, rhs: usize) -> Self::Output { + VirtAddr::from(self.0 - rhs as u64) + } +} + +impl ops::Rem for VirtAddr { + type Output = VirtAddr; + + fn rem(self, rhs: VirtAddr) -> Self::Output { + VirtAddr(self.0 % rhs.0) + } +} + +impl ops::Rem for VirtAddr { + type Output = u64; + + fn rem(self, rhs: Self::Output) -> Self::Output { + self.0 % rhs + } +} + +impl ops::Rem for VirtAddr { + type Output = usize; + + fn rem(self, rhs: Self::Output) -> Self::Output { + self.as_usize() % rhs + } +} + +impl ops::BitAnd for VirtAddr { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self::Output { + VirtAddr(self.0 & rhs.0) + } +} + +impl ops::BitAnd for VirtAddr { + type Output = VirtAddr; + + fn bitand(self, rhs: u64) -> Self::Output { + VirtAddr(self.0 & rhs) + } +} + +impl ops::BitAnd for VirtAddr { + type Output = VirtAddr; + + fn bitand(self, rhs: usize) -> Self::Output { + VirtAddr(self.0 & rhs as u64) + } +} + +impl ops::BitAnd for VirtAddr { + type Output = VirtAddr; + + fn bitand(self, rhs: i32) -> Self::Output { + VirtAddr(self.0 & rhs as u64) + } +} + +impl ops::BitOr for VirtAddr { + type Output = VirtAddr; + + fn bitor(self, rhs: VirtAddr) -> VirtAddr { + VirtAddr(self.0 | rhs.0) + } +} + +impl ops::BitOr for VirtAddr { + type Output = VirtAddr; + + fn bitor(self, rhs: u64) -> Self::Output { + VirtAddr(self.0 | rhs) + } +} + +impl ops::BitOr for VirtAddr { + type Output = VirtAddr; + + fn bitor(self, rhs: usize) -> Self::Output { + VirtAddr(self.0 | rhs as u64) + } +} + +impl ops::Shr for VirtAddr { + type Output = u64; + + fn shr(self, rhs: u64) -> Self::Output { + self.0 >> rhs as u64 + } +} + +impl ops::Shr for VirtAddr { + type Output = u64; + + fn shr(self, rhs: usize) -> Self::Output { + self.0 >> rhs as u64 + } +} + +impl ops::Shr for VirtAddr { + type Output = u64; + + fn shr(self, rhs: i32) -> Self::Output { + self.0 >> rhs as u64 + } +} + +impl fmt::Binary for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Display for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#x}", self.0) + } +} + +impl fmt::Debug for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:#x}", self.0) + } +} + +impl fmt::LowerHex for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Octal for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::UpperHex for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Pointer for VirtAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use core::fmt::LowerHex; + self.0.fmt(f) + } +} + +impl Hash for VirtAddr { + fn hash(&self, state: &mut H) { + self.0.hash(state); + } +} + +/// Log2 of base page size (12 bits). +pub const BASE_PAGE_SHIFT: usize = 12; + +/// Size of a base page (4 KiB) +pub const BASE_PAGE_SIZE: usize = 4096; + +/// Size of a mega page (2 MiB) +pub const MEGA_PAGE_SIZE: usize = 1024 * 1024 * 2; + +/// Size of a giga page (1 GiB) +pub const GIGA_PAGE_SIZE: usize = 1024 * 1024 * 1024; + +/// Size of a tera page (512 GiB) +pub const TERA_PAGE_SIZE: usize = 1024 * 1024 * 1024 * 512; diff --git a/src/arch/riscv/mm/mod.rs b/src/arch/riscv/mm/mod.rs new file mode 100644 index 0000000000..ca1564a238 --- /dev/null +++ b/src/arch/riscv/mm/mod.rs @@ -0,0 +1,13 @@ +pub mod addr; +pub mod paging; +pub mod physicalmem; +pub mod virtualmem; + +pub use addr::{PhysAddr, VirtAddr}; + +pub use self::paging::init_page_tables; + +pub fn init() { + physicalmem::init(); + virtualmem::init(); +} diff --git a/src/arch/riscv/mm/paging.rs b/src/arch/riscv/mm/paging.rs new file mode 100644 index 0000000000..c41a2931c9 --- /dev/null +++ b/src/arch/riscv/mm/paging.rs @@ -0,0 +1,748 @@ +use core::marker::PhantomData; +use core::usize; + +use riscv::asm::sfence_vma; +use riscv::register::satp; + +use crate::arch::riscv::kernel::get_ram_address; +use crate::arch::riscv::mm::{physicalmem, PhysAddr, VirtAddr}; + +static mut ROOT_PAGETABLE: PageTable = PageTable::new(); + +/// Number of Offset bits of a virtual address for a 4 KiB page, which are shifted away to get its Page Frame Number (PFN). +const PAGE_BITS: usize = 12; + +/// Number of bits of the index in each table +const PAGE_MAP_BITS: usize = 9; + +/// A mask where PAGE_MAP_BITS are set to calculate a table index. +const PAGE_MAP_MASK: usize = 0x1FF; + +/// Number of page levels +const PAGE_LEVELS: usize = 3; + +bitflags! { + /// Flags for an PTE + /// + /// See The RISC-V Instruction Set Manual Volume II: Privileged Architecture + pub struct PageTableEntryFlags: u64 { + /// Set if this entry is valid. + const VALID = 1 << 0; + + /// Set if this page is readable + const READABLE = 1 << 1; + + /// Set if this page is writable + const WRITABLE = 1 << 2; + + /// Set if this page is executable + const EXECUTABLE = 1 << 3; + + /// Set if memory referenced by this entry shall be accessible from user-mode + const USER_ACCESSIBLE = 1 << 4; + + /// Set if mapping exists in all address spaces + const GLOBAL = 1 << 5; + + /// Set if software has accessed this entry + const ACCESSED = 1 << 6; + + /// Only for page entries: Set if software has written to the memory referenced by this entry. + const DIRTY = 1 << 7; + + /// The RSW field is reserved for use by supervisor + const RSW = 1 << 8 | 1 << 9; + } +} + +impl PageTableEntryFlags { + /// An empty set of flags for unused/zeroed table entries. + /// Needed as long as empty() is no const function. + const BLANK: PageTableEntryFlags = PageTableEntryFlags { bits: 0 }; + + pub fn device(&mut self) -> &mut Self { + self + } + + pub fn normal(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::EXECUTABLE); + self.insert(PageTableEntryFlags::READABLE); + self + } + + pub fn read_only(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::WRITABLE); + self + } + + pub fn writable(&mut self) -> &mut Self { + self.insert(PageTableEntryFlags::WRITABLE); + self + } + + pub fn execute_disable(&mut self) -> &mut Self { + self.remove(PageTableEntryFlags::EXECUTABLE); + self + } +} + +/// An entry in either table +#[derive(Clone, Copy, Debug)] +pub struct PageTableEntry { + /// Physical memory address this entry refers, combined with flags from PageTableEntryFlags. + physical_address_and_flags: PhysAddr, +} + +impl PageTableEntry { + /// Return the stored physical address. + pub fn address(&self) -> PhysAddr { + PhysAddr( + ( + self.physical_address_and_flags.as_u64() & !(0x3FFu64) + //& !(0x3FFu64 << 54) + ) << 2, + ) + } + + /// Returns whether this entry is valid (present). + fn is_present(&self) -> bool { + (self.physical_address_and_flags & PageTableEntryFlags::VALID.bits()) != 0 + } + + /// Returns `true` if the page is accessible from the user space + fn is_user(self) -> bool { + (self.physical_address_and_flags & PageTableEntryFlags::USER_ACCESSIBLE.bits()) != 0 + } + + /// Returns `true` if the page is readable + fn is_readable(self) -> bool { + (self.physical_address_and_flags & PageTableEntryFlags::READABLE.bits()) != 0 + } + + /// Returns `true` if the page is writable + fn is_writable(self) -> bool { + (self.physical_address_and_flags & PageTableEntryFlags::WRITABLE.bits()) != 0 + } + + /// Returns `true` if the page is executable + fn is_executable(self) -> bool { + (self.physical_address_and_flags & PageTableEntryFlags::EXECUTABLE.bits()) != 0 + } + + /// Mark this as a valid (present) entry and set address translation and flags. + /// + /// # Arguments + /// + /// * `physical_address` - The physical memory address this entry shall translate to + /// * `flags` - Flags from PageTableEntryFlags (note that the VALID, GLOBAL, DIRTY and ACCESSED flags are set) + fn set(&mut self, physical_address: PhysAddr, flags: PageTableEntryFlags) { + // Verify that the offset bits for a 4 KiB page are zero. + assert_eq!( + physical_address % BasePageSize::SIZE as usize, + 0, + "Physical address is not on a 4 KiB page boundary (physical_address = {:#X})", + physical_address + ); + + let mut flags_to_set = flags; + flags_to_set.insert(PageTableEntryFlags::VALID); + flags_to_set.insert(PageTableEntryFlags::GLOBAL); + self.physical_address_and_flags = + PhysAddr((physical_address.as_u64() >> 2) | flags_to_set.bits()); + } +} + +/// A generic interface to support all possible page sizes. +/// +/// This is defined as a subtrait of Copy to enable #[derive(Clone, Copy)] for Page. +/// Currently, deriving implementations for these traits only works if all dependent types implement it as well. +pub trait PageSize: Copy { + /// The page size in bytes. + const SIZE: u64; + + /// The page table level at which a page of this size is mapped + const MAP_LEVEL: usize; + + /// Any extra flag that needs to be set to map a page of this size. + /// For example: PageTableEntryFlags::TABLE_OR_4KIB_PAGE. + const MAP_EXTRA_FLAG: PageTableEntryFlags; +} + +/// A 4 KiB page mapped in the L3Table. +#[derive(Clone, Copy)] +pub enum BasePageSize {} +impl PageSize for BasePageSize { + const SIZE: u64 = 4096; + const MAP_LEVEL: usize = 0; + const MAP_EXTRA_FLAG: PageTableEntryFlags = PageTableEntryFlags::BLANK; +} + +/// A 2 MiB page mapped in the L2Table. +#[derive(Clone, Copy)] +pub enum LargePageSize {} +impl PageSize for LargePageSize { + const SIZE: u64 = 2 * 1024 * 1024; + const MAP_LEVEL: usize = 1; + const MAP_EXTRA_FLAG: PageTableEntryFlags = PageTableEntryFlags::BLANK; +} + +/// A 1 GiB page mapped in the L1Table. +#[derive(Clone, Copy)] +pub enum HugePageSize {} +impl PageSize for HugePageSize { + const SIZE: u64 = 1024 * 1024 * 1024; + const MAP_LEVEL: usize = 2; + const MAP_EXTRA_FLAG: PageTableEntryFlags = PageTableEntryFlags::BLANK; +} + +/// A memory page of the size given by S. +#[derive(Clone, Copy)] +struct Page { + /// Virtual memory address of this page. + /// This is rounded to a page size boundary on creation. + virtual_address: VirtAddr, + + /// Required by Rust to support the S parameter. + size: PhantomData, +} + +impl Page { + /// Return the stored virtual address. + fn address(&self) -> VirtAddr { + self.virtual_address + } + + /// Flushes this page from the TLB of this CPU. + fn flush_from_tlb(&self) { + unsafe { + sfence_vma(0, self.virtual_address.as_usize()); + } + } + + /// Returns whether the given virtual address is a valid one in SV39 + /// The address is valid when bit 38 and all more significant bits match + fn is_valid_address(virtual_address: VirtAddr) -> bool { + if virtual_address.as_u64() & (1 << 38) != 0 { + return virtual_address.as_u64() >> 39 == (1 << (64 - 39)) - 1; + } else { + return virtual_address.as_u64() >> 39 == 0; + } + } + + /// Returns a Page including the given virtual address. + /// That means, the address is rounded down to a page size boundary. + fn including_address(virtual_address: VirtAddr) -> Self { + assert!( + Self::is_valid_address(virtual_address), + "Virtual address {:#X} is invalid", + virtual_address + ); + + Self { + virtual_address: align_down!(virtual_address, S::SIZE), + size: PhantomData, + } + } + + /// Returns a PageIter to iterate from the given first Page to the given last Page (inclusive). + fn range(first: Self, last: Self) -> PageIter { + assert!(first.virtual_address <= last.virtual_address); + PageIter { + current: first, + last: last, + } + } + + /// Returns the index of this page in the table given by L. + fn table_index(self) -> usize { + assert!(L::LEVEL >= S::MAP_LEVEL); + self.virtual_address.as_usize() >> PAGE_BITS >> (L::LEVEL * PAGE_MAP_BITS) & PAGE_MAP_MASK + } +} + +/// An iterator to walk through a range of pages of size S. +struct PageIter { + current: Page, + last: Page, +} + +impl Iterator for PageIter { + type Item = Page; + + fn next(&mut self) -> Option> { + if self.current.virtual_address <= self.last.virtual_address { + let p = self.current; + self.current.virtual_address += S::SIZE; + Some(p) + } else { + None + } + } +} + +/// An interface to allow for a generic implementation of struct PageTable for all 4 page tables. +/// Must be implemented by all page tables. +trait PageTableLevel { + /// Numeric page table level + const LEVEL: usize; +} + +/// An interface for page tables with sub page tables (all except L3Table). +/// Having both PageTableLevel and PageTableLevelWithSubtables leverages Rust's typing system to provide +/// a subtable method only for those that have sub page tables. +/// +/// Kudos to Philipp Oppermann for the trick! +trait PageTableLevelWithSubtables: PageTableLevel { + type SubtableLevel; +} + +/// The Level 2 Table (can map 1 GiB pages) +enum L2Table {} +impl PageTableLevel for L2Table { + const LEVEL: usize = 2; +} + +impl PageTableLevelWithSubtables for L2Table { + type SubtableLevel = L1Table; +} + +/// The Level 1 Table (can map 2 MiB pages) +enum L1Table {} +impl PageTableLevel for L1Table { + const LEVEL: usize = 1; +} + +impl PageTableLevelWithSubtables for L1Table { + type SubtableLevel = L0Table; +} + +/// The Level 0 Table (can map 4 KiB pages) +enum L0Table {} +impl PageTableLevel for L0Table { + const LEVEL: usize = 0; +} + +/// Representation of any page table in memory. +/// Parameter L supplies information for Rust's typing system to distinguish between the different tables. +#[repr(align(4096))] +struct PageTable { + /// Each page table has 512 entries (can be calculated using PAGE_MAP_BITS). + entries: [PageTableEntry; 1 << PAGE_MAP_BITS], + + /// Required by Rust to support the L parameter. + level: PhantomData, +} + +/// A trait defining methods every page table has to implement. +/// This additional trait is necessary to make use of Rust's specialization feature and provide a default +/// implementation of some methods. +trait PageTableMethods { + fn get_page_table_entry(&self, page: Page) -> Option; + fn map_page_in_this_table( + &mut self, + page: Page, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ); + fn map_page( + &mut self, + page: Page, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ); +} + +impl PageTable { + const fn new() -> Self { + PageTable { + entries: [PageTableEntry { + physical_address_and_flags: PhysAddr::zero(), + }; 1 << PAGE_MAP_BITS], + level: PhantomData, + } + } +} + +impl PageTableMethods for PageTable { + /// Maps a single page in this table to the given physical address. + /// + /// Must only be called if a page of this size is mapped at this page table level! + fn map_page_in_this_table( + &mut self, + page: Page, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ) { + assert_eq!(L::LEVEL, S::MAP_LEVEL); + let index = page.table_index::(); + let flush = self.entries[index].is_present(); + + self.entries[index].set( + physical_address, + S::MAP_EXTRA_FLAG | PageTableEntryFlags::ACCESSED | PageTableEntryFlags::DIRTY | flags, + ); + + if flush { + page.flush_from_tlb(); + } + } + + /// Returns the PageTableEntry for the given page if it is present, otherwise returns None. + /// + /// This is the default implementation called only for L0Table. + /// It is overridden by a specialized implementation for all tables with sub tables (all except L0Table). + default fn get_page_table_entry(&self, page: Page) -> Option { + assert_eq!(L::LEVEL, S::MAP_LEVEL); + let index = page.table_index::(); + + if self.entries[index].is_present() { + Some(self.entries[index]) + } else { + None + } + } + + /// Maps a single page to the given physical address. + /// + /// This is the default implementation that just calls the map_page_in_this_table method. + /// It is overridden by a specialized implementation for all tables with sub tables (all except L3Table). + default fn map_page( + &mut self, + page: Page, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ) { + self.map_page_in_this_table::(page, physical_address, flags) + } +} + +impl PageTableMethods for PageTable +where + L::SubtableLevel: PageTableLevel, +{ + /// Returns the PageTableEntry for the given page if it is present, otherwise returns None. + /// + /// This is the implementation for all tables with subtables (L1, L2). + /// It overrides the default implementation above. + fn get_page_table_entry(&self, page: Page) -> Option { + assert!(L::LEVEL >= S::MAP_LEVEL); + let index = page.table_index::(); + + if self.entries[index].is_present() { + if L::LEVEL > S::MAP_LEVEL { + let subtable = self.subtable::(page); + subtable.get_page_table_entry::(page) + } else { + Some(self.entries[index]) + } + } else { + None + } + } + + /// Maps a single page to the given physical address. + /// + /// This is the implementation for all tables with subtables (L1, L2). + /// It overrides the default implementation above. + fn map_page( + &mut self, + page: Page, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ) { + assert!(L::LEVEL >= S::MAP_LEVEL); + + // trace!( + // "Mapping frame {:#X} to page {:#X}", + // physical_address, + // page.virtual_address, + // ); + + if L::LEVEL > S::MAP_LEVEL { + let index = page.table_index::(); + + // trace!("L::LEVEL > S::MAP_LEVEL"); + + // trace!("self.entries[index] {:?} , index {}",self.entries[index], index); + + // Does the table exist yet? + if !self.entries[index].is_present() { + // Allocate a single 4 KiB page for the new entry and mark it as a valid, writable subtable. + let new_entry = physicalmem::allocate(BasePageSize::SIZE as usize).unwrap(); + self.entries[index].set(new_entry, PageTableEntryFlags::BLANK); + + // trace!("new_entry {:#X}", new_entry); + + // Mark all entries as unused in the newly created table. + let subtable = self.subtable::(page); + for entry in subtable.entries.iter_mut() { + entry.physical_address_and_flags = PhysAddr::zero(); + } + } + + let subtable = self.subtable::(page); + subtable.map_page::(page, physical_address, flags) + } else { + // Calling the default implementation from a specialized one is not supported (yet), + // so we have to resort to an extra function. + self.map_page_in_this_table::(page, physical_address, flags) + } + } +} + +impl PageTable +where + L::SubtableLevel: PageTableLevel, +{ + /// Returns the next subtable for the given page in the page table hierarchy. + /// + /// Must only be called if a page of this size is mapped in a subtable! + fn subtable(&self, page: Page) -> &mut PageTable { + assert!(L::LEVEL > S::MAP_LEVEL); + + // Calculate the address of the subtable. + let index = page.table_index::(); + // trace!("Index: {:#X}", index); + let subtable_address = self.entries[index].address().as_usize(); + // trace!("subtable_address: {:#X}", subtable_address); + unsafe { &mut *(subtable_address as *mut PageTable) } + } + + /// Maps a continuous range of pages. + /// + /// # Arguments + /// + /// * `range` - The range of pages of size S + /// * `physical_address` - First physical address to map these pages to + /// * `flags` - Flags from PageTableEntryFlags to set for the page table entry (e.g. WRITABLE or NO_EXECUTE). + /// The VALID and GLOBAL are already set automatically. + fn map_pages( + &mut self, + range: PageIter, + physical_address: PhysAddr, + flags: PageTableEntryFlags, + ) { + let mut current_physical_address = physical_address; + + for page in range { + self.map_page::(page, current_physical_address, flags); + current_physical_address += S::SIZE as u64; + } + } +} + +#[inline] +fn get_page_range(virtual_address: VirtAddr, count: usize) -> PageIter { + let first_page = Page::::including_address(virtual_address); + let last_page = Page::::including_address(virtual_address + (count - 1) * S::SIZE as usize); + Page::range(first_page, last_page) +} + +pub fn get_page_table_entry(virtual_address: VirtAddr) -> Option { + trace!("Looking up Page Table Entry for {:#X}", virtual_address); + + let page = Page::::including_address(virtual_address); + //let root_pagetable = unsafe { &mut *L2TABLE_ADDRESS }; + /* let root_pagetable = unsafe { + &mut *mem::transmute::<*mut u64, *mut PageTable>(L2TABLE_ADDRESS.as_mut_ptr()) + }; */ + unsafe { ROOT_PAGETABLE.get_page_table_entry(page) } +} + +pub fn get_physical_address(virtual_address: VirtAddr) -> Option { + trace!("Getting physical address for {:#X}", virtual_address); + + let page = Page::::including_address(virtual_address); + //let root_pagetable = unsafe { &mut *L2TABLE_ADDRESS }; + /* let root_pagetable = unsafe { + &mut *mem::transmute::<*mut u64, *mut PageTable>(L2TABLE_ADDRESS.as_mut_ptr()) + }; */ + let address = unsafe { ROOT_PAGETABLE.get_page_table_entry(page)?.address() }; + let offset = virtual_address & (S::SIZE - 1); + Some(PhysAddr(address.as_u64() | offset.as_u64())) + //PhysAddr(virtual_address.as_u64()) +} + +/// Translate a virtual memory address to a physical one. +/// Just like get_physical_address, but automatically uses the correct page size for the respective memory address. +pub fn virtual_to_physical(virtual_address: VirtAddr) -> Option { + // panic!("Not impemented!"); + /* if virtual_address < mm::kernel_start_address() { + // Parts of the memory below the kernel image are identity-mapped. + // However, this range should never be used in a virtual_to_physical call. + panic!( + "Trying to get the physical address of {:#X}, which is too low", + virtual_address + ); + } else if virtual_address < mm::kernel_end_address() { + // The kernel image is mapped in 2 MiB pages. + get_physical_address::(virtual_address) + } else if virtual_address < virtualmem::task_heap_start() { + // The kernel memory is mapped in 4 KiB pages. + get_physical_address::(virtual_address) + } else if virtual_address < virtualmem::task_heap_end() { + // The application memory is mapped in 2 MiB pages. + get_physical_address::(virtual_address) + } else { + // This range is currently unused by HermitCore. + panic!( + "Trying to get the physical address of {:#X}, which is too high", + virtual_address + ); + } */ + let mut vpn: [u64; PAGE_LEVELS] = [0; PAGE_LEVELS]; + + for i in 0..PAGE_LEVELS { + vpn[i] = (virtual_address >> (PAGE_BITS + i * PAGE_MAP_BITS)) & PAGE_MAP_MASK as u64; + trace!( + "i: {}, vpn[i]: {:#X}, {:#X}", + i, + vpn[i], + virtual_address >> (PAGE_BITS + i * PAGE_MAP_BITS) + ); + } + + let mut page_table_addr = unsafe { &ROOT_PAGETABLE as *const PageTable }; + for i in (0..PAGE_LEVELS).rev() { + let pte = unsafe { (*page_table_addr).entries[(vpn[i]) as usize] }; + // trace!("PTE: {:?} , i: {}, vpn[i]: {:#X}", pte, i, vpn[i]); + //Translation would raise a page-fault exception + assert!( + pte.is_present() && !(!pte.is_readable() && pte.is_writable()), + "Invalid PTE: {:?}", + pte + ); + + if pte.is_executable() || pte.is_readable() { + //PTE is a leaf + // trace!("PTE is a leaf"); + let mut phys_address = virtual_address.as_u64() & ((1 << PAGE_BITS) - 1); + for j in 0..i { + phys_address = phys_address | (vpn[j]) << (PAGE_BITS + j * PAGE_MAP_BITS); + } + let ppn = pte.address().as_u64(); + for j in i..PAGE_LEVELS { + // trace!( + // "ppn: {:#X}, {:#X}", + // ppn, + // ppn & (PAGE_MAP_MASK << (PAGE_BITS + j * PAGE_MAP_BITS)) as u64 + // ); + phys_address = phys_address + | (ppn & (PAGE_MAP_MASK << (PAGE_BITS + j * PAGE_MAP_BITS)) as u64); + } + return Some(PhysAddr(phys_address)); + } else { + //PTE is a pointer to the next level of the page table + assert!(i != 0); //pte should be a leaf if i=0 + page_table_addr = pte.address().as_usize() as *mut PageTable; + // trace!("PTE is pointer: {:?}", page_table_addr); + } + } + panic!("virtual_to_physical should never reach this point"); +} + +#[no_mangle] +pub extern "C" fn virt_to_phys(virtual_address: VirtAddr) -> PhysAddr { + virtual_to_physical(virtual_address).unwrap() +} + +pub fn map( + virtual_address: VirtAddr, + physical_address: PhysAddr, + count: usize, + flags: PageTableEntryFlags, +) { + trace!( + "Mapping physical address {:#X} to virtual address {:#X} ({} pages)", + physical_address, + virtual_address, + count + ); + + let range = get_page_range::(virtual_address, count); + unsafe { + ROOT_PAGETABLE.map_pages(range, physical_address, flags); + } + + //assert_eq!(virtual_address.as_u64(), physical_address.as_u64(), "Paging not implemented"); +} + +pub fn map_heap(virt_addr: VirtAddr, count: usize) { + let flags = { + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable().execute_disable(); + flags + }; + + let virt_addrs = (0..count).map(|n| virt_addr + n * S::SIZE as usize); + + for virt_addr in virt_addrs { + let phys_addr = physicalmem::allocate_aligned(S::SIZE as usize, S::SIZE as usize).unwrap(); + map::(virt_addr, phys_addr, 1, flags); + } +} + +pub fn unmap(virtual_address: VirtAddr, count: usize) { + trace!( + "Unmapping virtual address {:#X} ({} pages)", + virtual_address, + count + ); + + let range = get_page_range::(virtual_address, count); + /* let root_pagetable = unsafe { + &mut *mem::transmute::<*mut u64, *mut PageTable>(L2TABLE_ADDRESS.as_mut_ptr()) + }; */ + unsafe { + ROOT_PAGETABLE.map_pages(range, PhysAddr::zero(), PageTableEntryFlags::BLANK); + } +} + +#[inline] +pub fn get_application_page_size() -> usize { + LargePageSize::SIZE as usize +} + +pub fn identity_map(start_address: PhysAddr, end_address: PhysAddr) { + let first_page = Page::::including_address(VirtAddr(start_address.as_u64())); + let last_page = Page::::including_address(VirtAddr(end_address.as_u64())); + + trace!( + "identity_map address {:#X} to address {:#X}", + first_page.virtual_address, + last_page.virtual_address, + ); + + /* assert!( + last_page.address() < mm::kernel_start_address(), + "Address {:#X} to be identity-mapped is not below Kernel start address", + last_page.address() + ); */ + + /* let root_pagetable = unsafe { + &mut *mem::transmute::<*mut u64, *mut PageTable>(L2TABLE_ADDRESS.as_mut_ptr()) + }; */ + let range = Page::::range(first_page, last_page); + let mut flags = PageTableEntryFlags::empty(); + flags.normal().writable(); + unsafe { + ROOT_PAGETABLE.map_pages(range, PhysAddr(first_page.address().as_u64()), flags); + } +} + +pub fn init_page_tables() { + trace!("Identity map the physical memory using HugePages"); + + unsafe { + identity_map::( + get_ram_address(), + get_ram_address() + PhysAddr(physicalmem::total_memory_size() as u64 - 1), + ); + satp::write(0x8 << 60 | ((&ROOT_PAGETABLE as *const _ as usize) >> 12)) + } +} + +pub fn init_application_processor() { + trace!("Identity map the physical memory using HugePages"); + unsafe { satp::write(0x8 << 60 | ((&ROOT_PAGETABLE as *const _ as usize) >> 12)) } +} diff --git a/src/arch/riscv/mm/physicalmem.rs b/src/arch/riscv/mm/physicalmem.rs new file mode 100644 index 0000000000..0730e45afd --- /dev/null +++ b/src/arch/riscv/mm/physicalmem.rs @@ -0,0 +1,112 @@ +use core::alloc::AllocError; +use core::convert::TryInto; +use core::sync::atomic::{AtomicUsize, Ordering}; + +use hermit_sync::InterruptSpinMutex; + +use crate::arch::riscv::kernel::{get_limit, get_ram_address}; +use crate::arch::riscv::mm::paging::{BasePageSize, PageSize}; +use crate::arch::riscv::mm::PhysAddr; +use crate::mm; +use crate::mm::freelist::{FreeList, FreeListEntry}; + +static PHYSICAL_FREE_LIST: InterruptSpinMutex = InterruptSpinMutex::new(FreeList::new()); +static TOTAL_MEMORY: AtomicUsize = AtomicUsize::new(0); + +fn detect_from_limits() -> Result<(), ()> { + let limit = get_limit(); + if unsafe { limit } == 0 { + return Err(()); + } + + let entry = FreeListEntry::new( + mm::kernel_end_address().as_usize(), + get_ram_address().as_usize() + limit, + ); + PHYSICAL_FREE_LIST.lock().list.push_back(entry); + TOTAL_MEMORY.store(limit, Ordering::SeqCst); + + Ok(()) +} + +pub fn init() { + detect_from_limits().unwrap(); +} + +pub fn total_memory_size() -> usize { + TOTAL_MEMORY.load(Ordering::SeqCst) +} + +pub fn allocate(size: usize) -> Result { + assert!(size > 0); + assert_eq!( + size % BasePageSize::SIZE as usize, + 0, + "Size {:#X} is not a multiple of {:#X}", + size, + BasePageSize::SIZE as usize + ); + + Ok(PhysAddr( + PHYSICAL_FREE_LIST + .lock() + .allocate(size, None)? + .try_into() + .unwrap(), + )) +} + +pub fn allocate_aligned(size: usize, alignment: usize) -> Result { + assert!(size > 0); + assert!(alignment > 0); + assert_eq!( + size % alignment, + 0, + "Size {:#X} is not a multiple of the given alignment {:#X}", + size, + alignment + ); + assert_eq!( + alignment % BasePageSize::SIZE as usize, + 0, + "Alignment {:#X} is not a multiple of {:#X}", + alignment, + BasePageSize::SIZE as usize + ); + + Ok(PhysAddr( + PHYSICAL_FREE_LIST + .lock() + .allocate(size, Some(alignment))? + .try_into() + .unwrap(), + )) +} + +/// This function must only be called from mm::deallocate! +/// Otherwise, it may fail due to an empty node pool (POOL.maintain() is called in virtualmem::deallocate) +pub fn deallocate(physical_address: PhysAddr, size: usize) { + assert!( + physical_address >= PhysAddr(mm::kernel_end_address().as_u64()), + "Physical address {:#X} is not >= KERNEL_END_ADDRESS", + physical_address + ); + assert!(size > 0); + assert_eq!( + size % BasePageSize::SIZE as usize, + 0, + "Size {:#X} is not a multiple of {:#X}", + size, + BasePageSize::SIZE as usize + ); + + PHYSICAL_FREE_LIST + .lock() + .deallocate(physical_address.as_usize(), size); +} + +pub fn print_information() { + PHYSICAL_FREE_LIST + .lock() + .print_information(" PHYSICAL MEMORY FREE LIST "); +} diff --git a/src/arch/riscv/mm/virtualmem.rs b/src/arch/riscv/mm/virtualmem.rs new file mode 100644 index 0000000000..8d738b7b08 --- /dev/null +++ b/src/arch/riscv/mm/virtualmem.rs @@ -0,0 +1,161 @@ +use core::alloc::AllocError; +use core::convert::TryInto; + +use hermit_sync::InterruptSpinMutex; + +use crate::arch::riscv::kernel::get_ram_address; +use crate::arch::riscv::mm::paging::{BasePageSize, HugePageSize, PageSize}; +use crate::arch::riscv::mm::{physicalmem, PhysAddr, VirtAddr}; +use crate::mm; +use crate::mm::freelist::{FreeList, FreeListEntry}; + +static KERNEL_FREE_LIST: InterruptSpinMutex = InterruptSpinMutex::new(FreeList::new()); + +/// End of the virtual memory address space reserved for kernel memory (256 GiB). +/// This also marks the start of the virtual memory address space reserved for the task heap. +const KERNEL_VIRTUAL_MEMORY_END: VirtAddr = VirtAddr(0x4000000000); + +/// End of the virtual memory address space reserved for task memory (512 GiB). +/// This is the maximum contiguous virtual memory area possible with sv39 +const TASK_VIRTUAL_MEMORY_END: VirtAddr = VirtAddr(0x8000000000); + +pub fn init() { + let entry = FreeListEntry { + start: align_up!( + (get_ram_address() + PhysAddr(physicalmem::total_memory_size() as u64)).as_usize(), + HugePageSize::SIZE as usize + ), + end: KERNEL_VIRTUAL_MEMORY_END.as_usize(), + }; + KERNEL_FREE_LIST.lock().list.push_back(entry); +} + +pub fn allocate(size: usize) -> Result { + assert!(size > 0); + assert_eq!( + size % BasePageSize::SIZE as usize, + 0, + "Size {:#X} is not a multiple of {:#X}", + size, + BasePageSize::SIZE as usize + ); + + Ok(VirtAddr( + KERNEL_FREE_LIST + .lock() + .allocate(size, None)? + .try_into() + .unwrap(), + )) +} + +pub fn allocate_aligned(size: usize, alignment: usize) -> Result { + assert!(size > 0); + assert!(alignment > 0); + assert_eq!( + size % alignment, + 0, + "Size {:#X} is not a multiple of the given alignment {:#X}", + size, + alignment + ); + assert_eq!( + alignment % BasePageSize::SIZE as usize, + 0, + "Alignment {:#X} is not a multiple of {:#X}", + alignment, + BasePageSize::SIZE as usize + ); + + Ok(VirtAddr( + KERNEL_FREE_LIST + .lock() + .allocate(size, Some(alignment))? + .try_into() + .unwrap(), + )) +} + +pub fn deallocate(virtual_address: VirtAddr, size: usize) { + assert!( + virtual_address >= mm::kernel_end_address(), + "Virtual address {:#X} is not >= KERNEL_END_ADDRESS", + virtual_address + ); + assert!( + virtual_address < KERNEL_VIRTUAL_MEMORY_END, + "Virtual address {:#X} is not < KERNEL_VIRTUAL_MEMORY_END", + virtual_address + ); + assert_eq!( + virtual_address % BasePageSize::SIZE as usize, + 0, + "Virtual address {:#X} is not a multiple of {:#X}", + virtual_address, + BasePageSize::SIZE as usize + ); + assert!(size > 0); + assert_eq!( + size % BasePageSize::SIZE as usize, + 0, + "Size {:#X} is not a multiple of {:#X}", + size, + BasePageSize::SIZE as usize + ); + + KERNEL_FREE_LIST + .lock() + .deallocate(virtual_address.as_usize(), size); +} + +/*pub fn reserve(virtual_address: VirtAddr, size: usize) { + assert!( + virtual_address >= mm::kernel_end_address(), + "Virtual address {:#X} is not >= KERNEL_END_ADDRESS", + virtual_address + ); + assert!( + virtual_address < KERNEL_VIRTUAL_MEMORY_END, + "Virtual address {:#X} is not < KERNEL_VIRTUAL_MEMORY_END", + virtual_address + ); + assert_eq!( + virtual_address % BasePageSize::SIZE as usize, + 0, + "Virtual address {:#X} is not a multiple of {:#X}", + virtual_address, + BasePageSize::SIZE as usize + ); + assert!(size > 0); + assert_eq!( + size % BasePageSize::SIZE as usize, + 0, + "Size {:#X} is not a multiple of {:#X}", + size, + BasePageSize::SIZE as usize + ); + + let result = KERNEL_FREE_LIST.lock().reserve(virtual_address.as_usize(), size); + assert!( + result.is_ok(), + "Could not reserve {:#X} bytes of virtual memory at {:#X}", + size, + virtual_address + ); +}*/ + +pub fn print_information() { + KERNEL_FREE_LIST + .lock() + .print_information(" KERNEL VIRTUAL MEMORY FREE LIST "); +} + +#[inline] +pub fn task_heap_start() -> VirtAddr { + KERNEL_VIRTUAL_MEMORY_END +} + +#[inline] +pub fn task_heap_end() -> VirtAddr { + TASK_VIRTUAL_MEMORY_END +} diff --git a/src/arch/riscv/mod.rs b/src/arch/riscv/mod.rs new file mode 100644 index 0000000000..a459fd5479 --- /dev/null +++ b/src/arch/riscv/mod.rs @@ -0,0 +1,2 @@ +pub mod kernel; +pub mod mm; diff --git a/src/drivers/mod.rs b/src/drivers/mod.rs index 099cb8d742..66a20398d1 100644 --- a/src/drivers/mod.rs +++ b/src/drivers/mod.rs @@ -13,6 +13,8 @@ pub mod virtio; pub mod error { use core::fmt; + #[cfg(target_arch = "riscv64")] + use crate::drivers::net::gem::GEMError; #[cfg(feature = "pci")] use crate::drivers::net::rtl8139::RTL8139Error; use crate::drivers::virtio::error::VirtioError; @@ -22,6 +24,8 @@ pub mod error { InitVirtioDevFail(VirtioError), #[cfg(feature = "pci")] InitRTL8139DevFail(RTL8139Error), + #[cfg(target_arch = "riscv64")] + InitGEMDevFail(GEMError), } impl From for DriverError { @@ -37,6 +41,13 @@ pub mod error { } } + #[cfg(target_arch = "riscv64")] + impl From for DriverError { + fn from(err: GEMError) -> Self { + DriverError::InitGEMDevFail(err) + } + } + impl fmt::Display for DriverError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { @@ -47,6 +58,10 @@ pub mod error { DriverError::InitRTL8139DevFail(ref err) => { write!(f, "RTL8139 driver failed: {:?}", err) } + #[cfg(target_arch = "riscv64")] + DriverError::InitGEMDevFail(ref err) => { + write!(f, "GEM driver failed: {:?}", err) + } } } } diff --git a/src/drivers/net/gem.rs b/src/drivers/net/gem.rs new file mode 100644 index 0000000000..09774a19d2 --- /dev/null +++ b/src/drivers/net/gem.rs @@ -0,0 +1,739 @@ +#![allow(unused)] + +use core::convert::TryInto; +use core::mem; + +use riscv::register::*; +use tock_registers::interfaces::*; +use tock_registers::registers::*; +use tock_registers::{register_bitfields, register_structs}; + +use crate::arch::kernel::irq::*; +use crate::arch::kernel::pci; +use crate::arch::kernel::percore::core_scheduler; +use crate::arch::mm::paging::virt_to_phys; +use crate::arch::mm::VirtAddr; +use crate::drivers::error::DriverError; +use crate::drivers::net::{network_irqhandler, NetworkInterface}; + +//Base address of the control registers +//const GEM: *mut Registers = 0x1009_0000 as *mut Registers; //For Sifive FU540 +//const GEM_IRQ: u32 = 53; //For Sifive FU540 + +// https://github.com/torvalds/linux/blob/v4.15/drivers/net/ethernet/cadence/macb.h +register_structs! { + /// Register offsets + Registers { + // Control register: read-write + (0x000 => network_control: ReadWrite), + (0x004 => network_config: ReadWrite), + (0x008 => network_status: ReadOnly), + (0x00C => _reserved1), + (0x010 => dma_config: ReadWrite), + (0x014 => transmit_status: ReadWrite), + (0x018 => rx_qbar: ReadWrite), + (0x01c => tx_qbar: ReadWrite), + (0x020 => receive_status: ReadWrite), + (0x024 => int_status: ReadWrite), + (0x028 => int_enable: WriteOnly), + (0x02C => int_disable: WriteOnly), + (0x030 => _reserved3), + (0x034 => phy_maintenance: ReadWrite), + (0x038 => _reserved4), + (0x088 => spec_add1_bottom: ReadWrite), + (0x08C => spec_add1_top: ReadWrite), + (0x090 => _reserved5), + (0x1000 => @END), + } +} + +register_bitfields! [ + // First parameter is the register width. Can be u8, u16, u32, or u64. + u32, + + NetworkControl [ + STARTTX OFFSET(9) NUMBITS(1) [], + STATCLR OFFSET(5) NUMBITS(1) [], + MDEN OFFSET(4) NUMBITS(1) [], + TXEN OFFSET(3) NUMBITS(1) [], + RXEN OFFSET(2) NUMBITS(1) [], + ], + NetworkConfig [ + RXCHKSUMEN OFFSET(24) NUMBITS(1) [], + DBUS_WIDTH OFFSET(21) NUMBITS(2) [ + DBW32 = 0, + DBW64 = 1, + DBW128 = 2 + ], + MDCCLKDIV OFFSET(18) NUMBITS(3) [ + CLK_DIV8 = 0, + CLK_DIV16 = 1, + CLK_DIV32 = 2, + CLK_DIV48 = 3, + CLK_DIV64 = 4, + CLK_DIV96 = 5, + CLK_DIV128 = 6, + CLK_DIV224 = 7 + ], + FCSREM OFFSET(17) NUMBITS(1) [], + PAUSEEN OFFSET(13) NUMBITS(1) [], + GIGEEN OFFSET(10) NUMBITS(1) [], + MCASTHASHEN OFFSET(6) NUMBITS(1) [], + BCASTDI OFFSET(5) NUMBITS(1) [], + COPYALLEN OFFSET(4) NUMBITS(1) [], + FDEN OFFSET(1) NUMBITS(1) [], + ], + NetworkStatus [ + PHY_MGMT_IDLE OFFSET(2) NUMBITS(1) [], + ], + DMAConfig [ + RXBUF OFFSET(16) NUMBITS(8) [], + TCPCKSUM OFFSET(11) NUMBITS(1) [], + TXSIZE OFFSET(10) NUMBITS(1) [], + RXSIZE OFFSET(8) NUMBITS(2) [ + //Supported on all devices? + FULL_ADDRESSABLE_SPACE = 3 + ], + ENDIAN OFFSET(7) NUMBITS(1) [], + BLENGTH OFFSET(0) NUMBITS(5) [ + SINGLE = 0b00001, + INCR4 = 0b00100, + INCR8 = 0b01000, + INCR16 = 0b10000 + ], + ], + RecieveStatus [ + FRAMERX OFFSET(1) NUMBITS(1) [], + ], + TransmitStatus [ + TXCOMPL OFFSET(5) NUMBITS(1) [], + TXGO OFFSET(3) NUMBITS(1) [], + ], + Interrupts [ + TSU_SEC_INCR OFFSET(26) NUMBITS(1) [], + TXCOMPL OFFSET(7) NUMBITS(1) [], + FRAMERX OFFSET(1) NUMBITS(1) [], + + ], + PHYMaintenance [ + CLAUSE_22 OFFSET(30) NUMBITS(1) [], + OP OFFSET(28) NUMBITS(2) [ + READ = 0b10, + WRITE = 0b01, + ], + ADDR OFFSET(23) NUMBITS(5) [], + REG OFFSET(18) NUMBITS(5) [], + MUST_10 OFFSET(16) NUMBITS(2) [ + MUST_BE_10 = 0b10 + ], + DATA OFFSET(0) NUMBITS(16) [], + ], +]; + +/// PHY reg index +enum PhyReg { + Control = 0, + Status = 1, + ID1 = 2, + ID2 = 3, + ANAdvertisement = 4, + ANLinkPartnerAbility = 5, + ANExpansion = 6, + ANNextPageTransmit = 7, + ANLinkPartnerReceivedNextPage = 8, + ExtendedStatus = 15, +} + +/// PHY Status reg mask and offset +enum PhyStatus { + ANCompleteOffset = 5, + ANCompleteMask = 0x20, + ANCapOffset = 3, + ANCapMask = 0x4, +} + +enum PhyControl { + ANEnableOffset = 12, + ANEnableMask = 0x1000, +} + +enum PhyPartnerAbility { + ANEnableOffset = 12, + ANEnableMask = 0x1000, +} + +/// size of a receive buffer (must be multiple of 64) +const RX_BUF_LEN: u32 = 1600; +const RX_BUFFER_MULTIPLE: u32 = 64; +/// Number of receive buffers +const RX_BUF_NUM: u32 = 64; + +/// size of a transmit buffer +const TX_BUF_LEN: u32 = 1600; +/// Number of transmit buffers +const TX_BUF_NUM: u32 = 1; + +/// Marks tx buffer as last buffer of frame +const TX_DESC_LAST: u32 = 1 << 15; + +/// Marks tx buffer wrap buffer +const TX_DESC_WRAP: u32 = 1 << 30; + +/// Marks tx buffer as used +const TX_DESC_USED: u32 = 1 << 31; + +#[derive(Debug)] +pub enum GEMError { + InitFailed, + ResetFailed, + NoPhyFound, + Unknown, +} + +/// GEM network driver struct. +/// +/// Struct allows to control device queus as also +/// the device itself. +pub struct GEMDriver { + // Pointer to the registers of the controller + gem: *mut Registers, + mtu: u16, + irq: u32, + mac: [u8; 6], + rx_counter: u32, + rxbuffer: VirtAddr, + rxbuffer_list: VirtAddr, + tx_counter: u32, + txbuffer: VirtAddr, + txbuffer_list: VirtAddr, +} + +impl NetworkInterface for GEMDriver { + /// Returns the MAC address of the network interface + fn get_mac_address(&self) -> [u8; 6] { + self.mac + } + + /// Returns the current MTU of the device. + fn get_mtu(&self) -> u16 { + self.mtu + } + + fn get_tx_buffer(&mut self, len: usize) -> Result<(*mut u8, usize), ()> { + debug!("get_tx_buffer"); + + if len as u32 > TX_BUF_LEN { + error!("TX buffer is too small"); + return Err(()); + } + + self.handle_interrupt(); + + for i in 0..TX_BUF_NUM { + let index = (i + self.tx_counter) % TX_BUF_NUM; + let word1_addr = (self.txbuffer_list + (index * 8 + 4) as u64).as_mut_ptr::(); + let word1 = unsafe { core::ptr::read_volatile(word1_addr) }; + // Reuse a used buffer + if word1 & TX_DESC_USED != 0 { + // Clear used bit + unsafe { + core::ptr::write_volatile(word1_addr, word1 & (!TX_DESC_USED)); + } + + // Set new starting point to search for next buffer + self.tx_counter = (index + 1) % TX_BUF_NUM; + + // Address of the tx buffer + let buffer = self.txbuffer + (index * TX_BUF_LEN) as u64; + return Ok((buffer.as_mut_ptr::(), index as usize)); + } + } + + error!("Unable to get TX buffer"); + Err(()) + } + + fn free_tx_buffer(&self, token: usize) { + //Set used bit to indicate that the buffer can be reused + let word1_addr = (self.txbuffer_list + (token * 8 + 4) as u64).as_mut_ptr::(); + let word1 = unsafe { core::ptr::read_volatile(word1_addr) }; + unsafe { + core::ptr::write_volatile(word1_addr, word1 | TX_DESC_USED); + } + } + + fn send_tx_buffer(&mut self, id: usize, len: usize) -> Result<(), ()> { + debug!("send_tx_buffer"); + + // Address of word[1] of the buffer descriptor + let word1_addr = (self.txbuffer_list + (id * 8 + 4) as u64).as_mut_ptr::(); + let word1 = unsafe { core::ptr::read_volatile(word1_addr) }; + + unsafe { + // Set length of frame and mark as single buffer Ethernet frame + core::ptr::write_volatile( + word1_addr, + (word1 & TX_DESC_WRAP) | TX_DESC_LAST | len as u32, + ); + + // Enable TX + (*self.gem) + .network_control + .modify(NetworkControl::TXEN::SET); + // Start transmission + (*self.gem) + .network_control + .modify(NetworkControl::STARTTX::SET); + + // (*GEM).network_control.modify(NetworkControl::RXEN::CLEAR); + } + + Ok(()) + } + + fn has_packet(&self) -> bool { + debug!("has_packet"); + + match self.next_rx_index() { + Some(_) => true, + None => false, + } + } + + fn receive_rx_buffer(&mut self) -> Result<(&'static mut [u8], usize), ()> { + debug!("receive_rx_buffer"); + + // Scan the buffer descriptor queue starting from rx_count + match self.next_rx_index() { + Some(index) => { + let word1_addr = (self.rxbuffer_list + (index * 8 + 4) as u64); + let word1_entry = + unsafe { core::ptr::read_volatile(word1_addr.as_mut_ptr::()) }; + let length = word1_entry & 0x1FFF; + debug!("Recieved frame in buffer {}, length: {}", index, length); + + // Starting point to search for next frame + self.rx_counter = (index + 1) % RX_BUF_NUM; + let buffer = unsafe { + core::slice::from_raw_parts( + (self.rxbuffer.as_usize() + (index * RX_BUF_LEN) as usize) as *const u8, + length as usize, + ) + }; + trace!("BUFFER: {:x?}", buffer); + // SAFETY: This is a blatant lie and very unsound. + // The API must be fixed or the buffer may never touched again. + // TODO: + #[allow(mutable_transmutes)] + let buffer = unsafe { mem::transmute(buffer) }; + Ok((buffer, index as usize)) + } + None => Err(()), + } + } + + // Tells driver, that buffer is consumed and can be deallocated + fn rx_buffer_consumed(&mut self, handle: usize) { + debug!("rx_buffer_consumed: handle: {}", handle); + + let word0_addr = (self.rxbuffer_list + (handle * 8) as u64); + let word1_addr = word0_addr + 4 as u64; + + unsafe { + // Clear word1 (is this really necessary?) + core::ptr::write_volatile(word1_addr.as_mut_ptr::(), 0); + // Give back ownership to GEM + let word0_entry = core::ptr::read_volatile(word0_addr.as_mut_ptr::()); + core::ptr::write_volatile(word0_addr.as_mut_ptr::(), word0_entry & 0xFFFF_FFFE); + } + } + + fn set_polling_mode(&mut self, value: bool) { + debug!("set_polling_mode"); + if value { + // disable interrupts from the NIC + unsafe { + (*self.gem).int_disable.set(0x7FF_FEFF); + } + } else { + // Enable all known interrupts by setting the interrupt mask. + unsafe { + (*self.gem).int_enable.write(Interrupts::FRAMERX::SET); + } + } + } + + fn handle_interrupt(&mut self) -> bool { + let int_status = unsafe { (*self.gem).int_status.extract() }; + + let receive_status = unsafe { (*self.gem).receive_status.extract() }; + + let transmit_status = unsafe { (*self.gem).transmit_status.extract() }; + + debug!( + "handle_interrupt\nint_status: {:?}\nreceive_status: {:?}\ntransmit_status: {:?}", + int_status, receive_status, transmit_status + ); + + if transmit_status.is_set(TransmitStatus::TXCOMPL) { + debug!("TX COMPLETE"); + unsafe { + (*self.gem) + .int_status + .modify_no_read(int_status, Interrupts::TXCOMPL::SET); + (*self.gem) + .transmit_status + .modify_no_read(transmit_status, TransmitStatus::TXCOMPL::SET); + (*self.gem) + .network_control + .modify(NetworkControl::TXEN::CLEAR); + } + } + + let ret = + int_status.is_set(Interrupts::FRAMERX) && receive_status.is_set(RecieveStatus::FRAMERX); + + if ret { + debug!("RX COMPLETE"); + unsafe { + (*self.gem) + .int_status + .modify_no_read(int_status, Interrupts::FRAMERX::SET); + (*self.gem) + .receive_status + .modify_no_read(receive_status, RecieveStatus::FRAMERX::SET); + } + + // handle incoming packets + todo!(); + } + // increment_irq_counter((32 + self.irq).into()); + ret + } +} + +impl GEMDriver { + /// Returns the index of the next recieved frame + fn next_rx_index(&self) -> Option { + // Scan the buffer descriptor queue starting from rx_count + + for i in 0..RX_BUF_NUM { + let index = (i + self.rx_counter) % RX_BUF_NUM; + let word0_addr = (self.rxbuffer_list + (index * 8) as u64); + let word0_entry = unsafe { core::ptr::read_volatile(word0_addr.as_mut_ptr::()) }; + // Is buffer owned by GEM? + if (word0_entry & 0x1) != 0 { + return Some(index); + } + } + + None + } +} + +impl Drop for GEMDriver { + fn drop(&mut self) { + debug!("Dropping GEMDriver!"); + + // Software reset + // Clear the Network Control register + unsafe { + (*self.gem).network_control.set(0x0); + } + + crate::mm::deallocate(self.rxbuffer, (RX_BUF_LEN * RX_BUF_NUM) as usize); + crate::mm::deallocate(self.txbuffer, (TX_BUF_LEN * TX_BUF_NUM) as usize); + crate::mm::deallocate(self.rxbuffer_list, (8 * RX_BUF_NUM) as usize); + crate::mm::deallocate(self.txbuffer_list, (8 * TX_BUF_NUM) as usize); + } +} + +/// Inits the driver. Passing u32::MAX as phy_addr will trigger a search for the actual PHY address +pub fn init_device( + gem_base: VirtAddr, + irq: u32, + phy_addr: u32, + mac: [u8; 6], +) -> Result { + debug!("Init GEM at {:p}", gem_base); + + let gem = gem_base.as_mut_ptr::(); + + unsafe { + // Initialize the Controller + + // Clear the Network Control register + (*gem).network_control.set(0x0); + // Clear the Statistics registers + (*gem).network_control.modify(NetworkControl::STATCLR::SET); + // Clear the status registers + (*gem).receive_status.set(0x0F); + (*gem).transmit_status.set(0x0F); + // Disable all interrupts + (*gem).int_disable.set(0x7FF_FEFF); + // Clear the buffer queues + (*gem).rx_qbar.set(0x0); + (*gem).tx_qbar.set(0x0); + + // Configure the Controller + + // Enable Full Duplex + (*gem).network_config.modify(NetworkConfig::FDEN::SET); + // Enable Gigabit mode + (*gem).network_config.modify(NetworkConfig::GIGEEN::SET); + // Enable reception of broadcast or multicast frames + (*gem) + .network_config + .modify(NetworkConfig::BCASTDI::CLEAR + NetworkConfig::MCASTHASHEN::SET); + // Enable promiscuous mode + // (*GEM).network_config.modify(NetworkConfig::COPYALLEN::SET); + // Enable TCP/IP checksum offload feature on receive + (*gem).network_config.modify(NetworkConfig::RXCHKSUMEN::SET); + // Enable Pause frames + (*gem).network_config.modify(NetworkConfig::PAUSEEN::SET); + // Set the MDC clock divisor + //(CLK_DIV64 for up to 160 Mhz) TODO: Determine the correct value + (*gem) + .network_config + .modify(NetworkConfig::MDCCLKDIV::CLK_DIV64); + // Enable FCS remove + (*gem).network_config.modify(NetworkConfig::FCSREM::SET); + + // Set the MAC address + let bottom: u32 = ((mac[3] as u32) << 24) + + ((mac[2] as u32) << 16) + + ((mac[1] as u32) << 8) + + ((mac[0] as u32) << 0); + let top: u32 = ((mac[5] as u32) << 8) + ((mac[4] as u32) << 0); + (*gem).spec_add1_bottom.set(bottom); + (*gem).spec_add1_top.set(top); + + // Program the DMA configuration register + + // Set the receive buffer size (TODO: Jumbo packet support) + (*gem) + .dma_config + .modify(DMAConfig::RXBUF.val(RX_BUF_LEN / RX_BUFFER_MULTIPLE)); + // Set the receiver packet buffer memory size to the full configured addressable space + (*gem) + .dma_config + .modify(DMAConfig::RXSIZE::FULL_ADDRESSABLE_SPACE); + // Set the transmitter packet buffer memory size to the full configured addressable space + (*gem).dma_config.modify(DMAConfig::TXSIZE::SET); + // Enable TCP/IP checksum generation offload on the transmitter + (*gem).dma_config.modify(DMAConfig::TCPCKSUM::SET); + // Configure for Little Endian system + (*gem).dma_config.modify(DMAConfig::ENDIAN::CLEAR); + // Configure fixed burst length to INCR16 + (*gem).dma_config.modify(DMAConfig::BLENGTH::INCR16); + + // Program the Network Control Register + + // Enable MDIO and enable transmitter/receiver + // (*gem).network_control.modify( + // NetworkControl::MDEN::SET + NetworkControl::TXEN::SET + NetworkControl::RXEN::SET, + // ); + (*gem).network_control.modify(NetworkControl::MDEN::SET); + + // PHY Initialization + let mut phy_addr = phy_addr; + if phy_addr == u32::MAX { + // Detect PHY + warn! {"No PHY address provided. Trying to find PHY ..."} + for i in 0..32 { + match phy_read(gem, i, PhyReg::Control) { + 0xFFFF => (), //Invalid + 0x0 => (), //Invalid + _ => { + phy_addr = i; + warn!("PHY found with address {}", phy_addr); + break; + } + } + if i == 31 { + error!("No PHY found"); + return Err(DriverError::InitGEMDevFail(GEMError::NoPhyFound)); + } + } + } + + // Clause 28 auto-negotiation https://opencores.org/websvn/filedetails?repname=1000base-x&path=%2F1000base-x%2Ftrunk%2Fdoc%2F802.3-2008_section2.pdf&bcsi_scan_91c2e97ef32f18a3=V1Ygi7liGXdis80J3CYk1MUlxZsSAAAACY4+BA%3D%3D+ + // This is PHY specific and may not work on all PHYs + let phy_status = phy_read(gem, phy_addr, PhyReg::Status); + + // Chck for auto-negotiation ability + if (phy_status & PhyStatus::ANCapMask as u16) == 0 { + warn!("PHY does not support auto-negotiation"); + // TODO + //return Err(DriverError::InitGEMDevFail(GEMError::NoPhyFound)); + } else { + // Keep default values in Auto-Negotiation advertisement register + // Enable AN + let phy_control = phy_read(gem, phy_addr, PhyReg::Control); + phy_write( + gem, + phy_addr, + PhyReg::Control, + PhyControl::ANEnableMask as u16 | phy_control, + ); + + // Wait for AN to complete + while (phy_read(gem, phy_addr, PhyReg::Status) | PhyStatus::ANCompleteMask as u16) == 0 + { + } + + // Read partner ability register + let partner_ability = phy_read(gem, phy_addr, PhyReg::ANLinkPartnerAbility); + + // Get the supported Speed and Duplex + // TODO - Next Page does not seem to be emulated by QEMU + + //info!("PHY auto-negotiation completed:\n Speed: {}\nDuplex", ,); + debug!( + "PHY auto-negotiation completed: Partner Ability {:x}", + partner_ability + ); + } + } + + // Configure the Buffer Descriptors + + // Allocate Receive Buffer + let rxbuffer = crate::mm::allocate((RX_BUF_LEN * RX_BUF_NUM) as usize, true); + // Allocate Receive Buffer Descriptor List + let rxbuffer_list = crate::mm::allocate((8 * RX_BUF_NUM) as usize, true); + // Allocate Transmit Buffer + let txbuffer = crate::mm::allocate((TX_BUF_LEN * TX_BUF_NUM) as usize, true); + // Allocate Transmit Buffer Descriptor List + let txbuffer_list = crate::mm::allocate((8 * TX_BUF_NUM) as usize, true); + + if txbuffer.is_zero() + || rxbuffer.is_zero() + || rxbuffer_list.is_zero() + || txbuffer_list.is_zero() + { + error!("Unable to allocate buffers for GEM"); + return Err(DriverError::InitGEMDevFail(GEMError::Unknown)); + } + + debug!( + "Allocate TxBuffer at 0x{:x} and RxBuffer at 0x{:x}", + txbuffer, rxbuffer + ); + + unsafe { + // Init Receive Buffer Descriptor List + for i in 0..RX_BUF_NUM { + let word0 = (rxbuffer_list + (i * 8) as u64).as_mut_ptr::(); + let word1 = (rxbuffer_list + (i * 8 + 4) as u64).as_mut_ptr::(); + let buffer = virt_to_phys(rxbuffer + (i * RX_BUF_LEN) as u64); + if (buffer.as_u64() & 0b11) != 0 { + error!("Wrong buffer alignment"); + return Err(DriverError::InitGEMDevFail(GEMError::Unknown)); + } + // This can fail if address of buffers is > 32 bit + // TODO: 64-bit addresses + let mut word0_entry: u32 = buffer.as_u64().try_into().unwrap(); + + // Mark the last descriptor in the buffer descriptor list with the wrap bit + if i == RX_BUF_NUM - 1 { + word0_entry |= 0b10; + } + core::ptr::write_volatile(word0, word0_entry); + core::ptr::write_volatile(word1, 0x0); + } + + let rx_qbar: u32 = virt_to_phys(rxbuffer_list).as_u64().try_into().unwrap(); + debug!("Set rx_qbar to {:x}", rx_qbar); + (*gem).rx_qbar.set(rx_qbar); + + // Init Transmit Buffer Descriptor List + for i in 0..TX_BUF_NUM { + let word0 = (txbuffer_list + (i * 8) as u64).as_mut_ptr::(); + let word1 = (txbuffer_list + (i * 8 + 4) as u64).as_mut_ptr::(); + let buffer = virt_to_phys(txbuffer + (i * TX_BUF_LEN) as u64); + + // This can fail if address of buffers is > 32 bit + // TODO: 64-bit addresses + let mut word0_entry: u32 = buffer.as_u64().try_into().unwrap(); + let mut word1_entry: u32 = TX_DESC_USED; + // Mark the last descriptor in the buffer descriptor list with the wrap bit + if i == TX_BUF_NUM - 1 { + word1_entry |= TX_DESC_WRAP; + } + core::ptr::write_volatile(word0, word0_entry); + core::ptr::write_volatile(word1, word1_entry); + } + + let tx_qbar: u32 = virt_to_phys(txbuffer_list).as_u64().try_into().unwrap(); + debug!("Set tx_qbar to {:x}", tx_qbar); + (*gem).tx_qbar.set(tx_qbar); + + // Configure Interrupts + debug!( + "Install interrupt handler for GEM at {:x}", + network_irqhandler as usize + ); + irq_install_handler(irq, network_irqhandler as usize); + (*gem).int_enable.write(Interrupts::FRAMERX::SET); // + Interrupts::TXCOMPL::SET + + // Enable the Controller (again?) + // Enable the transmitter + (*gem).network_control.modify(NetworkControl::TXEN::SET); + // Enable the receiver + (*gem).network_control.modify(NetworkControl::RXEN::SET); + } + + debug!( + "MAC address {:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] + ); + + Ok(GEMDriver { + gem: gem, + mtu: 1500, + irq: irq, + mac: mac, + rx_counter: 0, + rxbuffer: rxbuffer, + rxbuffer_list: rxbuffer_list, + tx_counter: 0, + txbuffer: txbuffer, + txbuffer_list: txbuffer_list, + }) +} + +unsafe fn phy_read(gem: *mut Registers, addr: u32, reg: PhyReg) -> u16 { + // Check that no MDIO operation is in progress + wait_for_mdio(gem); + // Initiate the data shift operation over MDIO + (*gem).phy_maintenance.write( + PHYMaintenance::CLAUSE_22::SET + + PHYMaintenance::OP::READ + + PHYMaintenance::ADDR.val(addr) + + PHYMaintenance::REG.val(reg as u32) + + PHYMaintenance::MUST_10::MUST_BE_10, + ); + wait_for_mdio(gem); + (*gem).phy_maintenance.read(PHYMaintenance::DATA) as u16 +} + +unsafe fn phy_write(gem: *mut Registers, addr: u32, reg: PhyReg, data: u16) { + // Check that no MDIO operation is in progress + wait_for_mdio(gem); + // Initiate the data shift operation over MDIO + (*gem).phy_maintenance.write( + PHYMaintenance::CLAUSE_22::SET + + PHYMaintenance::OP::WRITE + + PHYMaintenance::ADDR.val(addr) + + PHYMaintenance::REG.val(reg as u32) + + PHYMaintenance::MUST_10::MUST_BE_10 + + PHYMaintenance::DATA.val(data as u32), + ); + wait_for_mdio(gem); +} + +unsafe fn wait_for_mdio(gem: *mut Registers) { + // Check that no MDIO operation is in progress + while !(*gem).network_status.is_set(NetworkStatus::PHY_MGMT_IDLE) {} +} diff --git a/src/drivers/net/mod.rs b/src/drivers/net/mod.rs index 2cb3b06b62..94aa545e04 100644 --- a/src/drivers/net/mod.rs +++ b/src/drivers/net/mod.rs @@ -6,7 +6,14 @@ pub mod virtio_net; #[cfg(feature = "pci")] pub mod virtio_pci; +#[cfg(target_arch = "riscv64")] +pub mod gem; + +#[cfg(target_arch = "x86_64")] use crate::arch::kernel::apic; +#[cfg(target_arch = "riscv64")] +use crate::arch::kernel::irq::external_eoi; +#[cfg(target_arch = "x86_64")] use crate::arch::kernel::irq::ExceptionStackFrame; #[cfg(not(feature = "pci"))] use crate::arch::kernel::mmio; @@ -70,3 +77,32 @@ pub extern "x86-interrupt" fn network_irqhandler(_stack_frame: ExceptionStackFra core_scheduler().scheduler(); } } + +#[cfg(target_arch = "riscv64")] +pub fn network_irqhandler() { + debug!("Receive network interrupt"); + + // PLIC end of interrupt + external_eoi(); + + #[cfg(feature = "pci")] + let check_scheduler = match pci::get_network_driver() { + Some(driver) => driver.lock().handle_interrupt(), + _ => { + debug!("Unable to handle interrupt!"); + false + } + }; + #[cfg(not(feature = "pci"))] + let check_scheduler = match mmio::get_network_driver() { + Some(driver) => driver.lock().handle_interrupt(), + _ => { + debug!("Unable to handle interrupt!"); + false + } + }; + + if check_scheduler { + core_scheduler().scheduler(); + } +} diff --git a/src/drivers/net/virtio_mmio.rs b/src/drivers/net/virtio_mmio.rs index 320c586848..b28fcfa1f8 100644 --- a/src/drivers/net/virtio_mmio.rs +++ b/src/drivers/net/virtio_mmio.rs @@ -5,6 +5,7 @@ use alloc::collections::VecDeque; use alloc::rc::Rc; use alloc::vec::Vec; +#[cfg(target_arch = "x86_64")] use core::arch::x86_64::_mm_mfence; use core::cell::RefCell; use core::convert::TryInto; @@ -38,8 +39,10 @@ impl NetDevCfgRaw { unsafe { loop { let before = read_volatile(&self.config_generation); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let mtu = read_volatile(&self.mtu); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let after = read_volatile(&self.config_generation); @@ -57,9 +60,11 @@ impl NetDevCfgRaw { unsafe { loop { let before = read_volatile(&self.config_generation); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let mut src = self.mac.iter(); mac.fill_with(|| read_volatile(src.next().unwrap())); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let after = read_volatile(&self.config_generation); @@ -75,8 +80,10 @@ impl NetDevCfgRaw { unsafe { loop { let before = read_volatile(&self.config_generation); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let status = read_volatile(&self.status); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let after = read_volatile(&self.config_generation); @@ -92,8 +99,10 @@ impl NetDevCfgRaw { unsafe { loop { let before = read_volatile(&self.config_generation); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let max_pairs = read_volatile(&self.max_virtqueue_pairs); + #[cfg(target_arch = "x86_64")] _mm_mfence(); let after = read_volatile(&self.config_generation); diff --git a/src/drivers/net/virtio_net.rs b/src/drivers/net/virtio_net.rs index abb20f0dac..3fbc6428a6 100644 --- a/src/drivers/net/virtio_net.rs +++ b/src/drivers/net/virtio_net.rs @@ -13,6 +13,7 @@ use core::result::Result; use self::constants::{FeatureSet, Features, NetHdrGSO, Status, MAX_NUM_VQ}; use self::error::VirtioNetError; +#[cfg(target_arch = "x86_64")] use crate::arch::kernel::percore::increment_irq_counter; use crate::config::VIRTIO_MAX_QUEUE_SIZE; #[cfg(not(feature = "pci"))] @@ -652,6 +653,7 @@ impl NetworkInterface for VirtioNetDriver { } fn handle_interrupt(&mut self) -> bool { + #[cfg(target_arch = "x86_64")] increment_irq_counter((32 + self.irq).into()); let result = if self.isr_stat.is_interrupt() { diff --git a/src/drivers/virtio/transport/mmio.rs b/src/drivers/virtio/transport/mmio.rs index d8dd6e1053..7b2fd38155 100644 --- a/src/drivers/virtio/transport/mmio.rs +++ b/src/drivers/virtio/transport/mmio.rs @@ -10,6 +10,9 @@ use core::sync::atomic::{fence, Ordering}; use core::u8; use crate::arch::mm::PhysAddr; +#[cfg(target_arch = "riscv64")] +use crate::arch::riscv::kernel::irq::*; +#[cfg(target_arch = "x86_64")] use crate::arch::x86_64::kernel::irq::*; use crate::drivers::error::DriverError; use crate::drivers::net::network_irqhandler; @@ -368,6 +371,7 @@ pub fn init_device( info!("Virtio network driver initialized."); // Install interrupt handler irq_install_handler(irq_no, network_irqhandler as usize); + #[cfg(target_arch = "x86_64")] add_irq_name(irq_no, "virtio_net"); Ok(VirtioDriver::Network(virt_net_drv)) diff --git a/src/lib.rs b/src/lib.rs index 6e535ab021..969199b988 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,8 +85,6 @@ mod scheduler; mod synch; mod syscalls; -hermit_entry::define_entry_version!(); - #[doc(hidden)] pub fn _print(args: ::core::fmt::Arguments<'_>) { use core::fmt::Write; @@ -276,6 +274,10 @@ extern "C" fn initd(_arg: usize) { #[cfg(feature = "tcp")] crate::net::init(); + // Initialize MMIO Drivers if on riscv64 + #[cfg(target_arch = "riscv64")] + riscv::kernel::init_drivers(); + syscalls::init(); // Get the application arguments and environment variables. @@ -342,6 +344,8 @@ fn boot_processor_main() -> ! { // Compiles up to here - loop prevents linker errors loop {} } + + #[cfg(not(target_arch = "riscv64"))] scheduler::add_current_core(); if !env::is_uhyve() { @@ -372,6 +376,7 @@ fn boot_processor_main() -> ! { #[cfg(all(target_os = "none", feature = "smp"))] fn application_processor_main() -> ! { arch::application_processor_init(); + #[cfg(not(target_arch = "riscv64"))] scheduler::add_current_core(); info!("Entering idle loop for application processor"); @@ -379,6 +384,8 @@ fn application_processor_main() -> ! { synch_all_cores(); let core_scheduler = core_scheduler(); + + trace!("core_scheduler: {:p}", &core_scheduler); // Run the scheduler loop. core_scheduler.run(); } diff --git a/src/macros.rs b/src/macros.rs index da658f904e..ea6379e280 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -117,6 +117,7 @@ macro_rules! kernel_function { // https://github.com/hermitcore/libhermit-rs/issues/471 #[cfg(any( target_arch = "aarch64", + target_arch = "riscv64", all(target_arch = "x86_64", feature = "newlib") ))] macro_rules! kernel_function { diff --git a/src/scheduler/mod.rs b/src/scheduler/mod.rs index f83a1dc3cb..741b436e2b 100644 --- a/src/scheduler/mod.rs +++ b/src/scheduler/mod.rs @@ -7,10 +7,15 @@ use core::sync::atomic::{AtomicU32, Ordering}; use crossbeam_utils::Backoff; use hermit_sync::{without_interrupts, *}; +#[cfg(target_arch = "riscv64")] +use riscv::register::sstatus; use crate::arch; use crate::arch::irq; use crate::arch::percore::*; +#[cfg(target_arch = "riscv64")] +use crate::arch::switch::switch_to_task; +#[cfg(not(target_arch = "riscv64"))] use crate::arch::switch::{switch_to_fpu_owner, switch_to_task}; use crate::kernel::scheduler::TaskStacks; use crate::scheduler::task::*; @@ -366,6 +371,18 @@ impl PerCoreScheduler { }) } + #[cfg(target_arch = "riscv64")] + pub fn set_current_kernel_stack(&self) { + let current_task_borrowed = self.current_task.borrow(); + + set_kernel_stack( + (current_task_borrowed.stacks.get_kernel_stack() + + current_task_borrowed.stacks.get_kernel_stack_size() + - TaskStacks::MARKER_SIZE) + .as_u64(), + ); + } + /// Save the FPU context for the current FPU owner and restore it for the current task, /// which wants to use the FPU now. pub fn fpu_switch(&mut self) { @@ -535,14 +552,28 @@ impl PerCoreScheduler { unsafe { *last_stack_pointer }, new_stack_pointer ); - self.current_task = task; // Finally save our current context and restore the context of the new task. + #[cfg(not(target_arch = "riscv64"))] if is_idle || Rc::ptr_eq(&self.current_task, &self.fpu_owner) { unsafe { + self.current_task = task; switch_to_fpu_owner(last_stack_pointer, new_stack_pointer.as_usize()); } } else { + unsafe { + self.current_task = task; + switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); + } + } + + #[cfg(target_arch = "riscv64")] + { + if sstatus::read().fs() == sstatus::FS::Dirty { + self.current_task.borrow_mut().last_fpu_state.save(); + } + task.borrow().last_fpu_state.restore(); + self.current_task = task; unsafe { switch_to_task(last_stack_pointer, new_stack_pointer.as_usize()); } diff --git a/src/syscalls/interfaces/uhyve.rs b/src/syscalls/interfaces/uhyve.rs index 2014ea052f..25f9595260 100644 --- a/src/syscalls/interfaces/uhyve.rs +++ b/src/syscalls/interfaces/uhyve.rs @@ -60,6 +60,13 @@ fn uhyve_send(port: u16, data: &mut T) { } } +/// forward a request to the hypervisor uhyve +#[inline] +#[cfg(target_arch = "riscv64")] +fn uhyve_send(port: u16, data: &mut T) { + unimplemented!() +} + const MAX_ARGC_ENVC: usize = 128; #[repr(C, packed)] diff --git a/targets/riscv64gc-unknown-none-hermitkernel.json b/targets/riscv64gc-unknown-none-hermitkernel.json new file mode 100644 index 0000000000..ffdd075003 --- /dev/null +++ b/targets/riscv64gc-unknown-none-hermitkernel.json @@ -0,0 +1,25 @@ +{ + "arch": "riscv64", + "code-model": "medium", + "cpu": "generic-rv64", + "data-layout": "e-m:e-p:64:64-i64:64-i128:128-n64-S128", + "disable-redzone": true, + "executables": true, + "features": "+m,+a,+f,+d,+c", + "linker": "rust-lld", + "linker-flavor": "ld.lld", + "llvm-abiname": "lp64d", + "llvm-target": "riscv64-unknown-hermit", + "max-atomic-width": 64, + "panic-strategy": "abort", + "position-independent-executables": true, + "pre-link-args": { + "ld.lld": [ + "--build-id", + "--hash-style=gnu", + "--Bstatic" + ] + }, + "static-position-independent-executables": true, + "target-pointer-width": "64" +} diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs index 55ba65c5a0..dbf1f39225 100644 --- a/xtask/src/arch.rs +++ b/xtask/src/arch.rs @@ -6,6 +6,7 @@ use anyhow::anyhow; pub enum Arch { X86_64, AArch64, + Riscv64, } impl Arch { @@ -13,6 +14,7 @@ impl Arch { match self { Self::X86_64 => "x86_64", Self::AArch64 => "aarch64", + Self::Riscv64 => "riscv64", } } @@ -20,6 +22,7 @@ impl Arch { match self { Self::X86_64 => "x86_64-unknown-none", Self::AArch64 => "aarch64-unknown-none-softfloat", + Self::Riscv64 => "riscv64gc-unknown-none-hermitkernel", } } @@ -27,6 +30,7 @@ impl Arch { match self { Arch::X86_64 => "x86_64-unknown-hermit", Arch::AArch64 => "aarch64-unknown-hermit", + Arch::Riscv64 => "riscv64gc-unknown-none-hermitkernel", } } @@ -34,6 +38,10 @@ impl Arch { match self { Arch::X86_64 => &["--target=x86_64-unknown-hermit", "-Zbuild-std=core"], Arch::AArch64 => &["--target=aarch64-unknown-hermit", "-Zbuild-std=core"], + Arch::Riscv64 => &[ + "--target=targets/riscv64gc-unknown-none-hermitkernel.json", + "-Zbuild-std=core", + ], } } @@ -47,6 +55,11 @@ impl Arch { "-Zbuild-std=core,alloc", "-Zbuild-std-features=compiler-builtins-mem", ], + Self::Riscv64 => &[ + "--target=targets/riscv64gc-unknown-none-hermitkernel.json", + "-Zbuild-std=core,alloc", + "-Zbuild-std-features=compiler-builtins-mem", + ], } } @@ -54,6 +67,7 @@ impl Arch { match self { Self::X86_64 => &[], Self::AArch64 => &["-Crelocation-model=pic"], + Self::Riscv64 => &[], } } } @@ -65,6 +79,7 @@ impl FromStr for Arch { match s { "x86_64" => Ok(Self::X86_64), "aarch64" => Ok(Self::AArch64), + "riscv64" => Ok(Self::Riscv64), s => Err(anyhow!("Unsupported arch: {s}")), } }