From 3713d7ce1caa74dd7ee3f31d46119c9ebec5808e Mon Sep 17 00:00:00 2001 From: Manami Mori Date: Mon, 7 Feb 2022 12:19:14 +0900 Subject: [PATCH] Version 0.2.0 --- .github/workflows/rust.yml | 46 ++ src/Dockerfile | 15 + src/Makefile | 25 +- src/README.md | 43 ++ src/build_docker_image.sh | 13 + src/build_hypervisor_by_docker.sh | 38 + src/common/Cargo.toml | 2 +- src/common/Makefile | 6 + src/common/src/acpi.rs | 6 + src/common/src/acpi/madt.rs | 6 + src/common/src/cpu.rs | 303 +++++++- src/common/src/lib.rs | 16 +- src/common/src/paging.rs | 170 +++++ src/common/src/serial_port.rs | 6 + src/hypervisor_bootloader/Cargo.toml | 2 +- src/hypervisor_bootloader/Makefile | 6 + src/hypervisor_bootloader/src/console.rs | 6 + src/hypervisor_bootloader/src/dtb.rs | 6 + src/hypervisor_bootloader/src/elf.rs | 6 + src/hypervisor_bootloader/src/main.rs | 131 +--- src/hypervisor_bootloader/src/paging.rs | 671 +++++++----------- src/hypervisor_bootloader/src/panic.rs | 12 +- src/hypervisor_bootloader/src/pci.rs | 43 ++ src/hypervisor_bootloader/src/serial_port.rs | 6 + src/hypervisor_kernel/Cargo.toml | 4 +- src/hypervisor_kernel/Makefile | 6 + src/hypervisor_kernel/src/drivers.rs | 11 + src/hypervisor_kernel/src/drivers/i210.rs | 375 ++++++++++ src/hypervisor_kernel/src/emulation.rs | 264 +++++++ src/hypervisor_kernel/src/emulation/load.rs | 304 ++++++++ src/hypervisor_kernel/src/emulation/store.rs | 229 ++++++ src/hypervisor_kernel/src/main.rs | 286 +++++--- src/hypervisor_kernel/src/memory_hook.rs | 201 ++++++ src/hypervisor_kernel/src/multi_core.rs | 89 ++- src/hypervisor_kernel/src/paging.rs | 627 ++++++++++++++++ src/hypervisor_kernel/src/panic.rs | 12 +- src/hypervisor_kernel/src/pci.rs | 84 +++ src/hypervisor_kernel/src/psci.rs | 16 +- src/hypervisor_kernel/src/serial_port.rs | 22 +- .../src/serial_port/arm_pl011.rs | 6 + .../src/serial_port/arm_sbsa_generic_uart.rs | 6 + .../src/serial_port/meson_gx_uart.rs | 6 + src/uefi/Cargo.toml | 2 +- src/uefi/Makefile | 6 + src/uefi/src/boot_service.rs | 6 + src/uefi/src/boot_service/memory_service.rs | 6 + src/uefi/src/file.rs | 6 + src/uefi/src/lib.rs | 6 + src/uefi/src/loaded_image.rs | 6 + src/uefi/src/output.rs | 6 + 50 files changed, 3539 insertions(+), 637 deletions(-) create mode 100644 .github/workflows/rust.yml create mode 100644 src/Dockerfile create mode 100644 src/README.md create mode 100644 src/build_docker_image.sh create mode 100644 src/build_hypervisor_by_docker.sh create mode 100644 src/common/src/paging.rs create mode 100644 src/hypervisor_bootloader/src/pci.rs create mode 100644 src/hypervisor_kernel/src/drivers.rs create mode 100644 src/hypervisor_kernel/src/drivers/i210.rs create mode 100644 src/hypervisor_kernel/src/emulation.rs create mode 100644 src/hypervisor_kernel/src/emulation/load.rs create mode 100644 src/hypervisor_kernel/src/emulation/store.rs create mode 100644 src/hypervisor_kernel/src/memory_hook.rs create mode 100644 src/hypervisor_kernel/src/paging.rs create mode 100644 src/hypervisor_kernel/src/pci.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..99aa494 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,46 @@ +name: Rust + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install tools + run: | + sudo apt-get update + sudo apt-get install -y build-essential make + rustup default nightly + rustup component add rust-src + rustup update + cargo install cargo-xbuild + mkdir bin + + - name: Build bootloader + run: | + cd src + make bootloader + mv hypervisor_bootloader/target/aarch64-uefi/release/hypervisor_bootloader ../bin/BOOTAA64.EFI + + - name: Build kernel + run: | + cd src + make kernel + mv hypervisor_kernel/target/aarch64-none/release/hypervisor_kernel ../bin/hypervisor_kernel + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: Thin Hypervisor + path: bin/ + + diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..d7d6866 --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,15 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + +FROM rust:latest + +RUN rustup default nightly \ + && rustup component add rust-src +RUN rustup update +RUN apt-get update && apt-get install -y \ + build-essential make \ + && rm -rf /var/lib/apt/lists/* +RUN cargo install cargo-xbuild diff --git a/src/Makefile b/src/Makefile index b1e50f8..6232cc0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,3 +1,9 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + MAKE = make -C COMMON_MODULE = common UEFI_MODULE = uefi @@ -11,15 +17,19 @@ QEMU = qemu-system-aarch64 RM = rm -rf MOUNT = mount UMOUNT = umount -UEFI_FD ?= QEMU_EFI.fd +QEMU_EFI ?= QEMU_EFI.fd -all: .FORCE - $(MAKE) $(BOOTLOADER) - $(MAKE) $(KERNEL) +all: bootloader kernel $(MKDIR) bin/EFI/BOOT/ $(CP) $(BOOTLOADER)/target/$(BOOTLOADER_ARCH)/release/$(BOOTLOADER) bin/EFI/BOOT/BOOTAA64.EFI $(CP) $(KERNEL)/target/$(KERNEL_ARCH)/release/$(KERNEL) bin/EFI/BOOT/$(KERNEL) +bootloader: .FORCE + $(MAKE) $(BOOTLOADER) + +kernel: .FORCE + $(MAKE) $(KERNEL) + clean: $(RM) bin $(MAKE) $(COMMON_MODULE) clean @@ -34,10 +44,10 @@ fmt: $(MAKE) $(KERNEL) fmt run: all - $(QEMU) -m 1G -cpu cortex-a53 -machine virt-2.12,virtualization=on -smp 4 -nographic -bios $(UEFI_FD) -drive file=fat:rw:bin/,format=raw,media=disk + $(QEMU) -m 1G -cpu cortex-a53 -machine virt-2.12,virtualization=on -smp 4 -nographic -bios $(QEMU_EFI) -drive file=fat:rw:bin/,format=raw,media=disk debug: all - $(QEMU) -m 1G -cpu cortex-a53 -machine virt-2.12,virtualization=on -smp 4 -monitor stdio -bios $(UEFI_FD) -drive file=fat:rw:bin/,format=raw,media=disk + $(QEMU) -m 1G -cpu cortex-a53 -machine virt-2.12,virtualization=on -smp 4 -monitor stdio -bios $(QEMU_EFI) -drive file=fat:rw:bin/,format=raw,media=disk write: $(MOUNT) $(DEVICE) /mnt @@ -45,7 +55,4 @@ write: $(CP) bin/EFI/BOOT/$(KERNEL) /mnt/EFI/BOOT/$(KERNEL) $(UMOUNT) /mnt -default: - $(MAKE) all - .FORCE: diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..5a114bd --- /dev/null +++ b/src/README.md @@ -0,0 +1,43 @@ +# How to build the hypervisor + +## By Rust toolchain +(TBD) + + +## By docker +### Requirements +- Docker (Tested by `Docker version 20.10.8, build 3967b7d28e`) + - I tested by non-root users (See [this](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user) to run docker command by non-root user) + +### Steps (commands list) + +```bash +cd path/to/repo-root/src +./build_docker_image.sh #Build docker image to build +./build_hypervisor_by_docker.sh #Build the hypervisor by the docker image +``` +More detail, please see the scripts. + +# How to run the hypervisor +## On QEMU +First, please install QEMU that support to emulate `QEMU 2.12 ARM Virtual Machine`, `cortex-a53` CPU. +Then, run the following command to run the built hypervisor. + +```bash +cd path/to/repo-root/src +make QEMU_EFI=/usr/share/qemu-efi/QEMU_EFI.fd run #Please set the path of your QEMU_EFI.fd to QEMU_EFI +``` + +## On a physical machine from an USB memory stick +### Requirement +- Prepare a USB memory which has an EFI (FAT) partition that has `/EFI/BOOT/` directory. Please confirm that there is no important file in the partition. +- Prepare a physical machine that has ARMv8-A or later, and UEFI firmware. + +### Steps +1. Attach your USB memory stick to the development machine which built the hypervisor binary. +2. Identify the EFI partition (in the following description, `/dev/sdX1` is the EFI partition). +3. Run `sudo make DEVICE=/dev/sdX1 write` to copy the binary. + !! Please be carefully not to specifying a wrong partition as `DEVICE` because the script mount/unmount the partition and copy the binary file with root privilege.!! +4. Detach the USB memory from the development machine, and attach it to the physical machine to run the hypervisor. +5. Boot the physical machine with UEFI, and specify `BOOTAA64.EFI` in the EFI partition as the EFI application to boot. + diff --git a/src/build_docker_image.sh b/src/build_docker_image.sh new file mode 100644 index 0000000..63263bb --- /dev/null +++ b/src/build_docker_image.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + +IMG_NAME="rust-build-thin-hypervisor" +timestamp=$(date +%Y%m%d_%H%M%S) + +docker build -f Dockerfile -t "${IMG_NAME}:${timestamp}" . \ + && docker tag "${IMG_NAME}:${timestamp}" "${IMG_NAME}:latest" diff --git a/src/build_hypervisor_by_docker.sh b/src/build_hypervisor_by_docker.sh new file mode 100644 index 0000000..ddccf82 --- /dev/null +++ b/src/build_hypervisor_by_docker.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + +cd $(dirname $0) +PWD="$(pwd)" +CONTAINER_NAME="rust-build-thin-hypervisor" +IMG_NAME="rust-build-thin-hypervisor" + +do_clean=false + +while (( $# > 0 )) +do + case $1 in + --clean) + do_clean=true + ;; + *) + echo "Unknow argument" + exit 1 + ;; + esac + shift 1 +done + +build_cmdline='cd /workspace && make' +if $do_clean; then + build_cmdline="${build_cmdline} clean" +fi + +docker run -it --rm --name ${CONTAINER_NAME} \ + -v ${PWD}:/workspace \ + ${IMG_NAME}:latest \ + bash -c "${build_cmdline}" diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index e020b17..0e8e5b7 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "common" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] diff --git a/src/common/Makefile b/src/common/Makefile index c957bdb..673508d 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -1,3 +1,9 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + CARGO = cargo clean: diff --git a/src/common/src/acpi.rs b/src/common/src/acpi.rs index fcdd456..27d12bd 100644 --- a/src/common/src/acpi.rs +++ b/src/common/src/acpi.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Advanced Configuration and Power Interface //! diff --git a/src/common/src/acpi/madt.rs b/src/common/src/acpi/madt.rs index 74f7ccb..a8fc83a 100644 --- a/src/common/src/acpi/madt.rs +++ b/src/common/src/acpi/madt.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Multiple APIC Description Table //! diff --git a/src/common/src/cpu.rs b/src/common/src/cpu.rs index 9e6619b..00e1059 100644 --- a/src/common/src/cpu.rs +++ b/src/common/src/cpu.rs @@ -1,14 +1,118 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! CPU Specified Assembly functions //! use crate::{bitmask, PAGE_MASK, PAGE_SHIFT}; +use core::arch::asm; + #[derive(Clone)] pub struct InterruptFlag(u64); -const DAIF_IRQ_BIT: u64 = 7; -const DAIF_FIQ_BIT: u64 = 6; +/* CPU Bit Fields */ +pub const DAIF_IRQ_BIT: u64 = 7; +pub const DAIF_FIQ_BIT: u64 = 6; + +/* CNTHCTL_EL2 Register */ +pub const CNTHCTL_EL2_EL1PTEN: u64 = 1 << 11; +pub const CNTHCTL_EL2_EL1PCTEN: u64 = 1 << 10; +pub const CNTHCTL_EL2_EL0PTEN: u64 = 1 << 9; +pub const CNTHCTL_EL2_EL0PCTEN: u64 = 1 << 8; + +/* CPACR_EL1 Register */ +pub const CPACR_EL1_TTA_BIT_OFFSET: u64 = 28; +//pub const CPACR_EL1_TTA: u64 = 1 << CPACR_EL1_TTA_BIT_OFFSET; +pub const CPACR_EL1_FPEN_BITS_OFFSET: u64 = 20; +//pub const CPACR_EL1_FPEN: u64 = 0b11 << CPACR_EL1_FPEN_BITS_OFFSET; +pub const CPACR_EL1_ZEN_BITS_OFFSET: u64 = 16; +//pub const CPACR_EL1_ZEN: u64 = 0b11 << CPACR_EL1_ZEN_BITS_OFFSET; + +/* CPTR_EL2 Register */ +pub const CPTR_EL2_TTA_BIT_OFFSET_WITH_E2H: u64 = 28; +pub const CPTR_EL2_TTA_WITH_E2H: u64 = 1 << CPTR_EL2_TTA_BIT_OFFSET_WITH_E2H; +pub const CPTR_EL2_TTA_BIT_OFFSET_WITHOUT_E2H: u64 = 20; +pub const CPTR_EL2_TTA_WITHOUT_E2H: u64 = 1 << CPTR_EL2_TTA_BIT_OFFSET_WITHOUT_E2H; +pub const CPTR_EL2_FPEN_BITS_OFFSET: u64 = 20; +pub const CPTR_EL2_FPEN: u64 = 0b11 << CPTR_EL2_FPEN_BITS_OFFSET; +pub const CPTR_EL2_FPEN_NO_TRAP: u64 = 0b11 << CPTR_EL2_FPEN_BITS_OFFSET; +pub const CPTR_EL2_ZEN_BITS_OFFSET: u64 = 16; +pub const CPTR_EL2_ZEN: u64 = 0b11 << CPTR_EL2_ZEN_BITS_OFFSET; +pub const CPTR_EL2_ZEN_NO_TRAP: u64 = 0b11 << CPTR_EL2_ZEN_BITS_OFFSET; +//pub const CPTR_EL2_RES1: u64 = 0b11111111 | (1 << 9) | (0b11 << 12); + +/* TCR_EL2 Register */ +pub const TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H: u64 = 32; +pub const TCR_EL2_DS_WITHOUT_E2H: u64 = 1 << TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_TCMA_BIT_OFFSET_WITHOUT_E2H: u64 = 30; +pub const TCR_EL2_TCMA_WITHOUT_E2H: u64 = 1 << TCR_EL2_TCMA_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_TBID_BIT_OFFSET_WITHOUT_E2H: u64 = 29; +pub const TCR_EL2_TBID_WITHOUT_E2H: u64 = 1 << TCR_EL2_TBID_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_HWU_BITS_OFFSET_WITHOUT_E2H: u64 = 25; +pub const TCR_EL2_HWU_WITHOUT_E2H: u64 = 0b1111 << TCR_EL2_HWU_BITS_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_HPD_BIT_OFFSET_WITHOUT_E2H: u64 = 24; +pub const TCR_EL2_HPD_WITHOUT_E2H: u64 = 1 << TCR_EL2_HPD_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_HD_BIT_OFFSET_WITHOUT_E2H: u64 = 22; +pub const TCR_EL2_HD_WITHOUT_E2H: u64 = 1 << TCR_EL2_HD_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_HA_BIT_OFFSET_WITHOUT_E2H: u64 = 21; +pub const TCR_EL2_HA_WITHOUT_E2H: u64 = 1 << TCR_EL2_HA_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_TBI_BIT_OFFSET_WITHOUT_E2H: u64 = 20; +pub const TCR_EL2_TBI_WITHOUT_E2H: u64 = 1 << TCR_EL2_TBI_BIT_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_PS_BITS_OFFSET_WITHOUT_E2H: u64 = 16; +pub const TCR_EL2_PS_WITHOUT_E2H: u64 = 0b111 << TCR_EL2_PS_BITS_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H: u64 = 14; +pub const TCR_EL2_TG0_WITHOUT_E2H: u64 = 0b11 << TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H; +pub const TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H: u64 = 0; +pub const TCR_EL2_T0SZ_WITHOUT_E2H: u64 = 0b111111 << TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H; + +/* TCR_EL1 Register */ +pub const TCR_EL1_DS_BIT_OFFSET: u64 = 59; +//pub const TCR_EL1_DS: u64 = 1 << TCR_EL1_DS_BIT_OFFSET; +pub const TCR_EL1_TCMA0_BIT_OFFSET: u64 = 57; +//pub const TCR_EL1_TCMA0: u64 = 1 << TCR_EL1_TCMA0_BIT_OFFSET; +pub const TCR_EL1_TBID0_BIT_OFFSET: u64 = 51; +//pub const TCR_EL1_TBID0: u64 = 1 << TCR_EL1_TBID0_BIT_OFFSET; +pub const TCR_EL1_HWU_BITS_OFFSET: u64 = 43; +//pub const TCR_EL1_HWU: u64 = 0b11111111 << TCR_EL1_HWU_BITS_OFFSET; +pub const TCR_EL1_HPD0_BIT_OFFSET: u64 = 41; +//pub const TCR_EL1_HPD0: u64 = 1 << TCR_EL1_HPD0_BIT_OFFSET; +pub const TCR_EL1_HD_BIT_OFFSET: u64 = 40; +//pub const TCR_EL1_HD: u64 = 1 << TCR_EL1_HD_BIT_OFFSET; +pub const TCR_EL1_HA_BIT_OFFSET: u64 = 39; +//pub const TCR_EL1_HA: u64 = 1 << TCR_EL1_HA_BIT_OFFSET; +pub const TCR_EL1_TBI0_BIT_OFFSET: u64 = 37; +//pub const TCR_EL1_TBI0: u64 = 1 << TCR_EL1_TBI0_BIT_OFFSET; +pub const TCR_EL1_IPS_BITS_OFFSET: u64 = 32; +//pub const TCR_EL1_IPS: u64 = 0b111 << TCR_EL1_IPS_BITS_OFFSET; +pub const TCR_EL1_EPD1: u64 = 1 << 23; + +/* HCR_EL2 Register */ +pub const HCR_EL2_FIEN: u64 = 1 << 47; +pub const HCR_EL2_API: u64 = 1 << 41; +pub const HCR_EL2_APK: u64 = 1 << 40; +//pub const HCR_EL2_TEA: u64 = 1 << 37; +pub const HCR_EL2_E2H: u64 = 1 << 34; +pub const HCR_EL2_RW: u64 = 1 << 31; +pub const HCR_EL2_TSC: u64 = 1 << 19; +pub const HCR_EL2_VM: u64 = 1 << 0; + +/* VTCR_EL2 Register */ +pub const VTCR_EL2_RES1: u64 = 1 << 31; +pub const VTCR_EL2_HWU_BITS_OFFSET: u64 = 25; +pub const VTCR_EL2_PS_BITS_OFFSET: u64 = 16; +pub const VTCR_EL2_TG0_BITS_OFFSET: u64 = 14; +pub const VTCR_EL2_SH0_BITS_OFFSET: u64 = 12; +pub const VTCR_EL2_ORG0_BITS_OFFSET: u64 = 10; +pub const VTCR_EL2_IRG0_BITS_OFFSET: u64 = 8; +pub const VTCR_EL2_SL0_BITS_OFFSET: u64 = 6; +pub const VTCR_EL2_SL0: u64 = 0b11 << VTCR_EL2_SL0_BITS_OFFSET; +pub const VTCR_EL2_T0SZ_BITS_OFFSET: u64 = 0; +pub const VTCR_EL2_T0SZ: u64 = 0b111111 << VTCR_EL2_T0SZ_BITS_OFFSET; /// SMC Calling Convention 1.2に沿ったSMCを発行 /// @@ -59,6 +163,104 @@ pub fn secure_monitor_call( }; } +#[inline(always)] +pub fn get_ttbr0_el2() -> u64 { + let ttbr0_el2: u64; + unsafe { asm!("mrs {:x}, ttbr0_el2", out(reg) ttbr0_el2) }; + return ttbr0_el2; +} + +#[inline(always)] +pub fn set_ttbr0_el2(ttbr0_el2: u64) { + unsafe { asm!("msr ttbr0_el2, {:x}", in(reg) ttbr0_el2) }; +} + +#[inline(always)] +pub fn set_ttbr0_el1(ttbr0_el1: u64) { + unsafe { asm!("msr ttbr0_el1, {:x}", in(reg) ttbr0_el1) }; +} + +#[inline(always)] +pub fn get_tcr_el2() -> u64 { + let tcr_el2: u64; + unsafe { asm!("mrs {:x}, tcr_el2", out(reg) tcr_el2) }; + return tcr_el2; +} + +#[inline(always)] +pub fn set_tcr_el2(tcr_el2: u64) { + unsafe { asm!("msr tcr_el2, {:x}", in(reg) tcr_el2) }; +} + +#[inline(always)] +pub fn get_vttbr_el2() -> u64 { + let vttbr_el2: u64; + unsafe { asm!("mrs {:x}, vttbr_el2", out(reg) vttbr_el2) }; + return vttbr_el2; +} + +#[inline(always)] +pub fn set_vttbr_el2(vttbr_el2: u64) { + unsafe { asm!("msr vttbr_el2, {:x}", in(reg) vttbr_el2) }; + flush_tlb_el1(); /* May be needless */ +} + +#[inline(always)] +pub fn get_vtcr_el2() -> u64 { + let vtcr_el2: u64; + unsafe { asm!("mrs {:x}, vtcr_el2", out(reg) vtcr_el2) }; + return vtcr_el2; +} + +#[inline(always)] +pub fn set_vtcr_el2(vtcr_el2: u64) { + unsafe { asm!("msr vtcr_el2, {:x}", in(reg) vtcr_el2) }; +} + +#[inline(always)] +pub fn get_mair_el2() -> u64 { + let mair_el2: u64; + unsafe { asm!("mrs {:x}, mair_el2",out(reg) mair_el2) }; + return mair_el2; +} + +#[inline(always)] +pub fn get_id_aa64mmfr0_el1() -> u64 { + let id_aa64mmfr0_el1: u64; + unsafe { asm!("mrs {:x}, id_aa64mmfr0_el1",out(reg) id_aa64mmfr0_el1) }; + return id_aa64mmfr0_el1; +} + +#[inline(always)] +pub fn flush_tlb_el2() { + unsafe { + asm!( + " + tlbi alle2 + dsb sy + isb" + ) + }; +} + +#[inline(always)] +pub fn flush_tlb_el1() { + unsafe { + asm!( + " + dsb ishst + tlbi alle1 + dsb ish + isb" + ) + }; +} + +#[inline(always)] +pub fn flush_tlb_vmalls12e1() { + unsafe { asm!("TLBI VMALLS12E1") }; +} + /// 現時点の割り込み状況を保存し、IRQ/FIQを禁止 /// /// # Return Value @@ -116,3 +318,100 @@ pub fn convert_virtual_address_to_physical_address_el2_read( Err(()) } } + +/// 渡された仮想アドレスをEL2での仮想アドレスと解釈し、物理アドレスに変換 +/// +/// AT S1E2Wを使用して、物理アドレスに変換します。マップされてない場合などはErrを返します。 +/// +/// # Arguments +/// virtual_address: 変換を行う仮想アドレス +/// +/// # Return Value +/// 変換に成功した場合はOk(physical_address)、失敗した場合はErr(()) +pub fn convert_virtual_address_to_physical_address_el2_write( + virtual_address: usize, +) -> Result { + let aligned_virtual_address = virtual_address & PAGE_MASK; + let offset = virtual_address & !PAGE_MASK; + let aligned_physical_address: usize; + unsafe { + asm!(" at S1E2W, {:x} + mrs {:x}, par_el1", + in(reg) (aligned_virtual_address), + out(reg) aligned_physical_address) + }; + + if (aligned_physical_address & 1) == 0 { + Ok((aligned_physical_address & bitmask!(51, PAGE_SHIFT)) + offset) + } else { + Err(()) + } +} + +/// 渡された仮想アドレスをEL1での仮想アドレスと解釈し、中間物理アドレス(IPA)に変換 +/// +/// AT S1E1Rを使用して、中間物理アドレスに変換します。マップされてない場合などはErrを返します。 +/// +/// # Arguments +/// virtual_address: 変換を行う仮想アドレス +/// +/// # Return Value +/// 変換に成功した場合はOk(physical_address)、失敗した場合はErr(()) +pub fn convert_virtual_address_to_intermediate_physical_address_el1_read( + virtual_address: usize, +) -> Result { + let aligned_virtual_address = virtual_address & PAGE_MASK; + let offset = virtual_address & !PAGE_MASK; + let aligned_physical_address: usize; + unsafe { + asm!(" at S1E1R, {:x} + mrs {:x}, par_el1", + in(reg) (aligned_virtual_address), + out(reg) aligned_physical_address) + }; + + if (aligned_physical_address & 1) == 0 { + Ok((aligned_physical_address & bitmask!(51, PAGE_SHIFT)) + offset) + } else { + Err(()) + } +} + +/// 渡された仮想アドレスをEL1での仮想アドレスと解釈し、中間物理アドレス(IPA)に変換 +/// +/// AT S1E1Wを使用して、中間物理アドレスに変換します。マップされてない場合などはErrを返します。 +/// +/// # Arguments +/// virtual_address: 変換を行う仮想アドレス +/// +/// # Return Value +/// 変換に成功した場合はOk(physical_address)、失敗した場合はErr(()) +pub fn convert_virtual_address_to_intermediate_physical_address_el1_write( + virtual_address: usize, +) -> Result { + let aligned_virtual_address = virtual_address & PAGE_MASK; + let offset = virtual_address & !PAGE_MASK; + let aligned_physical_address: usize; + unsafe { + asm!(" at S1E1W, {:x} + mrs {:x}, par_el1", + in(reg) (aligned_virtual_address), + out(reg) aligned_physical_address) + }; + + if (aligned_physical_address & 1) == 0 { + Ok((aligned_physical_address & bitmask!(51, PAGE_SHIFT)) + offset) + } else { + Err(()) + } +} + +/// Halt Loop +/// +/// CPUを待機状態にさせ停止させる +/// マルチコア制御には未対応 +pub fn halt_loop() -> ! { + loop { + unsafe { asm!("wfi") }; + } +} diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index cc67b65..32e82d2 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -1,8 +1,14 @@ -#![feature(asm)] +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + #![no_std] pub mod acpi; pub mod cpu; +pub mod paging; pub mod serial_port; use crate::serial_port::SerialPortInfo; @@ -24,6 +30,7 @@ pub const PAGE_SIZE: usize = 0x1000; pub const PAGE_SHIFT: usize = 12; pub const STAGE_2_PAGE_SIZE: usize = 0x1000; pub const STAGE_2_PAGE_SHIFT: usize = 12; +pub const STAGE_2_PAGE_MASK: usize = !0xFFF; /// 各CPUに割り当てるスタックのページ数 /// STACK_SIZE = STACK_PAGES << PAGE_SHIFT = STACK_PAGES * PAGE_SIZE pub const STACK_PAGES: usize = 16; @@ -37,9 +44,16 @@ macro_rules! bitmask { }; } +pub struct EcamInfo { + pub address: usize, + pub start_bus: u8, + pub end_bus: u8, +} + /// For communicating about system registers between hypervisor_bootloader and hypervisor_kernel pub struct SystemInformation { pub vbar_el2: u64, pub memory_pool: &'static ([MaybeUninit; ALLOC_SIZE / PAGE_SIZE], usize), pub serial_port: Option, + pub ecam_info: Option, } diff --git a/src/common/src/paging.rs b/src/common/src/paging.rs new file mode 100644 index 0000000..baaa652 --- /dev/null +++ b/src/common/src/paging.rs @@ -0,0 +1,170 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +//! +//! Paging +//! + +use crate::bitmask; +use crate::cpu::{ + TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H, TCR_EL2_DS_WITHOUT_E2H, + TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_T0SZ_WITHOUT_E2H, + TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_TG0_WITHOUT_E2H, +}; + +use core::arch::asm; + +pub const PAGE_TABLE_SIZE: usize = 0x1000; + +pub const PAGE_DESCRIPTORS_UPPER_ATTRIBUTES_OFFSET: u64 = 50; +pub const PAGE_DESCRIPTORS_CONTIGUOUS: u64 = 1 << 52; +pub const PAGE_DESCRIPTORS_NX_BIT_OFFSET: u64 = 54; + +pub const PAGE_DESCRIPTORS_NT: u64 = 1 << 16; +pub const PAGE_DESCRIPTORS_AF_BIT_OFFSET: u64 = 10; +pub const PAGE_DESCRIPTORS_AF: u64 = 1 << PAGE_DESCRIPTORS_AF_BIT_OFFSET; +pub const PAGE_DESCRIPTORS_SH_BITS_OFFSET: u64 = 8; +pub const PAGE_DESCRIPTORS_SH_INNER_SHAREABLE: u64 = 0b11 << PAGE_DESCRIPTORS_SH_BITS_OFFSET; +pub const PAGE_DESCRIPTORS_AP_BITS_OFFSET: u64 = 6; + +pub const MEMORY_PERMISSION_READABLE_BIT: u8 = 0; +pub const MEMORY_PERMISSION_WRITABLE_BIT: u8 = 1; +pub const MEMORY_PERMISSION_EXECUTABLE_BIT: u8 = 2; + +const STAGE_2_PAGE_ENTRY_ATTRIBUTE: u64 = + 1 << 10 /* AF bit */| + 0b11 << 8 /* SH bits (Inner sharable) */| + 0b1111 << 2 /* MemAttr(Write-back) */; + +#[derive(Copy, Clone)] +pub struct TTBR(u64); /* Translation Table Base Register */ + +impl TTBR { + const ASID: u64 = (u16::MAX as u64) << 48; + const CNP: u64 = 1 << 0; + + pub const fn new(ttbr: u64) -> Self { + Self(ttbr) + } + + pub const fn get_base_address(&self) -> usize { + ((self.0) & !(Self::CNP | Self::ASID)) as usize + } +} + +pub const fn extract_output_address(descriptor: u64, page_shift: usize) -> usize { + (descriptor + & bitmask!( + PAGE_DESCRIPTORS_UPPER_ATTRIBUTES_OFFSET - 1, + page_shift as u64 + )) as usize +} + +pub const fn is_descriptor_table_or_level_3_descriptor(descriptor: u64) -> bool { + descriptor & 0b11 == 0b11 +} + +pub const fn is_block_descriptor(descriptor: u64) -> bool { + descriptor & 0b11 == 0b01 +} + +pub const fn create_attributes_for_stage_1( + permission: u8, + memory_attribute: u8, + is_block_entry: bool, +) -> u64 { + let nx_bit: u64 = if (permission & (1 << MEMORY_PERMISSION_EXECUTABLE_BIT)) != 0 { + 0 + } else { + 1 + } << PAGE_DESCRIPTORS_NX_BIT_OFFSET; + let access_permission: u64 = if (permission & (1 << MEMORY_PERMISSION_WRITABLE_BIT)) != 0 { + 0b00 + } else { + 0b10 + } << PAGE_DESCRIPTORS_AP_BITS_OFFSET; + + nx_bit + | PAGE_DESCRIPTORS_AF + | PAGE_DESCRIPTORS_SH_INNER_SHAREABLE + | nx_bit + | access_permission + | (memory_attribute << 2) as u64 + | if is_block_entry { 0b01 } else { 0b11 } +} + +pub const fn create_attributes_for_stage_2( + permission: u8, + _is_dummy_page: bool, + is_unmap: bool, + is_block_entry: bool, +) -> u64 { + PAGE_DESCRIPTORS_AF + | (((permission as u64) & (!(1 << MEMORY_PERMISSION_EXECUTABLE_BIT))) << 6) + | STAGE_2_PAGE_ENTRY_ATTRIBUTE + | if is_unmap { + 0b00 + } else if is_block_entry { + 0b01 + } else { + 0b11 + } +} + +pub const fn table_level_to_table_shift( + translation_granule_shift: usize, + table_level: i8, +) -> usize { + translation_granule_shift + 9 * (3 - table_level) as usize +} + +/// 現時点ではTTBR0_EL2のみ対応 +pub const fn get_initial_page_table_level_and_bits_to_shift(tcr_el2: u64) -> (i8, usize) { + let tcr_el2_ds = + ((tcr_el2 & TCR_EL2_DS_WITHOUT_E2H) >> TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H) as u8; + let tcr_el2_tg0 = (tcr_el2 & TCR_EL2_TG0_WITHOUT_E2H) >> TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H; + let tcr_el2_t0sz = + ((tcr_el2 & TCR_EL2_T0SZ_WITHOUT_E2H) >> TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H) as usize; + let page_shift = 12 + (tcr_el2_tg0 << 1) as usize; + + /* aarch64/translation/walk/AArch64.TranslationTableWalk (J1-7982) */ + let first_level = 4 + - (1 + ((64 - tcr_el2_t0sz /* TTBR1_EL2ではここが t1sz(TCR_EL2[21:16] */ - page_shift - 1) + / (page_shift - 3))) as i8; + + if tcr_el2_ds == 0 && first_level == -1 { + panic!("5-Level Paging with DS == 0 is invalid."); + } + ( + first_level, + table_level_to_table_shift(page_shift, first_level), + ) +} + +pub fn get_suitable_memory_attribute_index_from_mair_el2(is_device: bool) -> u8 { + let mut mair_el2: u64; + let suitable_attribute: u64 = if is_device { 0x00 } else { 0xff }; + unsafe { asm!("mrs {:x},mair_el2",out(reg) mair_el2) }; + for index in 0..7 { + let attribute = mair_el2 & 0xff; + if attribute == suitable_attribute { + return index; + } + mair_el2 >>= 8; + } + panic!("Attr=={:#X} is not found...", suitable_attribute); +} + +pub const fn calculate_number_of_concatenated_page_tables( + t0sz: u8, + initial_lookup_level: i8, +) -> u8 { + if t0sz > (43 - ((3 - initial_lookup_level) as u8) * 9) { + 1 + } else { + 2u8.pow(((43 - ((3 - initial_lookup_level) as u8) * 9) - t0sz) as u32) + } +} diff --git a/src/common/src/serial_port.rs b/src/common/src/serial_port.rs index eaec93f..110b492 100644 --- a/src/common/src/serial_port.rs +++ b/src/common/src/serial_port.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Serial Port //! diff --git a/src/hypervisor_bootloader/Cargo.toml b/src/hypervisor_bootloader/Cargo.toml index 44849e4..2cb21c6 100644 --- a/src/hypervisor_bootloader/Cargo.toml +++ b/src/hypervisor_bootloader/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypervisor_bootloader" -version = "0.1.0" +version = "0.2.0" edition = "2021" diff --git a/src/hypervisor_bootloader/Makefile b/src/hypervisor_bootloader/Makefile index 85edc57..0727e5d 100644 --- a/src/hypervisor_bootloader/Makefile +++ b/src/hypervisor_bootloader/Makefile @@ -1,3 +1,9 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + NAME = hypervisor_bootloader TARGET = aarch64-uefi TARGET_JSON = config/$(TARGET).json diff --git a/src/hypervisor_bootloader/src/console.rs b/src/hypervisor_bootloader/src/console.rs index 8e5ec88..3a953ff 100644 --- a/src/hypervisor_bootloader/src/console.rs +++ b/src/hypervisor_bootloader/src/console.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Console Input/Output Manager //! diff --git a/src/hypervisor_bootloader/src/dtb.rs b/src/hypervisor_bootloader/src/dtb.rs index 398a142..aecf67e 100644 --- a/src/hypervisor_bootloader/src/dtb.rs +++ b/src/hypervisor_bootloader/src/dtb.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + const FDT_BEGIN_NODE: u32 = 0x00000001u32.to_be(); const FDT_END_NODE: u32 = 0x00000002u32.to_be(); const FDT_PROP: u32 = 0x00000003u32.to_be(); diff --git a/src/hypervisor_bootloader/src/elf.rs b/src/hypervisor_bootloader/src/elf.rs index f7ece88..9f81a2c 100644 --- a/src/hypervisor_bootloader/src/elf.rs +++ b/src/hypervisor_bootloader/src/elf.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Executable and Linkable Format //! diff --git a/src/hypervisor_bootloader/src/main.rs b/src/hypervisor_bootloader/src/main.rs index d3c7575..6bce497 100644 --- a/src/hypervisor_bootloader/src/main.rs +++ b/src/hypervisor_bootloader/src/main.rs @@ -1,12 +1,16 @@ -#![feature(asm)] +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +#![no_std] +#![no_main] #![feature(asm_sym)] -#![feature(const_size_of_val)] #![feature(maybe_uninit_array_assume_init)] #![feature(maybe_uninit_uninit_array)] #![feature(naked_functions)] #![feature(panic_info_message)] -#![no_std] -#![no_main] #[macro_use] mod console; @@ -14,9 +18,10 @@ mod dtb; mod elf; mod paging; mod panic; +mod pci; mod serial_port; -use common::cpu::{local_irq_fiq_restore, local_irq_fiq_save, InterruptFlag}; +use common::cpu::*; use common::{ HypervisorKernelMainType, SystemInformation, ALLOC_SIZE, HYPERVISOR_PATH, HYPERVISOR_SERIAL_BASE_ADDRESS, HYPERVISOR_VIRTUAL_BASE_ADDRESS, MAX_PHYSICAL_ADDRESS, @@ -28,6 +33,7 @@ use uefi::{ EfiSystemTable, EFI_ACPI_20_TABLE_GUID, EFI_DTB_TABLE_GUID, }; +use core::arch::asm; use core::mem::{transmute, MaybeUninit}; static mut ORIGINAL_PAGE_TABLE: usize = 0; @@ -43,88 +49,6 @@ static mut SYSTEM_TABLE: *const EfiSystemTable = core::ptr::null(); static mut ACPI_20_TABLE_ADDRESS: Option = None; static mut DTB_ADDRESS: Option = None; -/* CNTHCTL_EL2 Register */ -const CNTHCTL_EL2_EL1PTEN: u64 = 1 << 11; -const CNTHCTL_EL2_EL1PCTEN: u64 = 1 << 10; -const CNTHCTL_EL2_EL0PTEN: u64 = 1 << 9; -const CNTHCTL_EL2_EL0PCTEN: u64 = 1 << 8; - -/* CPACR_EL1 Register */ -const CPACR_EL1_TTA_BIT_OFFSET: u64 = 28; -//const CPACR_EL1_TTA: u64 = 1 << CPACR_EL1_TTA_BIT_OFFSET; -const CPACR_EL1_FPEN_BITS_OFFSET: u64 = 20; -//const CPACR_EL1_FPEN: u64 = 0b11 << CPACR_EL1_FPEN_BITS_OFFSET; -const CPACR_EL1_ZEN_BITS_OFFSET: u64 = 16; -//const CPACR_EL1_ZEN: u64 = 0b11 << CPACR_EL1_ZEN_BITS_OFFSET; - -/* CPTR_EL2 Register */ -const CPTR_EL2_TTA_BIT_OFFSET_WITH_E2H: u64 = 28; -const CPTR_EL2_TTA_WITH_E2H: u64 = 1 << CPTR_EL2_TTA_BIT_OFFSET_WITH_E2H; -const CPTR_EL2_TTA_BIT_OFFSET_WITHOUT_E2H: u64 = 20; -const CPTR_EL2_TTA_WITHOUT_E2H: u64 = 1 << CPTR_EL2_TTA_BIT_OFFSET_WITHOUT_E2H; -const CPTR_EL2_FPEN_BITS_OFFSET: u64 = 20; -const CPTR_EL2_FPEN: u64 = 0b11 << CPTR_EL2_FPEN_BITS_OFFSET; -const CPTR_EL2_FPEN_NO_TRAP: u64 = 0b11 << CPTR_EL2_FPEN_BITS_OFFSET; -const CPTR_EL2_ZEN_BITS_OFFSET: u64 = 16; -const CPTR_EL2_ZEN: u64 = 0b11 << CPTR_EL2_ZEN_BITS_OFFSET; -const CPTR_EL2_ZEN_NO_TRAP: u64 = 0b11 << CPTR_EL2_ZEN_BITS_OFFSET; -//const CPTR_EL2_RES1: u64 = 0b11111111 | (1 << 9) | (0b11 << 12); - -/* TCR_EL2 Register */ -const TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H: u64 = 32; -const TCR_EL2_DS_WITHOUT_E2H: u64 = 1 << TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_TCMA_BIT_OFFSET_WITHOUT_E2H: u64 = 30; -const TCR_EL2_TCMA_WITHOUT_E2H: u64 = 1 << TCR_EL2_TCMA_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_TBID_BIT_OFFSET_WITHOUT_E2H: u64 = 29; -const TCR_EL2_TBID_WITHOUT_E2H: u64 = 1 << TCR_EL2_TBID_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_HWU_BITS_OFFSET_WITHOUT_E2H: u64 = 25; -const TCR_EL2_HWU_WITHOUT_E2H: u64 = 0b1111 << TCR_EL2_HWU_BITS_OFFSET_WITHOUT_E2H; -const TCR_EL2_HPD_BIT_OFFSET_WITHOUT_E2H: u64 = 24; -const TCR_EL2_HPD_WITHOUT_E2H: u64 = 1 << TCR_EL2_HPD_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_HD_BIT_OFFSET_WITHOUT_E2H: u64 = 22; -const TCR_EL2_HD_WITHOUT_E2H: u64 = 1 << TCR_EL2_HD_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_HA_BIT_OFFSET_WITHOUT_E2H: u64 = 21; -const TCR_EL2_HA_WITHOUT_E2H: u64 = 1 << TCR_EL2_HA_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_TBI_BIT_OFFSET_WITHOUT_E2H: u64 = 20; -const TCR_EL2_TBI_WITHOUT_E2H: u64 = 1 << TCR_EL2_TBI_BIT_OFFSET_WITHOUT_E2H; -const TCR_EL2_PS_BITS_OFFSET_WITHOUT_E2H: u64 = 16; -const TCR_EL2_PS_WITHOUT_E2H: u64 = 0b111 << TCR_EL2_PS_BITS_OFFSET_WITHOUT_E2H; -const TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H: u64 = 14; -const TCR_EL2_TG0_WITHOUT_E2H: u64 = 0b11 << TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H; -const TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H: u64 = 0; -const TCR_EL2_T0SZ_WITHOUT_E2H: u64 = 0b111111 << TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H; - -/* TCR_EL1 Register */ -const TCR_EL1_DS_BIT_OFFSET: u64 = 59; -//const TCR_EL1_DS: u64 = 1 << TCR_EL1_DS_BIT_OFFSET; -const TCR_EL1_TCMA0_BIT_OFFSET: u64 = 57; -//const TCR_EL1_TCMA0: u64 = 1 << TCR_EL1_TCMA0_BIT_OFFSET; -const TCR_EL1_TBID0_BIT_OFFSET: u64 = 51; -//const TCR_EL1_TBID0: u64 = 1 << TCR_EL1_TBID0_BIT_OFFSET; -const TCR_EL1_HWU_BITS_OFFSET: u64 = 43; -//const TCR_EL1_HWU: u64 = 0b11111111 << TCR_EL1_HWU_BITS_OFFSET; -const TCR_EL1_HPD0_BIT_OFFSET: u64 = 41; -//const TCR_EL1_HPD0: u64 = 1 << TCR_EL1_HPD0_BIT_OFFSET; -const TCR_EL1_HD_BIT_OFFSET: u64 = 40; -//const TCR_EL1_HD: u64 = 1 << TCR_EL1_HD_BIT_OFFSET; -const TCR_EL1_HA_BIT_OFFSET: u64 = 39; -//const TCR_EL1_HA: u64 = 1 << TCR_EL1_HA_BIT_OFFSET; -const TCR_EL1_TBI0_BIT_OFFSET: u64 = 37; -//const TCR_EL1_TBI0: u64 = 1 << TCR_EL1_TBI0_BIT_OFFSET; -const TCR_EL1_IPS_BITS_OFFSET: u64 = 32; -//const TCR_EL1_IPS: u64 = 0b111 << TCR_EL1_IPS_BITS_OFFSET; -const TCR_EL1_EPD1: u64 = 1 << 23; - -/* HCR_EL2 Register */ -const HCR_EL2_FIEN: u64 = 1 << 47; -const HCR_EL2_API: u64 = 1 << 41; -const HCR_EL2_APK: u64 = 1 << 40; -const HCR_EL2_TEA: u64 = 1 << 37; -const HCR_EL2_E2H: u64 = 1 << 34; -const HCR_EL2_RW: u64 = 1 << 31; -const HCR_EL2_TSC: u64 = 1 << 19; -const HCR_EL2_VM: u64 = 1 << 0; - #[no_mangle] extern "C" fn efi_main(image_handle: EfiHandle, system_table: *mut EfiSystemTable) { unsafe { @@ -154,7 +78,7 @@ extern "C" fn efi_main(image_handle: EfiHandle, system_table: *mut EfiSystemTabl paging::dump_page_table(); paging::setup_stage_2_translation().expect("Failed to setup Stage2 Paging"); - map_memory_pool(efi_boot_services); + map_memory_pool(); detect_acpi_and_dtb(system_table); @@ -162,7 +86,6 @@ extern "C" fn efi_main(image_handle: EfiHandle, system_table: *mut EfiSystemTabl if let Some(s) = &mut serial { let aligned_address = s.physical_address & PAGE_MASK; paging::map_address( - efi_boot_services, aligned_address, HYPERVISOR_SERIAL_BASE_ADDRESS, PAGE_SIZE, @@ -175,6 +98,12 @@ extern "C" fn efi_main(image_handle: EfiHandle, system_table: *mut EfiSystemTabl s.virtual_address = HYPERVISOR_SERIAL_BASE_ADDRESS + (s.physical_address - aligned_address); } + let ecam_info = if let Some(rsdp) = unsafe { ACPI_20_TABLE_ADDRESS } { + pci::detect_pci_space(rsdp) + } else { + None + }; + let stack_address = allocate_memory(STACK_PAGES).expect("Failed to alloc stack"); println!( "Stack for BSP: {:#X}", @@ -186,6 +115,7 @@ extern "C" fn efi_main(image_handle: EfiHandle, system_table: *mut EfiSystemTabl vbar_el2: 0, memory_pool: unsafe { &MEMORY_POOL }, serial_port: serial, + ecam_info, }; unsafe { (transmute::(entry_point))(&mut system_info) }; println!("Returned from the hypervisor"); @@ -262,10 +192,9 @@ fn init_memory_pool(b_s: *const EfiBootServices) { /// アクセスできるようにする。 /// /// また該当領域をVTTBR_EL2でダミーのページへのアクセスするように設定する。 -fn map_memory_pool(b_s: *const EfiBootServices) { +fn map_memory_pool() { let allocated_memory = unsafe { MEMORY_POOL.0[0].assume_init() }; paging::map_address( - b_s, allocated_memory, allocated_memory, ALLOC_SIZE, @@ -278,7 +207,7 @@ fn map_memory_pool(b_s: *const EfiBootServices) { /*paging::unmap_address_from_vttbr_el2(b_s, allocated_memory, ALLOC_SIZE) .expect("Failed to unmap allocated address.");*/ let dummy_page = allocate_memory(1).expect("Failed to alloc dummy page"); - paging::map_dummy_page_into_vttbr_el2(b_s, allocated_memory, ALLOC_SIZE, dummy_page) + paging::map_dummy_page_into_vttbr_el2(allocated_memory, ALLOC_SIZE, dummy_page) .expect("Failed to map dummy page"); } @@ -360,11 +289,12 @@ fn load_hypervisor(image_handle: EfiHandle, b_s: *const boot_service::EfiBootSer } /* Switch PageTable */ - let cloned_page_table = paging::copy_page_table(b_s); + let cloned_page_table = paging::copy_page_table(); unsafe { - ORIGINAL_PAGE_TABLE = paging::set_ttbr0_el2(cloned_page_table); - asm!("mrs {:x}, tcr_el2",out(reg) ORIGINAL_TCR_EL2) + ORIGINAL_PAGE_TABLE = get_ttbr0_el2() as usize; + ORIGINAL_TCR_EL2 = get_tcr_el2(); }; + set_ttbr0_el2(cloned_page_table as u64); println!( "Switched TTBR0_EL2 from {:#X} to {:#X}", unsafe { ORIGINAL_PAGE_TABLE }, @@ -419,7 +349,6 @@ fn load_hypervisor(image_handle: EfiHandle, b_s: *const boot_service::EfiBootSer ); } paging::map_address( - b_s, physical_base_address, info.virtual_base_address, pages << PAGE_SHIFT, @@ -537,7 +466,7 @@ fn set_up_el1() { }; /* TTBR0_EL1 */ - paging::set_ttbr0_el1(unsafe { ORIGINAL_PAGE_TABLE }); + set_ttbr0_el1(unsafe { ORIGINAL_PAGE_TABLE } as u64); /* TCR_EL1 */ if is_e2h_enabled { @@ -583,12 +512,8 @@ fn set_up_el1() { } /* HCR_EL2 */ - let hcr_el2: u64 = HCR_EL2_FIEN - | HCR_EL2_API - | HCR_EL2_APK - | HCR_EL2_RW - | HCR_EL2_TSC - | HCR_EL2_VM; + let hcr_el2: u64 = + HCR_EL2_FIEN | HCR_EL2_API | HCR_EL2_APK | HCR_EL2_RW | HCR_EL2_TSC | HCR_EL2_VM; unsafe { asm!("msr hcr_el2, {:x}",in(reg) hcr_el2); asm!("isb"); diff --git a/src/hypervisor_bootloader/src/paging.rs b/src/hypervisor_bootloader/src/paging.rs index c4c1faf..f7e96c3 100644 --- a/src/hypervisor_bootloader/src/paging.rs +++ b/src/hypervisor_bootloader/src/paging.rs @@ -1,121 +1,59 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Paging //! -use super::allocate_memory; - use super::{ - TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H, TCR_EL2_DS_WITHOUT_E2H, TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H, - TCR_EL2_TG0_WITHOUT_E2H, + allocate_memory, TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H, TCR_EL2_DS_WITHOUT_E2H, + TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_TG0_WITHOUT_E2H, }; use crate::{ TCR_EL2_PS_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_PS_WITHOUT_E2H, TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_T0SZ_WITHOUT_E2H, }; -use common::{bitmask, PAGE_SHIFT, PAGE_SIZE, STAGE_2_PAGE_SHIFT, STAGE_2_PAGE_SIZE}; -use uefi::boot_service::EfiBootServices; - -const STAGE_2_PAGE_ENTRY_ATTRIBUTE: u64 = - 1 << 10 /* AF bit */| - 0b11 << 8 /* SH bits (Inner sharable) */| - 0b11 << 6 /* S2AP bit (Read/Write) */ | - 0b1111 << 2 /* MemAttr(Write-back) */; - -const PAGE_DESCRIPTORS_UPPER_ATTRIBUTES_OFFSET: u64 = 50; -//const PAGE_DESCRIPTORS_CONTIGUOUS: u64 = 1 << 52; -const PAGE_DESCRIPTORS_NX_BIT_OFFSET: u64 = 54; - -const PAGE_DESCRIPTORS_NT: u64 = 1 << 16; -const PAGE_DESCRIPTORS_AF_BIT_OFFSET: u64 = 10; -const PAGE_DESCRIPTORS_AF: u64 = 1 << PAGE_DESCRIPTORS_AF_BIT_OFFSET; -const PAGE_DESCRIPTORS_SH_BITS_OFFSET: u64 = 8; -const PAGE_DESCRIPTORS_SH_INNER_SHAREABLE: u64 = 0b11 << PAGE_DESCRIPTORS_SH_BITS_OFFSET; -const PAGE_DESCRIPTORS_AP_BITS_OFFSET: u64 = 6; - -const MEMORY_PERMISSION_READABLE_BIT: u8 = 0; -const MEMORY_PERMISSION_WRITABLE_BIT: u8 = 1; -const MEMORY_PERMISSION_EXECUTABLE_BIT: u8 = 2; - -const VTCR_EL2_RES1: u64 = 1 << 31; -const VTCR_EL2_HWU_BITS_OFFSET: u64 = 25; -const VTCR_EL2_PS_BITS_OFFSET: u64 = 16; -const VTCR_EL2_TG0_BITS_OFFSET: u64 = 14; -const VTCR_EL2_SH0_BITS_OFFSET: u64 = 12; -const VTCR_EL2_ORG0_BITS_OFFSET: u64 = 10; -const VTCR_EL2_IRG0_BITS_OFFSET: u64 = 8; -const VTCR_EL2_SL0_BITS_OFFSET: u64 = 6; -const VTCR_EL2_SL0: u64 = 0b11 << VTCR_EL2_SL0_BITS_OFFSET; -const VTCR_EL2_T0SZ_BITS_OFFSET: u64 = 0; -const VTCR_EL2_T0SZ: u64 = 0b111111 << VTCR_EL2_SL0_BITS_OFFSET; - -#[derive(Copy, Clone)] -struct TTBR(u64); /* Translation Table Base Register */ - -impl TTBR { - const ASID: u64 = (u16::MAX as u64) << 48; - const CNP: u64 = 1 << 0; - - pub fn new(ttbr: u64) -> Self { - Self(ttbr) - } - - pub fn get_base_address(&self) -> usize { - ((self.0) & !(Self::CNP | Self::ASID)) as usize - } -} -const fn extract_output_address(descriptor: u64) -> usize { - (descriptor - & bitmask!( - PAGE_DESCRIPTORS_UPPER_ATTRIBUTES_OFFSET - 1, - PAGE_SHIFT as u64 - )) as usize -} +use common::cpu::{ + flush_tlb_el2, get_id_aa64mmfr0_el1, get_mair_el2, get_tcr_el2, get_ttbr0_el2, get_vtcr_el2, + get_vttbr_el2, set_tcr_el2, set_vtcr_el2, set_vttbr_el2, VTCR_EL2_HWU_BITS_OFFSET, + VTCR_EL2_IRG0_BITS_OFFSET, VTCR_EL2_ORG0_BITS_OFFSET, VTCR_EL2_PS_BITS_OFFSET, VTCR_EL2_RES1, + VTCR_EL2_SH0_BITS_OFFSET, VTCR_EL2_SL0, VTCR_EL2_SL0_BITS_OFFSET, VTCR_EL2_T0SZ, + VTCR_EL2_T0SZ_BITS_OFFSET, VTCR_EL2_TG0_BITS_OFFSET, +}; +use common::paging::{ + calculate_number_of_concatenated_page_tables, create_attributes_for_stage_1, + create_attributes_for_stage_2, extract_output_address, + get_initial_page_table_level_and_bits_to_shift, + get_suitable_memory_attribute_index_from_mair_el2, is_block_descriptor, + is_descriptor_table_or_level_3_descriptor, table_level_to_table_shift, + MEMORY_PERMISSION_EXECUTABLE_BIT, MEMORY_PERMISSION_READABLE_BIT, + MEMORY_PERMISSION_WRITABLE_BIT, PAGE_DESCRIPTORS_CONTIGUOUS, PAGE_DESCRIPTORS_NT, + PAGE_TABLE_SIZE, TTBR, +}; +use common::{PAGE_SHIFT, PAGE_SIZE, STAGE_2_PAGE_SHIFT, STAGE_2_PAGE_SIZE}; -const fn is_descriptor_table_or_level_3_descriptor(descriptor: u64) -> bool { - descriptor & 0b11 == 0b11 -} -const fn is_block_descriptor(descriptor: u64) -> bool { - descriptor & 0b11 == 0b01 -} +fn _copy_page_table(table_address: usize, current_level: i8) -> usize { + let cloned_table_address = allocate_memory(1).expect("Failed to allocate page table"); -pub fn set_ttbr0_el1(address: usize) { - unsafe { - asm!(" - msr ttbr0_el1, {:x} - ", in(reg) address) + let cloned_table = unsafe { + &mut *(cloned_table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) }; -} - -pub fn set_ttbr0_el2(address: usize) -> usize { - let mut old: usize; unsafe { - asm!(" - mrs {:x}, ttbr0_el2 - msr ttbr0_el2, {:x} - dsb ishst - tlbi alle2 - dsb ish - isb - ", out(reg) old, in(reg) address) - }; - return old; -} - -fn _copy_page_table(b_s: *const EfiBootServices, table_address: usize, current_level: i8) -> usize { - let cloned_table_address = allocate_memory(1).expect("Failed to allocate page table"); - unsafe { ((*b_s).copy_mem)(cloned_table_address, table_address, PAGE_SIZE) }; - let cloned_table = unsafe { - &mut *(cloned_table_address as *mut [u64; PAGE_SIZE / core::mem::size_of::()]) + *cloned_table = + *(table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) }; if current_level == 3 { return cloned_table_address; } for e in cloned_table { if is_descriptor_table_or_level_3_descriptor(*e) { - let next_level_table_address = extract_output_address(*e); + let next_level_table_address = extract_output_address(*e, PAGE_SHIFT); *e = ((*e) & !(next_level_table_address as u64)) - | (_copy_page_table(b_s, next_level_table_address, current_level + 1) as u64); + | (_copy_page_table(next_level_table_address, current_level + 1) as u64); } } return cloned_table_address; @@ -124,61 +62,42 @@ fn _copy_page_table(b_s: *const EfiBootServices, table_address: usize, current_l /// Copy TTBR0_EL2 for Hypervisor /// /// ハイパーバイザー向けのページテーブルを複製する。 -pub fn copy_page_table(b_s: *const EfiBootServices) -> usize /*cloned page table address */ -{ - let mut page_table_address: usize; - unsafe { asm!("mrs {:x}, ttbr0_el2",out(reg) page_table_address) }; - page_table_address = TTBR::new(page_table_address as u64).get_base_address(); - let tcr_el2: u64; - unsafe { asm!("mrs {:x}, tcr_el2", out(reg) tcr_el2) }; +/// +/// # Return Value +/// Cloned Page Table Address +pub fn copy_page_table() -> usize { + let page_table_address = TTBR::new(get_ttbr0_el2()).get_base_address(); + let tcr_el2 = get_tcr_el2(); let first_table_level = get_initial_page_table_level_and_bits_to_shift(tcr_el2).0; - /* 4段階の場合は0番目のエントリーの次段階テーブルもコピーしておく */ - return _copy_page_table(b_s, page_table_address, first_table_level); + return _copy_page_table(page_table_address, first_table_level); } /// Map physical Address Recursively /// /// permission: Bit0:Readable, Bit1: Writable, Bit2: Executable fn map_address_recursive( - b_s: *const EfiBootServices, physical_address: &mut usize, virtual_address: &mut usize, num_of_remaining_pages: &mut usize, table_address: usize, table_level: i8, - shift_level: usize, permission: u8, memory_attribute: u8, /* MemAttr */ t0sz: u8, ) -> Result<(), ()> { + let shift_level = table_level_to_table_shift(PAGE_SHIFT, table_level); + let mut table_index = (*virtual_address >> shift_level) & 0x1FF; + if table_level == 3 { - assert_eq!(shift_level, PAGE_SHIFT); - let current_table = - unsafe { &mut *(table_address as *mut [u64; PAGE_SIZE / core::mem::size_of::()]) }; - let table_index = (*virtual_address >> shift_level) & 0x1FF; + let current_table = unsafe { + &mut *(table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; let num_of_pages = if *num_of_remaining_pages + table_index > 512 { 512 - table_index } else { *num_of_remaining_pages }; - let nx_bit: u64 = if (permission & (1 << MEMORY_PERMISSION_EXECUTABLE_BIT)) != 0 { - 0 - } else { - 1 - } << PAGE_DESCRIPTORS_NX_BIT_OFFSET; - let access_permission: u64 = if (permission & (1 << MEMORY_PERMISSION_WRITABLE_BIT)) != 0 { - 0b00 - } else { - 0b10 - } << PAGE_DESCRIPTORS_AP_BITS_OFFSET; - - let attributes = nx_bit - | PAGE_DESCRIPTORS_AF - | PAGE_DESCRIPTORS_SH_INNER_SHAREABLE - | nx_bit - | access_permission - | (memory_attribute << 2) as u64 - | 0b11; + let attributes = create_attributes_for_stage_1(permission, memory_attribute, false); for index in table_index..(table_index + num_of_pages) { current_table[index] = *physical_address as u64 | attributes; @@ -188,11 +107,12 @@ fn map_address_recursive( *num_of_remaining_pages -= num_of_pages; return Ok(()); } - let current_table = - unsafe { &mut *(table_address as *mut [u64; PAGE_SIZE / core::mem::size_of::()]) }; - let mut table_index = (*virtual_address >> shift_level) & 0x1FF; + let current_table = unsafe { + &mut *(table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; while *num_of_remaining_pages != 0 { + #[cfg(debug_assertions)] println!( "{:#X}: Level{}'s Table Index: {:#X}", *virtual_address, table_level, table_index @@ -201,60 +121,95 @@ fn map_address_recursive( break; } let target_descriptor = &mut current_table[table_index]; - if !is_descriptor_table_or_level_3_descriptor(*target_descriptor) { - let allocated_table_address = - allocate_page_table_for_stage_1(table_level, t0sz, false)?; - if is_block_descriptor(*target_descriptor) { - #[cfg(debug_assertions)] + if table_level > 1 + && (*physical_address & ((1usize << shift_level) - 1)) == 0 + && (*virtual_address & ((1usize << shift_level) - 1)) == 0 + && *num_of_remaining_pages >= 512usize.pow((3 - table_level) as u32) + { + println!( + "Creating BlockEntry: VA: {:#X}, PA: {:#X}, TableLevel: {}", + *virtual_address, *physical_address, table_level + ); + if is_descriptor_table_or_level_3_descriptor(*target_descriptor) { println!( - "Convert the block descriptor({:#b}) to table descriptor", - *target_descriptor + "PageTable:({:#X}) will be deleted.", + extract_output_address(*target_descriptor, PAGE_SHIFT) ); + /* TODO: free page table */ + } + let attributes = create_attributes_for_stage_1(permission, memory_attribute, true); + *target_descriptor = *physical_address as u64 | attributes; + *physical_address += 1 << shift_level; + *virtual_address += 1 << shift_level; + *num_of_remaining_pages -= 512usize.pow((3 - table_level) as u32); + } else { + let mut created_entry: Option = None; - let mut block_physical_address = extract_output_address(*target_descriptor); - let mut descriptor_attribute = *target_descriptor ^ (block_physical_address as u64); - let next_level_page = unsafe { - &mut *(allocated_table_address - as *mut [u64; PAGE_SIZE / core::mem::size_of::()]) - }; + if !is_descriptor_table_or_level_3_descriptor(*target_descriptor) { + let allocated_table_address = + allocate_page_table_for_stage_1(table_level, t0sz, false)?; - if table_level + 1 == 3 { - descriptor_attribute |= 0b11; - descriptor_attribute &= !PAGE_DESCRIPTORS_NT; - } + if is_block_descriptor(*target_descriptor) { + #[cfg(debug_assertions)] + println!( + "Convert the block descriptor({:#b}) to table descriptor", + *target_descriptor + ); - for e in next_level_page { - *e = (block_physical_address as u64) | descriptor_attribute; - block_physical_address += 1 << (shift_level - 9); + let mut block_physical_address = + extract_output_address(*target_descriptor, PAGE_SHIFT); + let mut descriptor_attribute = + *target_descriptor ^ (block_physical_address as u64); + let next_level_page = unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + + if table_level + 1 == 3 { + descriptor_attribute |= 0b11; + descriptor_attribute &= !PAGE_DESCRIPTORS_NT; + } + + for e in next_level_page { + *e = (block_physical_address as u64) | descriptor_attribute; + block_physical_address += 1 << (shift_level - 9); + } + } else { + /* set_mem */ + for e in unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + } { + *e = 0; + } } - } else { - unsafe { ((*b_s).set_mem)(allocated_table_address, PAGE_SIZE, 0) }; - } - /* TODO: 52bit OA support */ - *target_descriptor = allocated_table_address as u64 | 0b11; - println!("Allocated: {:#X}", allocated_table_address); + /* TODO: 52bit OA support */ + created_entry = Some(allocated_table_address as u64 | 0b11); + println!("Allocated: {:#X}", allocated_table_address); + } + map_address_recursive( + physical_address, + virtual_address, + num_of_remaining_pages, + extract_output_address(created_entry.unwrap_or(*target_descriptor), PAGE_SHIFT), + table_level + 1, + permission, + memory_attribute, + t0sz, + )?; + + if let Some(new_descriptor) = created_entry { + *target_descriptor = new_descriptor; + } } - map_address_recursive( - b_s, - physical_address, - virtual_address, - num_of_remaining_pages, - extract_output_address(*target_descriptor), - table_level + 1, - shift_level - 9, - permission, - memory_attribute, - t0sz, - )?; table_index += 1; } return Ok(()); } pub fn map_address( - b_s: *const EfiBootServices, mut physical_address: usize, mut virtual_address: usize, size: usize, @@ -273,11 +228,8 @@ pub fn map_address( size }; let mut num_of_needed_pages = aligned_size >> PAGE_SHIFT; - let mut page_table: u64; - let tcr_el2: u64; - unsafe { - asm!("mrs {:x}, tcr_el2", out(reg) tcr_el2); - } + let tcr_el2 = get_tcr_el2(); + let mut tcr_el2_t0sz = ((tcr_el2 & TCR_EL2_T0SZ_WITHOUT_E2H) >> TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H) as u32; @@ -301,25 +253,17 @@ pub fn map_address( ); /* TODO: adjust process */ } else { - unsafe { - asm!("msr tcr_el2, {:x}", in(reg) new_tcr_el2); - } + set_tcr_el2(new_tcr_el2); tcr_el2_t0sz = min_t0sz; } } - unsafe { - asm!("mrs {:x}, ttbr0_el2",out(reg) page_table); - } - map_address_recursive( - b_s, &mut physical_address, &mut virtual_address, &mut num_of_needed_pages, - TTBR(page_table).get_base_address(), + TTBR::new(get_ttbr0_el2()).get_base_address(), table_level, - shift_level, (readable as u8) << MEMORY_PERMISSION_READABLE_BIT | (writable as u8) << MEMORY_PERMISSION_WRITABLE_BIT | (executable as u8) << MEMORY_PERMISSION_EXECUTABLE_BIT, @@ -334,14 +278,7 @@ pub fn map_address( ); return Err(()); } - unsafe { - asm!( - " - tlbi alle2 - dsb sy - isb" - ) - }; + flush_tlb_el2(); println!( "Mapped {:#X} Bytes({} Pages)", aligned_size, @@ -354,44 +291,58 @@ pub fn map_address( /// /// permission: Bit0:Readable, Bit1: Writable, Bit2: Executable fn map_address_recursive_stage2( - b_s: *const EfiBootServices, physical_address: &mut usize, virtual_address: &mut usize, num_of_remaining_pages: &mut usize, table_address: usize, table_level: i8, - shift_level: usize, is_unmap: bool, + permission: u8, concatenated_tables: u8, t0sz: u8, is_dummy_page: bool, ) -> Result<(), ()> { + let shift_level = table_level_to_table_shift(STAGE_2_PAGE_SHIFT, table_level); + let mut table_index = + (*virtual_address >> shift_level) & (0x200 * (concatenated_tables as usize) - 1); + if table_level == 3 { - assert_eq!(shift_level, STAGE_2_PAGE_SHIFT); - assert_eq!(concatenated_tables, 1); - let current_table = unsafe { - &mut *(table_address as *mut [u64; STAGE_2_PAGE_SIZE / core::mem::size_of::()]) - }; - let table_index = (*virtual_address >> shift_level) & 0x1FF; - let num_of_pages = if *num_of_remaining_pages + table_index > 512 { - 512 - table_index + let table_len = + (PAGE_TABLE_SIZE * (concatenated_tables as usize)) / core::mem::size_of::(); + + let current_table = + unsafe { core::slice::from_raw_parts_mut(table_address as *mut u64, table_len) }; + + let num_of_pages = if *num_of_remaining_pages + table_index > table_len { + table_len - table_index } else { *num_of_remaining_pages }; - let attributes: u64 = PAGE_DESCRIPTORS_AF - | if is_dummy_page { - 0 + if STAGE_2_PAGE_SIZE == 0x1000 { + let contiguous_first_entry = &mut current_table[table_index & !0xF]; + *contiguous_first_entry &= !PAGE_DESCRIPTORS_CONTIGUOUS; + } + let attributes = create_attributes_for_stage_2(permission, is_dummy_page, is_unmap, false); + let end_index = table_index + num_of_pages; + for index in table_index..end_index { + if STAGE_2_PAGE_SIZE == 0x1000 + && (index & 0xF) == 0 + && !is_dummy_page + && (end_index - index) >= 16 + { + println!( + "Enable CONTIGUOUS_BIT(index: {:#X}, end_index: {:#X}", + index, end_index + ); + current_table[index] = + *physical_address as u64 | attributes | PAGE_DESCRIPTORS_CONTIGUOUS; } else { - 0 /*PAGE_DESCRIPTORS_CONTIGUOUS*/ + current_table[index] = *physical_address as u64 | attributes; } - | STAGE_2_PAGE_ENTRY_ATTRIBUTE - | if is_unmap { 0b00 } else { 0b11 }; - for index in table_index..(table_index + num_of_pages) { - current_table[index] = *physical_address as u64 | attributes; if !is_dummy_page { - *physical_address += PAGE_SIZE; + *physical_address += STAGE_2_PAGE_SIZE; } - *virtual_address += PAGE_SIZE; + *virtual_address += STAGE_2_PAGE_SIZE; } *num_of_remaining_pages -= num_of_pages; return Ok(()); @@ -399,13 +350,12 @@ fn map_address_recursive_stage2( let current_table = unsafe { core::slice::from_raw_parts_mut( table_address as *mut u64, - (STAGE_2_PAGE_SIZE * concatenated_tables as usize) / core::mem::size_of::(), + (PAGE_TABLE_SIZE * concatenated_tables as usize) / core::mem::size_of::(), ) }; - let mut table_index = - (*virtual_address >> shift_level) & ((0x200 * concatenated_tables as usize) - 1); while *num_of_remaining_pages != 0 { + #[cfg(debug_assertions)] println!( "{:#X}: Level{}'s Table Index: {:#X}", *virtual_address, table_level, table_index @@ -414,62 +364,102 @@ fn map_address_recursive_stage2( break; } let target_descriptor = &mut current_table[table_index]; - if *target_descriptor & 0b11 != 0b11 { - let allocated_table_address = - allocate_page_table_for_stage_2(table_level, t0sz, false, 1)?; - - if *target_descriptor & 0b11 == 0b01 { - #[cfg(debug_assertions)] + if !is_dummy_page + && table_level > 1 + && (*physical_address & ((1usize << shift_level) - 1)) == 0 + && (*virtual_address & ((1usize << shift_level) - 1)) == 0 + && *num_of_remaining_pages >= 512usize.pow((3 - table_level) as u32) + { + println!( + "Creating BlockEntry: VA: {:#X}, PA: {:#X}, TableLevel: {}(Stage 2)", + *virtual_address, *physical_address, table_level + ); + if is_descriptor_table_or_level_3_descriptor(*target_descriptor) { println!( - "Convert the block descriptor({:#b}) to table descriptor", - *target_descriptor + "PageTable:({:#X}) will be deleted.", + extract_output_address(*target_descriptor, STAGE_2_PAGE_SHIFT) ); + /* TODO: free page table */ + } + let attributes = + create_attributes_for_stage_2(permission, is_dummy_page, is_unmap, true); - let mut block_physical_address = extract_output_address(*target_descriptor); - let mut descriptor_attribute = *target_descriptor ^ (block_physical_address as u64); - let next_level_page = unsafe { - &mut *(allocated_table_address - as *mut [u64; PAGE_SIZE / core::mem::size_of::()]) - }; + *target_descriptor = *physical_address as u64 | attributes; - if table_level + 1 == 3 { - descriptor_attribute |= 0b11; - descriptor_attribute &= !PAGE_DESCRIPTORS_NT; /* Clear nT Bit */ - } + *physical_address += 1 << shift_level; + *virtual_address += 1 << shift_level; + *num_of_remaining_pages -= 512usize.pow((3 - table_level) as u32); + } else { + let mut created_entry: Option = None; + + if *target_descriptor & 0b11 != 0b11 { + let allocated_table_address = + allocate_page_table_for_stage_2(table_level, t0sz, false, 1)?; + + if *target_descriptor & 0b11 == 0b01 { + #[cfg(debug_assertions)] + println!( + "Convert the block descriptor({:#b}) to table descriptor", + *target_descriptor + ); - for e in next_level_page { - *e = (block_physical_address as u64) | descriptor_attribute; - block_physical_address += 1 << (shift_level - 9); + let mut block_physical_address = + extract_output_address(*target_descriptor, STAGE_2_PAGE_SHIFT); + let mut descriptor_attribute = + *target_descriptor ^ (block_physical_address as u64); + let next_level_page = unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + + if table_level + 1 == 3 { + descriptor_attribute |= 0b11; + descriptor_attribute &= !PAGE_DESCRIPTORS_NT; /* Clear nT Bit */ + } + + for e in next_level_page { + *e = (block_physical_address as u64) | descriptor_attribute; + block_physical_address += 1 << (shift_level - 9); + } + } else { + /* set_mem */ + for e in unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + } { + *e = 0; + } } - } else { - unsafe { ((*b_s).set_mem)(allocated_table_address, PAGE_SIZE, 0) }; - } - /* TODO: 52bit OA support */ - *target_descriptor = allocated_table_address as u64 | 0b11; - println!("Allocated: {:#X}", allocated_table_address); + /* TODO: 52bit OA support */ + created_entry = Some(allocated_table_address as u64 | 0b11); + println!("Allocated: {:#X}", allocated_table_address); + } + map_address_recursive_stage2( + physical_address, + virtual_address, + num_of_remaining_pages, + extract_output_address( + created_entry.unwrap_or(*target_descriptor), + STAGE_2_PAGE_SHIFT, + ), + table_level + 1, + is_unmap, + permission, + 1, + t0sz, + is_dummy_page, + )?; + if let Some(d) = created_entry { + *target_descriptor = d; + } } - map_address_recursive_stage2( - b_s, - physical_address, - virtual_address, - num_of_remaining_pages, - extract_output_address(*target_descriptor), - table_level + 1, - shift_level - 9, - is_unmap, - 1, - t0sz, - is_dummy_page, - )?; table_index += 1; } - unsafe { asm!("TLBI VMALLS12E1") }; return Ok(()); } pub fn map_dummy_page_into_vttbr_el2( - b_s: *const EfiBootServices, mut virtual_address: usize, size: usize, mut dummy_page: usize, /*4 KiB Page Physical Address*/ @@ -479,8 +469,7 @@ pub fn map_dummy_page_into_vttbr_el2( return Err(()); } let mut num_of_needed_pages = size >> STAGE_2_PAGE_SHIFT; - let vtcr_el2: u64; - unsafe { asm!("mrs {:x}, vtcr_el2",out(reg) vtcr_el2) }; + let vtcr_el2 = get_vtcr_el2(); let vtcr_el2_sl0 = ((vtcr_el2 & VTCR_EL2_SL0) >> VTCR_EL2_SL0_BITS_OFFSET) as u8; let vtcr_el2_t0sz = ((vtcr_el2 & VTCR_EL2_T0SZ) >> VTCR_EL2_T0SZ_BITS_OFFSET) as u8; let initial_look_up_level: i8 = match vtcr_el2_sl0 { @@ -490,23 +479,18 @@ pub fn map_dummy_page_into_vttbr_el2( 0b11 => 3, _ => unreachable!(), }; - let shift_level = STAGE_2_PAGE_SHIFT + 9 * ((3 - initial_look_up_level) as usize); - let vttbr_el2: u64; - - unsafe { - asm!("mrs {:x}, vttbr_el2",out(reg) vttbr_el2); - } let original_dummy_page = dummy_page; map_address_recursive_stage2( - b_s, &mut dummy_page, &mut virtual_address, &mut num_of_needed_pages, - TTBR::new(vttbr_el2).get_base_address(), + TTBR::new(get_vttbr_el2()).get_base_address(), initial_look_up_level, - shift_level, false, + (1 << MEMORY_PERMISSION_READABLE_BIT) + | (1 << MEMORY_PERMISSION_WRITABLE_BIT) + | (1 << MEMORY_PERMISSION_EXECUTABLE_BIT), calculate_number_of_concatenated_page_tables(vtcr_el2_t0sz, initial_look_up_level), vtcr_el2_t0sz, true, @@ -517,57 +501,6 @@ pub fn map_dummy_page_into_vttbr_el2( return Ok(()); } -/// VTTBR_EL2の該当アドレス範囲を全てInvalid Entryにする。 -/// 現在はInitial Table Levelは0までのみの対応 -#[allow(dead_code)] -pub fn unmap_address_from_vttbr_el2( - b_s: *const EfiBootServices, - mut address: usize, - size: usize, -) -> Result<(), ()> { - if (size & ((1usize << STAGE_2_PAGE_SHIFT) - 1)) != 0 { - println!("Size({:#X}) is not aligned.", size); - return Err(()); - } - let mut num_of_needed_pages = size >> STAGE_2_PAGE_SHIFT; - let vtcr_el2: u64; - unsafe { asm!("mrs {:x}, vtcr_el2",out(reg) vtcr_el2) }; - let vtcr_el2_sl0 = ((vtcr_el2 & VTCR_EL2_SL0) >> VTCR_EL2_SL0_BITS_OFFSET) as u8; - let vtcr_el2_t0sz = ((vtcr_el2 & VTCR_EL2_T0SZ) >> VTCR_EL2_T0SZ_BITS_OFFSET) as u8; - let initial_look_up_level: i8 = match vtcr_el2_sl0 { - 0b00 => 2, - 0b01 => 1, - 0b10 => 0, - 0b11 => 3, - _ => unreachable!(), - }; - let shift_level = STAGE_2_PAGE_SHIFT + 9 * ((3 - initial_look_up_level) as usize); - let vttbr_el2: u64; - - assert!(address < (1 << (64 - vtcr_el2_t0sz))); - - unsafe { - asm!("mrs {:x}, vttbr_el2",out(reg) vttbr_el2); - } - - map_address_recursive_stage2( - b_s, - &mut 0, - &mut address, - &mut num_of_needed_pages, - TTBR::new(vttbr_el2).get_base_address(), - initial_look_up_level, - shift_level, - true, - calculate_number_of_concatenated_page_tables(vtcr_el2_t0sz, initial_look_up_level), - vtcr_el2_t0sz, - false, - )?; - assert_eq!(num_of_needed_pages, 0); - println!("Unmapped {:#X} Bytes({} Pages)", size, size >> PAGE_SHIFT); - return Ok(()); -} - pub fn setup_stage_2_translation() -> Result<(), ()> { const FIRST_LEVEL: i8 = 1; let top_shift_level: u8 = 12 + 9 * (3 - FIRST_LEVEL) as u8; @@ -576,8 +509,7 @@ pub fn setup_stage_2_translation() -> Result<(), ()> { let t0sz: u8 = 24; /* 2^(64 - 24) = 1TiB */ let number_of_tables = calculate_number_of_concatenated_page_tables(t0sz, FIRST_LEVEL) as usize; - let id_aa64mmfr0_el1: u64; - unsafe { asm!("mrs {:x}, id_aa64mmfr0_el1",out(reg) id_aa64mmfr0_el1) }; + let id_aa64mmfr0_el1 = get_id_aa64mmfr0_el1(); assert!((id_aa64mmfr0_el1 & 0b1111) as u8 >= ps); let mut physical_address = 0; @@ -586,7 +518,7 @@ pub fn setup_stage_2_translation() -> Result<(), ()> { let top_level_page_table = unsafe { core::slice::from_raw_parts_mut( table_address as *mut u64, - (STAGE_2_PAGE_SIZE * number_of_tables) / core::mem::size_of::(), + (PAGE_TABLE_SIZE * number_of_tables) / core::mem::size_of::(), ) }; @@ -595,10 +527,18 @@ pub fn setup_stage_2_translation() -> Result<(), ()> { allocate_page_table_for_stage_2(FIRST_LEVEL + 1, t0sz, false, 1)?; let second_level_page_table = unsafe { &mut *(second_table_address - as *mut [u64; STAGE_2_PAGE_SIZE / core::mem::size_of::()]) + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) }; + let attribute = create_attributes_for_stage_2( + (1 << MEMORY_PERMISSION_EXECUTABLE_BIT) + | (1 << MEMORY_PERMISSION_WRITABLE_BIT) + | (1 << MEMORY_PERMISSION_READABLE_BIT), + false, + false, + true, + ); for e2 in second_level_page_table { - *e2 = physical_address /* | PAGE_DESCRIPTORS_CONTIGUOUS */ | STAGE_2_PAGE_ENTRY_ATTRIBUTE | 0b01; + *e2 = physical_address | attribute; physical_address += 1 << (top_shift_level - 9); } *e = (second_table_address as u64) | 0b11; @@ -616,17 +556,9 @@ pub fn setup_stage_2_translation() -> Result<(), ()> { ((sl0 as u64) << VTCR_EL2_SL0_BITS_OFFSET) | ((t0sz as u64) << VTCR_EL2_T0SZ_BITS_OFFSET); - unsafe { asm!("msr vtcr_el2, {:x}",in(reg) vtcr_el2) }; - unsafe { asm!("msr vttbr_el2, {:x}",in(reg) table_address) }; - unsafe { - asm!( - " - dsb ishst - tlbi alle2 - dsb ish - isb" - ) - }; + set_vtcr_el2(vtcr_el2); + set_vttbr_el2(table_address as u64); + return Ok(()); } @@ -666,7 +598,7 @@ pub fn dump_page_table_recursive( virtual_base_address, *virtual_base_address + granule, level3_descriptor, - extract_output_address(level3_descriptor), + extract_output_address(level3_descriptor, PAGE_SHIFT), (level3_descriptor >> 2) & 0b111 ); } @@ -691,18 +623,18 @@ pub fn dump_page_table_recursive( virtual_base_address, *virtual_base_address + granule, descriptor, - extract_output_address(descriptor), + extract_output_address(descriptor, PAGE_SHIFT), (descriptor >> 2) & 0b111 ); *virtual_base_address += granule; } else { - let next_level_table = extract_output_address(descriptor); + let next_level_table = extract_output_address(descriptor, PAGE_SHIFT); println!( "{:#X} ~ {:#X}: Table: {:#b} (OA: {:#X})", virtual_base_address, *virtual_base_address + granule, descriptor, - extract_output_address(descriptor) + extract_output_address(descriptor, PAGE_SHIFT) ); dump_page_table_recursive( next_level_table, @@ -719,10 +651,7 @@ pub fn dump_page_table_recursive( #[allow(dead_code)] pub fn dump_page_table() { - let tcr_el2: u64; - unsafe { - asm!("mrs {:x}, tcr_el2", out(reg) tcr_el2); - }; + let tcr_el2 = get_tcr_el2(); let tcr_el2_ds = ((tcr_el2 & TCR_EL2_DS_WITHOUT_E2H) >> TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H) as u8; let tcr_el2_tg0 = @@ -800,8 +729,7 @@ pub fn dump_page_table() { } drop(current_level); - let mair_el2: u64; - unsafe { asm!("mrs {:x}, mair_el2",out(reg) mair_el2) }; + let mair_el2 = get_mair_el2(); println!( "MAIR: {:#X}(Using MemAttr: {})", @@ -809,11 +737,7 @@ pub fn dump_page_table() { get_suitable_memory_attribute_index_from_mair_el2(false) ); - let mut page_table_address: usize; - unsafe { - asm!("mrs {:x}, ttbr0_el2",out(reg) page_table_address); - } - page_table_address = TTBR::new(page_table_address as u64).get_base_address(); + let page_table_address = TTBR::new(get_ttbr0_el2()).get_base_address(); println!("PageTable: {:#X}", page_table_address); @@ -826,40 +750,6 @@ pub fn dump_page_table() { ); } -/// 現時点ではTTBR0_EL2のみ対応 -pub fn get_initial_page_table_level_and_bits_to_shift(tcr_el2: u64) -> (i8, usize) { - let tcr_el2_ds = - ((tcr_el2 & TCR_EL2_DS_WITHOUT_E2H) >> TCR_EL2_DS_BIT_OFFSET_WITHOUT_E2H) as u8; - let tcr_el2_tg0 = (tcr_el2 & TCR_EL2_TG0_WITHOUT_E2H) >> TCR_EL2_TG0_BITS_OFFSET_WITHOUT_E2H; - let tcr_el2_t0sz = - ((tcr_el2 & TCR_EL2_T0SZ_WITHOUT_E2H) >> TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H) as usize; - let page_shift = 12 + (tcr_el2_tg0 << 1) as usize; - - /* aarch64/translation/walk/AArch64.TranslationTableWalk (J1-7982) */ - let first_level = 4 - - (1 + ((64 - tcr_el2_t0sz /* TTBR1_EL2ではここが t1sz(TCR_EL2[21:16] */ - page_shift - 1) - / (page_shift - 3))) as i8; - - if tcr_el2_ds == 0 && first_level == -1 { - println!("5-Level Paging with DS == 0 is invalid."); - } - (first_level, page_shift + 9 * (3 - first_level) as usize) -} - -fn get_suitable_memory_attribute_index_from_mair_el2(is_device: bool) -> u8 { - let mut mair_el2: u64; - let suitable_attribute: u64 = if is_device { 0x00 } else { 0xff }; - unsafe { asm!("mrs {:x},mair_el2",out(reg) mair_el2) }; - for index in 0..7 { - let attribute = mair_el2 & 0xff; - if attribute == suitable_attribute { - return index; - } - mair_el2 >>= 8; - } - panic!("Attr=={:#X} is not found...", suitable_attribute); -} - /// Allocate page table for stage 1 with suitable address alignment #[inline(always)] fn allocate_page_table_for_stage_1( @@ -932,12 +822,3 @@ fn allocate_page_table_for_stage_2( }; } } - -#[inline(always)] -const fn calculate_number_of_concatenated_page_tables(t0sz: u8, initial_lookup_level: i8) -> u8 { - if t0sz > (43 - ((3 - initial_lookup_level) as u8) * 9) { - 1 - } else { - 2u8.pow(((43 - ((3 - initial_lookup_level) as u8) * 9) - t0sz) as u32) - } -} diff --git a/src/hypervisor_bootloader/src/panic.rs b/src/hypervisor_bootloader/src/panic.rs index 402c74a..0ec2e23 100644 --- a/src/hypervisor_bootloader/src/panic.rs +++ b/src/hypervisor_bootloader/src/panic.rs @@ -1,9 +1,17 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Panic Handler //! use crate::console::DEFAULT_CONSOLE; +use common::cpu::halt_loop; + use core::panic; #[panic_handler] @@ -23,7 +31,5 @@ pub fn panic(info: &panic::PanicInfo) -> ! { ); } - loop { - unsafe { asm!("wfi") } - } + halt_loop() } diff --git a/src/hypervisor_bootloader/src/pci.rs b/src/hypervisor_bootloader/src/pci.rs new file mode 100644 index 0000000..8ea6ce6 --- /dev/null +++ b/src/hypervisor_bootloader/src/pci.rs @@ -0,0 +1,43 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +use crate::paging::map_address; + +use common::acpi::get_acpi_table; +use common::EcamInfo; + +pub fn detect_pci_space(rsdp: usize) -> Option { + let mcfg = get_acpi_table(rsdp, b"MCFG"); + if let Err(e) = mcfg { + println!("Failed to get MCFG table: {:?}", e); + return None; + } + let mcfg = mcfg.unwrap(); + /* Currently, supporting only one ECAM Address */ + let pcie_ecam_address = unsafe { *((mcfg + 44) as *const u64) } as usize; + let start_bus_number = unsafe { *((mcfg + 54) as *const u8) }; + let end_bus_number = unsafe { *((mcfg + 55) as *const u8) }; + println!( + "ECAM Address: {:#X}, Start: {:#X}, End: {:#X}", + pcie_ecam_address, start_bus_number, end_bus_number + ); + assert_eq!(pcie_ecam_address & (8 * 1024 * 1024 - 1), 0); + map_address( + pcie_ecam_address, + pcie_ecam_address, + ((1 + end_bus_number as usize) << 20) - 1, + true, + true, + false, + true, + ) + .expect("Failed to map ECAM Space"); + return Some(EcamInfo { + address: pcie_ecam_address, + start_bus: start_bus_number, + end_bus: end_bus_number, + }); +} diff --git a/src/hypervisor_bootloader/src/serial_port.rs b/src/hypervisor_bootloader/src/serial_port.rs index 93ed1a2..884466a 100644 --- a/src/hypervisor_bootloader/src/serial_port.rs +++ b/src/hypervisor_bootloader/src/serial_port.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + use crate::{dtb, ACPI_20_TABLE_ADDRESS, DTB_ADDRESS}; use common::acpi::{get_acpi_table, GeneralAddressStructure}; diff --git a/src/hypervisor_kernel/Cargo.toml b/src/hypervisor_kernel/Cargo.toml index 352a5a6..c8e7680 100644 --- a/src/hypervisor_kernel/Cargo.toml +++ b/src/hypervisor_kernel/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "hypervisor_kernel" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] -common = { path = "../common" } \ No newline at end of file +common = { path = "../common" } diff --git a/src/hypervisor_kernel/Makefile b/src/hypervisor_kernel/Makefile index df93429..e3fe5b4 100644 --- a/src/hypervisor_kernel/Makefile +++ b/src/hypervisor_kernel/Makefile @@ -1,3 +1,9 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + NAME = hypervisor_kernel TARGET = aarch64-none TARGET_JSON = config/$(TARGET).json diff --git a/src/hypervisor_kernel/src/drivers.rs b/src/hypervisor_kernel/src/drivers.rs new file mode 100644 index 0000000..c07a6f4 --- /dev/null +++ b/src/hypervisor_kernel/src/drivers.rs @@ -0,0 +1,11 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +//! +//! MemoryMapped I/O Interrupt Handlers +//! + +pub mod i210; diff --git a/src/hypervisor_kernel/src/drivers/i210.rs b/src/hypervisor_kernel/src/drivers/i210.rs new file mode 100644 index 0000000..96b900c --- /dev/null +++ b/src/hypervisor_kernel/src/drivers/i210.rs @@ -0,0 +1,375 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +//! +//! Intel(R) Ethernet Controller I210 +//! + +use crate::memory_hook::{ + add_memory_load_hook_handler, add_memory_store_hook_handler, remove_memory_load_hook_handler, + remove_memory_store_hook_handler, LoadAccessHandlerEntry, LoadHookResult, + StoreAccessHandlerEntry, StoreHookResult, +}; +use crate::pci::{get_configuration_space_data, get_ecam_target_address}; +use crate::{paging, StoredRegisters}; + +use common::{bitmask, PAGE_SIZE, STAGE_2_PAGE_MASK, STAGE_2_PAGE_SIZE}; + +pub const VENDOR_ID: u16 = 0x8086; +pub const DEVICE_ID: u16 = 0x1533; + +static mut EEPROM_BLOCK_BASE: u32 = 0; +static mut EEPROM_BLOCK_END: u32 = 0; +static mut IS_64BIT_BAR: bool = false; +static mut CURRENT_MEMORY_BAR: usize = 0; +static mut CURRENT_EXPANSION_ROM_BAR: usize = 0; +static mut FLBAR_SIZE: usize = 1024 * 64; + +const FLASH_SECURITY_REGISTERS_BASE: usize = 0x12000; +const EEWR: usize = 0x12018; +const EEWR_ALIAS: usize = 0x1101C; +const FLSWCTL: usize = 0x12048; +const FLSWDATA: usize = 0x1204C; +const I_NVM_DATA: usize = 0x12120; +const I_NVM_DATA_LEN: usize = (0x1221C - I_NVM_DATA) + 1; + +static I210_LOAD_HANDLERS: [LoadAccessHandlerEntry; 2] = [ + LoadAccessHandlerEntry::new(EEWR, 4, i210_eeprom_write_register_load_handler), + LoadAccessHandlerEntry::new(EEWR_ALIAS, 4, i210_eeprom_write_register_load_handler), +]; + +static I210_STORE_HANDLERS: [StoreAccessHandlerEntry; 4] = [ + StoreAccessHandlerEntry::new(EEWR, 4, i210_eeprom_write_register_store_handler), + StoreAccessHandlerEntry::new(EEWR_ALIAS, 4, i210_eeprom_write_register_store_handler), + StoreAccessHandlerEntry::new( + FLSWCTL, + FLSWDATA - FLSWCTL, + i210_i_flash_burst_registers_store_handler, + ), + StoreAccessHandlerEntry::new(I_NVM_DATA, I_NVM_DATA_LEN, i210_i_nvm_data_store_handler), +]; + +pub fn setup_device(ecam_address: usize, bus: u8, device: u8, function: u8) { + let class_code = get_configuration_space_data(ecam_address, bus, device, function, 0x09, 3); + if class_code != 0x020000 && class_code != 0x010000 { + println!("Unsupported I210 Class Code: {:#X}", class_code); + return; + } + println!("I210 Ethernet controller: ClassCode: {:#X}", class_code); + let is_64bit_bar = + ((get_configuration_space_data(ecam_address, bus, device, function, 0x10, 4) >> 2) & 1) + != 0; + if is_64bit_bar { + println!("64bit BAR Mode"); + } else { + println!("32bit BAR Mode"); + } + unsafe { IS_64BIT_BAR = is_64bit_bar }; + + add_memory_store_hook_handler(StoreAccessHandlerEntry::new( + get_ecam_target_address(ecam_address, bus, device, function) + 0x10, + 4 * 2, + i210_pci_bar_address_store_handler, + )) + .expect("Failed to add the handler for memory bar"); + paging::add_memory_access_trap( + get_ecam_target_address(ecam_address, bus, device, function), + 0x1000, + true, + false, + ) + .expect("Failed to setup memory trap."); + let memory_bar = ((get_configuration_space_data(ecam_address, bus, device, function, 0x10, 4)) + & !0b1111) as usize + | (if is_64bit_bar { + ((get_configuration_space_data(ecam_address, bus, device, function, 0x14, 4)) as usize) + << 32 + } else { + 0 + }); + println!("I210 Base Address Register: {:#X}", memory_bar); + unsafe { CURRENT_MEMORY_BAR = memory_bar }; + setup_memory_trap(memory_bar); + println!("Add I210 Ethernet Controller BAR Handler"); + + /* Expansion ROM */ + let expansion_rom_base_address = + get_configuration_space_data(ecam_address, bus, device, function, 0x30, 4); + if (expansion_rom_base_address & 1) == 1 { + let expansion_rom_bar = (expansion_rom_base_address & bitmask!(31, 11)) as usize; + + paging::map_address( + expansion_rom_bar, + expansion_rom_bar, + PAGE_SIZE, + true, + false, + false, + true, + ) + .expect("Failed to map Expansion ROM"); + let pci_e_control_2 = unsafe { *((expansion_rom_bar + 0x28 * 2) as *const u16) }; + let flbar_size = 64 * 1024 * 2usize.pow(((pci_e_control_2 >> 1) & 0b111) as u32); + println!( + "Expansion ROM: {:#X}, FLBAR_SIZE: {:#X}", + expansion_rom_bar, flbar_size + ); + unsafe { FLBAR_SIZE = flbar_size }; + setup_expansion_rom_memory_trap(expansion_rom_bar); + add_memory_store_hook_handler(StoreAccessHandlerEntry::new( + get_ecam_target_address(ecam_address, bus, device, function) + 0x30, + 4, + i210_pci_expansion_rom_bar_address_store_handler, + )) + .expect("Failed to add the handler for expansion rom bar"); + } else { + println!("No Expansion ROM"); + } + + /* TODO: Inspect BARCTRL field */ + //let bar_ctrl = unsafe { *((memory_bar + 0x5BFC) as *const u32) }; + let eeprom_block_base = unsafe { *((memory_bar + 0x1210C) as *const u32) }; + let eeprom_block_end = unsafe { *((memory_bar + 0x12110) as *const u32) }; + println!( + "EEPROM: 1st: {:#X} ~ {:#X}, 2nd: {:#X}", + eeprom_block_base & bitmask!(10, 0), + eeprom_block_end & bitmask!(10, 0), + (eeprom_block_base & bitmask!(22, 12)) >> 12 + ); + unsafe { + EEPROM_BLOCK_BASE = eeprom_block_base; + EEPROM_BLOCK_END = eeprom_block_end; + } +} + +fn setup_memory_trap(new_memory_bar: usize) { + pr_debug!("I210 Base Address Register: {:#X}", new_memory_bar); + paging::map_address( + new_memory_bar, + new_memory_bar, + 128 * 1024, + true, + true, + false, + true, + ) + .expect("Failed to map"); + + /* Set up to trap registers' area */ + paging::add_memory_access_trap( + new_memory_bar + FLASH_SECURITY_REGISTERS_BASE, + STAGE_2_PAGE_SIZE, + false, + false, + ) + .expect("Failed to map EEPROM Register"); + paging::add_memory_access_trap( + (new_memory_bar + EEWR_ALIAS) & STAGE_2_PAGE_MASK, + STAGE_2_PAGE_SIZE, + false, + false, + ) + .expect("Failed to map EEPROM Register(Alias)"); + + /* Set up load access handlers */ + for e in &I210_LOAD_HANDLERS { + let mut e = e.clone(); + e.set_target_address(e.get_target_address() + new_memory_bar); + add_memory_load_hook_handler(e).expect("Failed to set up the load handler"); + } + + /* Set up store access handlers */ + for e in &I210_STORE_HANDLERS { + let mut e = e.clone(); + e.set_target_address(e.get_target_address() + new_memory_bar); + add_memory_store_hook_handler(e).expect("Failed to set up the store handler"); + } +} + +fn remove_memory_trap(bar_address: usize) { + pr_debug!("Remove I210 Base Address Register Trap: {:#X}", bar_address); + + /* Remove the trap of registers' area */ + paging::remove_memory_access_trap( + bar_address + FLASH_SECURITY_REGISTERS_BASE, + STAGE_2_PAGE_SIZE, + ) + .expect("Failed to map EEPROM Register"); + paging::remove_memory_access_trap( + (bar_address + EEWR_ALIAS) & STAGE_2_PAGE_MASK, + STAGE_2_PAGE_SIZE, + ) + .expect("Failed to map EEPROM Register(Alias)"); + + /* Remove load access handlers */ + for e in &I210_LOAD_HANDLERS { + let mut e = e.clone(); + e.set_target_address(e.get_target_address() + bar_address); + remove_memory_load_hook_handler(e).expect("Failed to remove the load handler"); + } + + /* Remove store access handlers */ + for e in &I210_STORE_HANDLERS { + let mut e = e.clone(); + e.set_target_address(e.get_target_address() + bar_address); + remove_memory_store_hook_handler(e).expect("Failed to remove the store handler"); + } +} + +fn setup_expansion_rom_memory_trap(expansion_rom_bar: usize) { + paging::add_memory_access_trap(expansion_rom_bar, unsafe { FLBAR_SIZE }, true, false) + .expect("Failed to add the trap for Expansion ROM"); + add_memory_store_hook_handler(StoreAccessHandlerEntry::new( + expansion_rom_bar, + unsafe { FLBAR_SIZE }, + i210_expansion_rom_store_handler, + )) + .expect("Failed to add the handler for Expansion ROM"); +} + +fn remove_expansion_rom_memory_trap(expansion_rom_bar: usize) { + paging::remove_memory_access_trap(expansion_rom_bar, unsafe { FLBAR_SIZE }) + .expect("Failed to add the trap for Expansion ROM"); + remove_memory_store_hook_handler(StoreAccessHandlerEntry::new( + expansion_rom_bar, + unsafe { FLBAR_SIZE }, + i210_expansion_rom_store_handler, + )) + .expect("Failed to add the handler for Expansion ROM"); +} + +fn i210_pci_bar_address_store_handler( + accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + data: u64, +) -> Result { + let offset = accessing_memory_address & 0xFFF; + if offset == 0x10 { + pr_debug!("Writing I210 BAR0: {:#X}", data); + } else { + pr_debug!("Writing I210 BAR1: {:#X}", data); + } + if offset != 0x10 || !unsafe { IS_64BIT_BAR } { + let new_bar = ((if offset == 0x10 { + data & bitmask!(31, 17) + } else { + assert_eq!(offset, 0x14); + (unsafe { *((accessing_memory_address - 0x04) as *const u32) } & bitmask!(31, 17)) + as u64 + }) | (if unsafe { IS_64BIT_BAR } { + assert_eq!(offset, 0x14); + data + } else { + 0 + })) as usize; + pr_debug!( + "Change I210 BAR: {:#X} => {:#X}", + unsafe { CURRENT_MEMORY_BAR }, + new_bar + ); + remove_memory_trap(unsafe { CURRENT_MEMORY_BAR }); + setup_memory_trap(new_bar); + unsafe { CURRENT_MEMORY_BAR = new_bar }; + } + return Ok(StoreHookResult::PassThrough); +} + +fn i210_pci_expansion_rom_bar_address_store_handler( + _accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + data: u64, +) -> Result { + let new_expansion_rom_bar = (data & bitmask!(31, 11)) as usize; + pr_debug!( + "Change I210 Expansion ROM BAR: {:#X} => {:#X}", + unsafe { CURRENT_EXPANSION_ROM_BAR }, + new_expansion_rom_bar + ); + remove_expansion_rom_memory_trap(unsafe { CURRENT_EXPANSION_ROM_BAR }); + setup_expansion_rom_memory_trap(new_expansion_rom_bar); + return Ok(StoreHookResult::PassThrough); +} + +fn i210_eeprom_write_register_load_handler( + _accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + _is_64bit_register: bool, + _is_sign_extend_required: bool, +) -> Result { + pr_debug!("EEPROM Write Register Load Access"); + let data: u64 = 1 << 1; + pr_debug!("Return the alternative data: {:#X}", data); + return Ok(LoadHookResult::Data(data)); +} + +fn i210_eeprom_write_register_store_handler( + _accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + data: u64, +) -> Result { + println!("EEPROM Write Register Store Access"); + let address = ((data & bitmask!(12, 2)) >> 2) as u32; + pr_debug!("EEPROM Address: {:#X}, Data: {:#X}", address, data >> 16); + let eeprom_1st_block_end = unsafe { EEPROM_BLOCK_END & bitmask!(10, 0) }; + let eeprom_1st_block_start = unsafe { EEPROM_BLOCK_BASE & bitmask!(10, 0) }; + let eeprom_2nd_block_start = unsafe { (EEPROM_BLOCK_BASE & bitmask!(22, 12)) >> 12 }; + if eeprom_1st_block_end != 0 + && (eeprom_1st_block_start..=eeprom_1st_block_end).contains(&address) + { + pr_debug!("EEPROM 1st Block Access"); + } else if eeprom_2nd_block_start != 0 && address >= eeprom_2nd_block_start { + pr_debug!("EEPROM 2nd Block Access"); + } + return Ok(StoreHookResult::Cancel); +} + +fn i210_i_nvm_data_store_handler( + accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + data: u64, +) -> Result { + println!( + "iNVM Data Register Store Access: Offset: {:#X}, Data: {:#X}", + accessing_memory_address - unsafe { CURRENT_MEMORY_BAR } - I_NVM_DATA, + data + ); + + return Ok(StoreHookResult::Cancel); +} + +fn i210_i_flash_burst_registers_store_handler( + accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + data: u64, +) -> Result { + println!( + "iNVM Flash Burst Registers Store Access: Register: {}, Data: {:#X}", + if accessing_memory_address - unsafe { CURRENT_MEMORY_BAR } == FLSWCTL { + "FLSWCTL" + } else { + "FLSWDATA" + }, + data + ); + + return Ok(StoreHookResult::Cancel); +} + +fn i210_expansion_rom_store_handler( + _accessing_memory_address: usize, + _stored_registers: &mut StoredRegisters, + _access_size: u8, + _data: u64, +) -> Result { + println!("i210 Expansion ROM Store Access"); + return Ok(StoreHookResult::Cancel); +} diff --git a/src/hypervisor_kernel/src/emulation.rs b/src/hypervisor_kernel/src/emulation.rs new file mode 100644 index 0000000..34ef685 --- /dev/null +++ b/src/hypervisor_kernel/src/emulation.rs @@ -0,0 +1,264 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +/// +/// A64 Instructions' Emulator +/// +mod load; +mod store; + +use crate::{handler_panic, StoredRegisters}; + +use common::bitmask; +use common::cpu::{ + convert_virtual_address_to_intermediate_physical_address_el1_read, + convert_virtual_address_to_intermediate_physical_address_el1_write, + convert_virtual_address_to_physical_address_el2_read, +}; + +use core::arch::asm; + +const NORMAL_INSTRUCTION_SIZE: usize = 4; + +#[allow(unused_variables)] +pub fn data_abort_handler( + s_r: &mut StoredRegisters, + esr: u64, + elr: u64, + far: u64, + hpfar: u64, +) -> Result<(), ()> { + #[cfg(debug_assertions)] + if (esr & (1 << 24)) != 0 { + let sas = (esr >> 22) & 0b11; + match sas { + 0b00 => println!("Byte(8Bits) Access"), + 0b01 => println!("HalfWord(16Bits) Access"), + 0b10 => println!("Word(32Bits) Access"), + 0b11 => println!("DoubleWord(64Bits) Access"), + _ => unreachable!(), + } + if (esr & (1 << 21)) != 0 { + println!("Sign Extension is required."); + } + let srt = (esr >> 16) & 0b11111; + println!("SRT: {}", srt); + if (esr & (1 << 15)) != 0 { + println!("64Bit Register"); + } else { + println!("32Bit Register"); + } + if (esr & (1 << 10)) != 0 { + panic!("FAR is not valid"); + } + if (esr & (1 << 6)) != 0 { + println!("Write Access"); + } else { + println!("Read Access"); + } + } else { + println!("No Valid Instruction Syndrome Information."); + } + + /* TODO: check EL1 or EL0 */ + let instruction_intermediate_physical_address = + convert_virtual_address_to_intermediate_physical_address_el1_read(elr as usize).unwrap(); + pr_debug!( + "Target Instruction Address: {:#X} => {:#X}", + elr, + instruction_intermediate_physical_address + ); + let target_instruction = if let Ok(e) = convert_virtual_address_to_physical_address_el2_read( + instruction_intermediate_physical_address, + ) { + unsafe { *(e as *const u32) } + } else { + handler_panic!(s_r, "TODO: Mapping...") + }; + pr_debug!("Target Instruction: {:#X}", target_instruction); + + return emulate_instruction(s_r, target_instruction, elr, far, hpfar); +} + +fn emulate_instruction( + s_r: &mut StoredRegisters, + target_instruction: u32, + _elr: u64, + far: u64, + hpfar: u64, +) -> Result<(), ()> { + /* ARM DDI 0487G.a ID011921 C4-280 */ + let op0 = ((target_instruction & bitmask!(28, 25)) >> 25) as u8; + if (op0 & 0b0101) != 0b0100 { + handler_panic!(s_r, "Not Load/Store Instruction: {:#X}", target_instruction); + } + let op1 = (op0 >> 1) & 1; + let op0 = ((target_instruction & bitmask!(31, 28)) >> 28) as u8; + let op2 = ((target_instruction & bitmask!(24, 23)) >> 23) as u8; + let op3 = ((target_instruction & bitmask!(21, 16)) >> 16) as u8; + let op4 = ((target_instruction & bitmask!(11, 10)) >> 10) as u8; + pr_debug!( + "op0: {:#b}, op1: {:#b}, op2: {:#b}, op3: {:#b}, op4: {:#b}", + op0, + op1, + op2, + op3, + op4 + ); + let op0_half_bottom = op0 & 0b11; + if op0_half_bottom == 0b11 { + pr_debug!("Load/Store Register"); + if (op2 & 0b10) != 0 { + /* unsigned immediate (No post|pre indexing) */ + if op1 != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + let opc = ((target_instruction & bitmask!(23, 22)) >> 22) as u8; + return if opc == 0b00 { + pr_debug!("STR(unsigned immediate)"); + store::emulate_unsigned_immediate_store_register( + s_r, + target_instruction, + far, + hpfar, + ) + } else { + pr_debug!("LDR(unsigned immediate)"); + load::emulate_unsigned_immediate_load_register(s_r, target_instruction, far, hpfar) + }; + } else if (op3 & 0b100000) != 0 { + match op4 { + 0b00 => { + println!("Atomic Operation") + } + 0b10 => { + pr_debug!("Load/Store Register Offset"); + if op1 != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + let opc = ((target_instruction & bitmask!(23, 22)) >> 22) as u8; + return if opc == 0b00 { + /* when implements SIMD, opc of SIMD-STR may not be 0b00 */ + pr_debug!("STR Register Offset"); + store::emulate_store_register_register_offset( + s_r, + target_instruction, + far, + hpfar, + ) + } else { + pr_debug!("LDR Register Offset"); + load::emulate_load_register_register_offset( + s_r, + target_instruction, + far, + hpfar, + ) + }; + } + _ => { + println!("Load/Store pac") + } + } + } else { + if op1 != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + let opc = ((target_instruction & bitmask!(23, 22)) >> 22) as u8; + return if opc == 0b00 { + /* when implements SIMD, opc of SIMD-STR may not be 0b00 */ + pr_debug!("STR"); + store::emulate_store_register(s_r, target_instruction, far, hpfar) + } else { + pr_debug!("LDR"); + load::emulate_load_register(s_r, target_instruction, far, hpfar) + }; + } + } else if op0_half_bottom == 0b10 { + pr_debug!("Load/Store Register Pair"); + if op1 != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + return if (target_instruction & (1 << 22)) != 0 { + pr_debug!("LDP"); + load::emulate_load_pair(s_r, target_instruction, far, hpfar) + } else { + pr_debug!("STP"); + store::emulate_store_pair(s_r, target_instruction, far, hpfar) + }; + } else if op0_half_bottom == 0b01 { + { + if (op2 & (1 << 1)) == 0 { + pr_debug!("Load Register Literal"); + if op1 != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + return load::emulate_literal_load_register(s_r, target_instruction, far, hpfar); + } + } + } + println!("Unknown Instruction: {:#X}", target_instruction); + return Err(()); +} + +fn faulting_virtual_address_to_intermediate_physical_address(far: u64) -> Result { + /* TODO: check EL1 or EL0 */ + if let Ok(a) = convert_virtual_address_to_intermediate_physical_address_el1_read(far as usize) { + return Ok(a); + } + if let Ok(a) = convert_virtual_address_to_intermediate_physical_address_el1_write(far as usize) + { + return Ok(a); + } + println!("Failed to convert the virtual address(FAR: {:#X}", far); + return Err(()); +} + +fn advance_elr_el2() { + unsafe { + asm!(" + mrs {t}, elr_el2 + add {t}, {t}, {SIZE} + msr elr_el2, {t} + ", t = out(reg) _ ,SIZE = const NORMAL_INSTRUCTION_SIZE) + }; +} + +fn get_register_reference_mut(s_r: &mut StoredRegisters, index: u8) -> &mut u64 { + unsafe { + &mut core::mem::transmute::< + &mut StoredRegisters, + &mut [u64; core::mem::size_of::() / core::mem::size_of::()], + >(s_r)[index as usize] + } +} + +fn write_back_index_register_imm9(base_register: &mut u64, imm9_u32: u32) { + unsafe { + asm!(" + sbfx {imm9}, {imm9}, #0, #9 + add {base_reg}, {base_reg}, {imm9} + ", + imm9 = inout(reg) (imm9_u32 as u64) => _ , + base_reg = inout(reg) *base_register) + }; +} + +fn write_back_index_register_imm7(base_register: &mut u64, imm7_u32: u32) { + unsafe { + asm!(" + sbfx {imm7}, {imm7}, #0, #7 + add {base_reg}, {base_reg}, {imm7} + ", + imm7 = inout(reg) (imm7_u32 as u64) => _ , + base_reg = inout(reg) *base_register) + }; +} diff --git a/src/hypervisor_kernel/src/emulation/load.rs b/src/hypervisor_kernel/src/emulation/load.rs new file mode 100644 index 0000000..45f48bc --- /dev/null +++ b/src/hypervisor_kernel/src/emulation/load.rs @@ -0,0 +1,304 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +/// +/// A64 Load Instructions' Emulator +/// +/// Supported: ldr, ldp (except Atomic, SIMD) +/// +use super::{ + advance_elr_el2, faulting_virtual_address_to_intermediate_physical_address, + get_register_reference_mut, write_back_index_register_imm7, write_back_index_register_imm9, +}; +use crate::StoredRegisters; + +use crate::memory_hook::{memory_load_hook_handler, LoadHookResult}; +use common::cpu::convert_virtual_address_to_physical_address_el2_read; +use common::{bitmask, STAGE_2_PAGE_SHIFT}; + +pub fn emulate_load_register( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_load_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + //let op2 = ((target_instruction & bitmask!(24, 23)) >> 23) as u8; + //let op3 = ((target_instruction & bitmask!(21, 16)) >> 16) as u8; + let op4 = ((target_instruction & bitmask!(11, 10)) >> 10) as u8; + /* sf is usable only if sse is true */ + let sf = (target_instruction & (1 << 22)) == 0; + let sse = (target_instruction & (1 << 23)) != 0; + + pr_debug!( + "{}{} <= [{:#X}](IPA)(Sign Extend: {})({})", + if sse && !sf { 'W' } else { 'X' }, + target_register, + intermediate_physical_load_address, + sse, + match op4 { + 0b00 => "Unscaled", + 0b01 => "PostIndexed", + 0b10 => "Unprivileged", + 0b11 => "PreIndexed", + _ => unreachable!(), + } + ); + if op4 == 0b10 { + unimplemented!("UnPrivileged Access is not implemented..."); + } + + let size = (target_instruction >> 30) as u8; + if size == 0b11 && op4 == 0b00 && sse && !sf { + pr_debug!("Prefetch Memory Unscaled Signals."); + return Ok(()); + } + pr_debug!("Size: {:#b}", size); + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address, + target_register, + size, + sf, + sse, + )?; + if (op4 & 1) != 0 { + pr_debug!("Post/Pre Indexed"); + let imm9 = (target_instruction & bitmask!(20, 12)) >> 12; + let base_register = + get_register_reference_mut(s_r, ((target_instruction & bitmask!(9, 5)) >> 5) as u8); + write_back_index_register_imm9(base_register, imm9); + } + + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_unsigned_immediate_load_register( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_load_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + + /* sf is usable only if sse is true */ + let sf = (target_instruction & (1 << 22)) == 0; + let sse = (target_instruction & (1 << 23)) != 0; + let size = (target_instruction >> 30) as u8; + + pr_debug!("Size: {:#b}", size); + if size == 0b11 && sse && !sf { + pr_debug!("Prefetch Memory Immediate Signals."); + return Ok(()); + } + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address, + target_register, + size, + sf, + sse, + )?; + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_load_register_register_offset( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_load_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + + /* sf is usable only if sse is true */ + let sf = (target_instruction & (1 << 22)) == 0; + let sse = (target_instruction & (1 << 23)) != 0; + let size = (target_instruction >> 30) as u8; + if size == 0b11 && sse { + pr_debug!("Prefetch Memory Register Signals."); + return Ok(()); + } + + pr_debug!("Size: {:#b}", size); + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address, + target_register, + size, + sf, + sse, + )?; + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_literal_load_register( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_load_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + + let opc = (target_instruction >> 30) as u8; + /* sf is usable only if sse is true */ + let sf = opc == 0b01 || opc == 0b10; + let sse = opc == 0b10; + let size = if opc == 0b01 { 0b11 } else { 0b10 }; + + pr_debug!("Size: {:#b}", size); + if opc == 0b11 { + pr_debug!("Prefetch Memory Literal Signals"); + return Ok(()); + } + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address, + target_register, + size, + sf, + sse, + )?; + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_load_pair( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let op2 = ((target_instruction & bitmask!(24, 23)) >> 23) as u8; + let opc = (target_instruction >> 30) as u8; + let sf = (opc & (1 << 1)) != 0; + let sse = (opc & 1) != 0; + let is_pre_or_post_indexed = (op2 & 1) != 0; + let intermediate_physical_load_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + let target_register_1 = (target_instruction & bitmask!(4, 0)) as u8; + let target_register_2 = ((target_instruction & bitmask!(14, 10)) >> 10) as u8; + + pr_debug!( + "{}{}, {}{}(+{}) <= [{:#X}](IPA)(Sign Extend: {})({})", + if sf { 'X' } else { 'W' }, + target_register_1, + if sf { 'X' } else { 'W' }, + target_register_2, + if sf { 8 } else { 4 }, + intermediate_physical_load_address, + sse, + match op2 { + 0b00 => "NonAllocate", /* It means the memory area is unlikely to access repeatedly */ + 0b01 => "PostIndexed", + 0b10 => "Offset", + 0b11 => "PreIndexed", + _ => unreachable!(), + } + ); + + if (intermediate_physical_load_address >> STAGE_2_PAGE_SHIFT) + != ((intermediate_physical_load_address + ((if sf { 8 } else { 4 }) * 2) - 1) + >> STAGE_2_PAGE_SHIFT) + { + println!("LDP alignment error."); + return Err(()); + } + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address, + target_register_1, + if sf { 0b11 } else { 0b10 }, + sf, + sse, + )?; + load_from_address_and_store_into_register( + s_r, + intermediate_physical_load_address + if sf { 8 } else { 4 }, + target_register_2, + if sf { 0b11 } else { 0b10 }, + sf, + sse, + )?; + + if is_pre_or_post_indexed { + pr_debug!("Post/Pre Indexed"); + let imm7 = (target_instruction & bitmask!(21, 15)) >> 15; + let base_register = + get_register_reference_mut(s_r, ((target_instruction & bitmask!(9, 5)) >> 5) as u8); + write_back_index_register_imm7(base_register, imm7); + } + advance_elr_el2(); + return Ok(()); +} + +fn load_from_address_and_store_into_register( + s_r: &mut StoredRegisters, + intermediate_physical_load_address: usize, + target_register: u8, + size: u8, + sf: bool, /* sf is usable only if sse is true */ + sse: bool, +) -> Result<(), ()> { + let sf = !sse || sf; + + pr_debug!( + "{}{} <= [{:#X}](IPA)(Sign Extend: {})", + if sf { 'X' } else { 'W' }, + target_register, + intermediate_physical_load_address, + sse + ); + /* TODO: 物理アドレスへのアクセス関数を用意する。現方法だとVA!=PAの時に誤動作する */ + let physical_load_address = + convert_virtual_address_to_physical_address_el2_read(intermediate_physical_load_address) + .expect("Failed to convert IPA => PA"); + + pr_debug!( + "{}{} <= [{:#X}](PA)(Sign Extend: {})", + if sf { 'X' } else { 'W' }, + target_register, + physical_load_address, + sse + ); + if !sf && size == 0b11 { + println!("Invalid Instruction: Loading a 64bit data into the 32bit register."); + return Err(()); + } + + let hook_result = memory_load_hook_handler(physical_load_address, s_r, size, sf, sse)?; + let data = match hook_result { + LoadHookResult::PassThrough => { + if sse { + unimplemented!(); + } else { + match size { + 0b00 => unsafe { *(physical_load_address as *const u8) as u64 }, + 0b01 => unsafe { *(physical_load_address as *const u16) as u64 }, + 0b10 => unsafe { *(physical_load_address as *const u32) as u64 }, + 0b11 => unsafe { *(physical_load_address as *const u64) }, + _ => unreachable!(), + } + } + } + LoadHookResult::Data(d) => d, + }; + + pr_debug!("Data: {:#X}", data); + *get_register_reference_mut(s_r, target_register) = data; + + return Ok(()); +} diff --git a/src/hypervisor_kernel/src/emulation/store.rs b/src/hypervisor_kernel/src/emulation/store.rs new file mode 100644 index 0000000..41a4ac5 --- /dev/null +++ b/src/hypervisor_kernel/src/emulation/store.rs @@ -0,0 +1,229 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +/// +/// A64 Store Instructions' Emulator +/// +/// Supported: str, stp (except Atomic, SIMD) +/// +use super::{ + advance_elr_el2, faulting_virtual_address_to_intermediate_physical_address, + get_register_reference_mut, write_back_index_register_imm7, write_back_index_register_imm9, +}; +use crate::{handler_panic, StoredRegisters}; + +use crate::memory_hook::{memory_store_hook_handler, StoreHookResult}; +use common::cpu::convert_virtual_address_to_physical_address_el2_write; +use common::{bitmask, STAGE_2_PAGE_SHIFT}; + +pub fn emulate_store_register( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_store_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + //let op2 = ((target_instruction & bitmask!(24, 23)) >> 23) as u8; + //let op3 = ((target_instruction & bitmask!(21, 16)) >> 16) as u8; + let op4 = ((target_instruction & bitmask!(11, 10)) >> 10) as u8; + let size = (target_instruction >> 30) as u8; + + pr_debug!( + "[{:#X}](IPA) <= R{}({})", + intermediate_physical_store_address, + target_register, + match op4 { + 0b00 => "Unscaled", + 0b01 => "PostIndexed", + 0b10 => "Unprivileged", + 0b11 => "PreIndexed", + _ => unreachable!(), + } + ); + if op4 == 0b10 { + unimplemented!("UnPrivileged Access is not implemented..."); + } + + if ((target_instruction >> 26) & 1) != 0 { + /* V */ + handler_panic!(s_r, "SIMD is not supported: {:#X}", target_instruction); + } + + pr_debug!("Size: {:#b}", size); + store_register_into_address( + s_r, + intermediate_physical_store_address, + target_register, + size, + )?; + if (op4 & 1) != 0 { + pr_debug!("Post/Pre Indexed"); + let imm9 = (target_instruction & bitmask!(20, 12)) >> 12; + let base_register = + get_register_reference_mut(s_r, ((target_instruction & bitmask!(9, 5)) >> 5) as u8); + write_back_index_register_imm9(base_register, imm9); + } + + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_unsigned_immediate_store_register( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_store_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + let size = (target_instruction >> 30) as u8; + store_register_into_address( + s_r, + intermediate_physical_store_address, + target_register, + size, + )?; + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_store_register_register_offset( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let target_register = (target_instruction & bitmask!(4, 0)) as u8; + let intermediate_physical_store_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + let size = (target_instruction >> 30) as u8; + store_register_into_address( + s_r, + intermediate_physical_store_address, + target_register, + size, + )?; + advance_elr_el2(); + return Ok(()); +} + +pub fn emulate_store_pair( + s_r: &mut StoredRegisters, + target_instruction: u32, + far: u64, + _hpfar: u64, +) -> Result<(), ()> { + let op2 = ((target_instruction & bitmask!(24, 23)) >> 23) as u8; + let opc = (target_instruction >> 30) as u8; + let sf = (opc & (1 << 1)) != 0; + let sse = (opc & 1) != 0; + let is_pre_or_post_indexed = (op2 & 1) != 0; + let intermediate_physical_store_address = + faulting_virtual_address_to_intermediate_physical_address(far)?; + let target_register_1 = (target_instruction & bitmask!(4, 0)) as u8; + let target_register_2 = ((target_instruction & bitmask!(14, 10)) >> 10) as u8; + + pr_debug!( + "[{:#X}](IPA) <= {}{}, {}{}(+{}) (Sign Extend: {})({})", + intermediate_physical_store_address, + if sf { 'X' } else { 'W' }, + target_register_1, + if sf { 'X' } else { 'W' }, + target_register_2, + if sf { 8 } else { 4 }, + sse, + match op2 { + 0b00 => "NonAllocate", /* It means the memory area is unlikely to access repeatedly */ + 0b01 => "PostIndexed", + 0b10 => "Offset", + 0b11 => "PreIndexed", + _ => unreachable!(), + } + ); + + if (intermediate_physical_store_address >> STAGE_2_PAGE_SHIFT) + != ((intermediate_physical_store_address + ((if sf { 8 } else { 4 }) * 2) - 1) + >> STAGE_2_PAGE_SHIFT) + { + println!("STP alignment error."); + return Err(()); + } + if sse { + unimplemented!(); + } + store_register_into_address( + s_r, + intermediate_physical_store_address, + target_register_1, + if sf { 0b11 } else { 0b10 }, + )?; + store_register_into_address( + s_r, + intermediate_physical_store_address + if sf { 8 } else { 4 }, + target_register_2, + if sf { 0b11 } else { 0b10 }, + )?; + + if is_pre_or_post_indexed { + pr_debug!("Post/Pre Indexed"); + let imm7 = (target_instruction & bitmask!(21, 15)) >> 15; + let base_register = + get_register_reference_mut(s_r, ((target_instruction & bitmask!(9, 5)) >> 5) as u8); + write_back_index_register_imm7(base_register, imm7); + } + advance_elr_el2(); + return Ok(()); +} + +fn store_register_into_address( + s_r: &mut StoredRegisters, + intermediate_physical_store_address: usize, + target_register: u8, + size: u8, +) -> Result<(), ()> { + /* TODO: 物理アドレスへのアクセス関数を用意する。現方法だとVA!=PAの時に誤動作する */ + let physical_store_address = + convert_virtual_address_to_physical_address_el2_write(intermediate_physical_store_address) + .expect("Failed to convert IPA => PA"); + + pr_debug!( + "[{:#X}](PA) <= R{}({})", + physical_store_address, + target_register, + match size { + 0b00 => " 8Bit", + 0b01 => "16Bit", + 0b10 => "32Bit", + 0b11 => "64Bit", + _ => unreachable!(), + } + ); + + let reg_data = *get_register_reference_mut(s_r, target_register); + let hook_result = memory_store_hook_handler(physical_store_address, s_r, size, reg_data)?; + let data = match hook_result { + StoreHookResult::PassThrough => reg_data, + StoreHookResult::AlternativeData(d) => d, + StoreHookResult::Cancel => { + pr_debug!("The store instruction is cancelled."); + return Ok(()); + } + }; + + pr_debug!("Data: {:#X}", data); + match size { + 0b00 => unsafe { *(physical_store_address as *mut u8) = data as u8 }, + 0b01 => unsafe { *(physical_store_address as *mut u16) = data as u16 }, + 0b10 => unsafe { *(physical_store_address as *mut u32) = data as u32 }, + 0b11 => unsafe { *(physical_store_address as *mut u64) = data }, + _ => unreachable!(), + }; + + return Ok(()); +} diff --git a/src/hypervisor_kernel/src/main.rs b/src/hypervisor_kernel/src/main.rs index 9d219a4..a626d3a 100644 --- a/src/hypervisor_kernel/src/main.rs +++ b/src/hypervisor_kernel/src/main.rs @@ -1,29 +1,43 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + #![no_std] #![no_main] -#![feature(asm)] #![feature(asm_const)] -#![feature(global_asm)] -#![feature(maybe_uninit_uninit_array)] -#![feature(maybe_uninit_extra)] +#![feature(const_fn_fn_ptr_basics)] +#![feature(const_mut_refs)] #![feature(maybe_uninit_array_assume_init)] +#![feature(maybe_uninit_uninit_array)] #![feature(naked_functions)] #![feature(panic_info_message)] #[macro_use] mod serial_port; +mod drivers; +mod emulation; +mod memory_hook; mod multi_core; +mod paging; mod panic; +mod pci; mod psci; +use crate::pci::init_pci; use crate::psci::{handle_psci_call, PsciFunctionId}; +use crate::serial_port::DEFAULT_SERIAL_PORT; use common::cpu::secure_monitor_call; use common::{SystemInformation, ALLOC_SIZE, PAGE_SIZE}; +use core::arch::{asm, global_asm}; use core::mem::MaybeUninit; const EC_HVC: u64 = 0b010110; const EC_SMC_AA64: u64 = 0b010111; +const EC_DATA_ABORT: u64 = 0b100100; const SMC_INSTRUCTION_SIZE: usize = 4; @@ -33,28 +47,45 @@ static mut MEMORY_POOL: ([MaybeUninit; ALLOC_SIZE / PAGE_SIZE], usize) = #[repr(C)] #[derive(Debug)] pub struct StoredRegisters { - x30: u64, - x29: u64, - x19: u64, - x18: u64, - x17: u64, - x16: u64, - x15: u64, - x14: u64, - x13: u64, - x12: u64, - x11: u64, - x10: u64, - x9: u64, - x8: u64, - x7: u64, - x6: u64, - x5: u64, - x4: u64, - x3: u64, - x2: u64, - x1: u64, x0: u64, + x1: u64, + x2: u64, + x3: u64, + x4: u64, + x5: u64, + x6: u64, + x7: u64, + x8: u64, + x9: u64, + x10: u64, + x11: u64, + x12: u64, + x13: u64, + x14: u64, + x15: u64, + x16: u64, + x17: u64, + x18: u64, + x19: u64, + x20: u64, + x21: u64, + x22: u64, + x23: u64, + x24: u64, + x25: u64, + x26: u64, + x27: u64, + x28: u64, + x29: u64, + x30: u64, + sp: u64, +} + +#[macro_export] +macro_rules! handler_panic { + ($s_r:expr, $($t:tt)*) => { + $crate::interrupt_handler_panic($s_r, format_args!($($t)*)) + }; } #[no_mangle] @@ -65,6 +96,9 @@ fn hypervisor_main(system_information: &mut SystemInformation) { unsafe { MEMORY_POOL = system_information.memory_pool.clone() }; println!("Hello,world from Hypervisor Kernel!!"); + if let Some(ecam_info) = &system_information.ecam_info { + init_pci(ecam_info.address, ecam_info.start_bus, ecam_info.end_bus); + } unsafe { asm!("adr {:x}, vector_table_el2", out(reg)system_information.vbar_el2 ) }; return; } @@ -79,16 +113,22 @@ pub fn allocate_memory(pages: usize) -> Result { #[no_mangle] extern "C" fn synchronous_exception_handler(regs: &mut StoredRegisters) { - println!("Synchronous Exception!!"); let esr_el2: u64; + let elr_el2: u64; let far_el2: u64; let hpfar_el2: u64; - unsafe { asm!("mrs {:x}, far_el2", out(reg) far_el2) }; unsafe { asm!("mrs {:x}, esr_el2", out(reg) esr_el2) }; + unsafe { asm!("mrs {:x}, elr_el2", out(reg) elr_el2) }; + unsafe { asm!("mrs {:x}, far_el2", out(reg) far_el2) }; unsafe { asm!("mrs {:x}, hpfar_el2", out(reg) hpfar_el2) }; - println!("ESR_EL2: {:#X}", esr_el2); - println!("FAR_EL2: {:#X}", far_el2); - println!("HPFAR_EL2: {:#X}", hpfar_el2); + + pr_debug!("Synchronous Exception!!"); + pr_debug!("ESR_EL2: {:#X}", esr_el2); + pr_debug!("ELR_EL2: {:#X}", elr_el2); + pr_debug!("FAR_EL2: {:#X}", far_el2); + pr_debug!("HPFAR_EL2: {:#X}", hpfar_el2); + pr_debug!("MPIDR_EL1: {:#X}", mpidr_el1); + pr_debug!("Registers: {:#X?}", regs); match (esr_el2 >> 26) & ((1 << 6) - 1) { EC_HVC => { @@ -103,14 +143,13 @@ extern "C" fn synchronous_exception_handler(regs: &mut StoredRegisters) { msr ELR_EL2, {t}", t = out(reg) _, size = const SMC_INSTRUCTION_SIZE ) }; let smc_number = esr_el2 & ((1 << 16) - 1); - println!("SecureMonitor Call: {:#X}", smc_number); - println!("Registers: {:#X?}", regs); + pr_debug!("SecureMonitor Call: {:#X}", smc_number); + pr_debug!("Registers: {:#X?}", regs); if smc_number == 0 { if let Ok(psci_function_id) = PsciFunctionId::try_from(regs.x0) { handle_psci_call(psci_function_id, regs); } else { - println!("Unknown Secure Monitor Call: {:#X}", regs.x0); - println!("Try to call secure monitor."); + pr_debug!("Unknown Secure Monitor Call: {:#X}", regs.x0); secure_monitor_call( &mut regs.x0, &mut regs.x1, @@ -136,11 +175,16 @@ extern "C" fn synchronous_exception_handler(regs: &mut StoredRegisters) { panic!("SMC {:#X} is not implemented.", smc_number); } } + EC_DATA_ABORT => { + pr_debug!("Data Abort"); + emulation::data_abort_handler(regs, esr_el2, elr_el2, far_el2, hpfar_el2) + .expect("Failed to emulate the instruction"); + } ec => { - panic!("Unknown EC: {:#X}", ec); + handler_panic!(regs, "Unknown EC: {:#X}", ec); } } - println!("Return to EL1."); + pr_debug!("Return to EL1."); } #[no_mangle] @@ -153,6 +197,30 @@ extern "C" fn s_error_exception_handler() { } } +#[track_caller] +fn interrupt_handler_panic(s_r: &StoredRegisters, f: core::fmt::Arguments) -> ! { + let esr_el2: u64; + let elr_el2: u64; + let far_el2: u64; + let hpfar_el2: u64; + let mpidr_el1: u64; + unsafe { asm!("mrs {:x}, esr_el2", out(reg) esr_el2) }; + unsafe { asm!("mrs {:x}, elr_el2", out(reg) elr_el2) }; + unsafe { asm!("mrs {:x}, far_el2", out(reg) far_el2) }; + unsafe { asm!("mrs {:x}, hpfar_el2", out(reg) hpfar_el2) }; + unsafe { asm!("mrs {:x}, mpidr_el1", out(reg) mpidr_el1) }; + if let Some(s) = unsafe { DEFAULT_SERIAL_PORT.as_ref() } { + unsafe { s.force_release_write_lock() }; + } + println!("ESR_EL2: {:#X}", esr_el2); + println!("ELR_EL2: {:#X}", elr_el2); + println!("FAR_EL2: {:#X}", far_el2); + println!("HPFAR_EL2: {:#X}", hpfar_el2); + println!("MPIDR_EL1: {:#X}", mpidr_el1); + println!("Registers: {:#X?}", s_r); + panic!("{}", f) +} + global_asm!( " .section .text @@ -194,33 +262,13 @@ s_error_current_spx: .balign 0x080 synchronous_lower_aa64: // up to 32 instructions - stp x1, x0, [sp, #-16]! - stp x3, x2, [sp, #-16]! - stp x5, x4, [sp, #-16]! - stp x7, x6, [sp, #-16]! - stp x9, x8, [sp, #-16]! - stp x11, x10, [sp, #-16]! - stp x13, x12, [sp, #-16]! - stp x15, x14, [sp, #-16]! - stp x17, x16, [sp, #-16]! - stp x19, x18, [sp, #-16]! - stp x30, x29, [sp, #-16]! + b synchronous_lower_aa64_save_registers +synchronous_lower_aa64_1: mov x29, sp mov x0, sp bl synchronous_exception_handler mov sp, x29 - ldp x30, x29, [sp], #16 - ldp x19, x18, [sp], #16 - ldp x17, x16, [sp], #16 - ldp x15, x14, [sp], #16 - ldp x13, x12, [sp], #16 - ldp x11, x10, [sp], #16 - ldp x9, x8, [sp], #16 - ldp x7, x6, [sp], #16 - ldp x5, x4, [sp], #16 - ldp x3, x2, [sp], #16 - ldp x1, x0, [sp], #16 - eret + b lower_aa64_restore_registers_and_eret .balign 0x080 irq_lower_aa64: @@ -233,33 +281,13 @@ fiq_lower_aa64: .balign 0x080 s_error_lower_aa64: // up to 32 instructions - stp x1, x0, [sp, #-16]! - stp x3, x2, [sp, #-16]! - stp x5, x4, [sp, #-16]! - stp x7, x6, [sp, #-16]! - stp x9, x8, [sp, #-16]! - stp x11, x10, [sp, #-16]! - stp x13, x12, [sp, #-16]! - stp x15, x14, [sp, #-16]! - stp x17, x16, [sp, #-16]! - stp x19, x18, [sp, #-16]! - stp x30, x29, [sp, #-16]! + b s_error_lower_aa64_save_registers +s_error_lower_aa64_1: mov x29, sp mov x0, sp bl s_error_exception_handler mov sp, x29 - ldp x30, x29, [sp], #16 - ldp x19, x18, [sp], #16 - ldp x17, x16, [sp], #16 - ldp x15, x14, [sp], #16 - ldp x13, x12, [sp], #16 - ldp x11, x10, [sp], #16 - ldp x9, x8, [sp], #16 - ldp x7, x6, [sp], #16 - ldp x5, x4, [sp], #16 - ldp x3, x2, [sp], #16 - ldp x1, x0, [sp], #16 - eret + b lower_aa64_restore_registers_and_eret .balign 0x080 synchronous_lower_aa32: @@ -278,3 +306,97 @@ s_error_lower_aa32: nop " ); + +global_asm!(" +synchronous_lower_aa64_save_registers: + sub sp, sp, {SR_SIZE} + stp x30, xzr, [sp, #( 15 * 16)] + stp x28, x29, [sp, #( 14 * 16)] + stp x26, x27, [sp, #( 13 * 16)] + stp x24, x25, [sp, #( 12 * 16)] + stp x22, x23, [sp, #( 11 * 16)] + stp x20, x21, [sp, #( 10 * 16)] + stp x18, x19, [sp, #( 9 * 16)] + stp x16, x17, [sp, #( 8 * 16)] + stp x14, x15, [sp, #( 7 * 16)] + stp x12, x13, [sp, #( 6 * 16)] + stp x10, x11, [sp, #( 5 * 16)] + stp x8, x9, [sp, #( 4 * 16)] + stp x6, x7, [sp, #( 3 * 16)] + stp x4, x5, [sp, #( 2 * 16)] + stp x2, x3, [sp, #( 1 * 16)] + stp x0, x1, [sp, #( 0 * 16)] + mrs x0, spsr_el2 + ubfx x0, x0, #0, #4 // and x0, x0, #0b1111 + cmp x0, #0b0101 // EL1h + b.ne 1f + mrs x0, sp_el1 + str x0, [sp, #( 15 * 16 + 8)] + b synchronous_lower_aa64_1 +1: + mrs x0, sp_el0 + str x0, [sp, #( 15 * 16 + 8)] + b synchronous_lower_aa64_1 + +s_error_lower_aa64_save_registers: + sub sp, sp, {SR_SIZE} + stp x30, xzr, [sp, #( 15 * 16)] + stp x28, x29, [sp, #( 14 * 16)] + stp x26, x27, [sp, #( 13 * 16)] + stp x24, x25, [sp, #( 12 * 16)] + stp x22, x23, [sp, #( 11 * 16)] + stp x20, x21, [sp, #( 10 * 16)] + stp x18, x19, [sp, #( 9 * 16)] + stp x16, x17, [sp, #( 8 * 16)] + stp x14, x15, [sp, #( 7 * 16)] + stp x12, x13, [sp, #( 6 * 16)] + stp x10, x11, [sp, #( 5 * 16)] + stp x8, x9, [sp, #( 4 * 16)] + stp x6, x7, [sp, #( 3 * 16)] + stp x4, x5, [sp, #( 2 * 16)] + stp x2, x3, [sp, #( 1 * 16)] + stp x0, x1, [sp, #( 0 * 16)] + mrs x0, spsr_el2 + ubfx x0, x0, #0, #4 // and x0, x0, #0b1111 + cmp x0, #0b0101 // EL1h + b.ne 1f + mrs x0, sp_el1 + str x0, [sp, #( 15 * 16 + 8)] + b s_error_lower_aa64_1 +1: + mrs x0, sp_el0 + str x0, [sp, #( 15 * 16 + 8)] + b s_error_lower_aa64_1 + + +lower_aa64_restore_registers_and_eret: + mrs x0, spsr_el2 + ubfx x0, x0, #0, #4 // and x0, x0, #0b1111 + cmp x0, #0b0101 // EL1h + b.ne 1f + ldr x0, [sp, #( 15 * 16 + 8)] + msr sp_el1, x0 + b 2f +1: + ldr x0, [sp, #( 15 * 16 + 8)] + msr sp_el0, x0 +2: + ldp x30, xzr, [sp, #( 15 * 16)] + ldp x28, x29, [sp, #( 14 * 16)] + ldp x26, x27, [sp, #( 13 * 16)] + ldp x24, x25, [sp, #( 12 * 16)] + ldp x22, x23, [sp, #( 11 * 16)] + ldp x20, x21, [sp, #( 10 * 16)] + ldp x18, x19, [sp, #( 9 * 16)] + ldp x16, x17, [sp, #( 8 * 16)] + ldp x14, x15, [sp, #( 7 * 16)] + ldp x12, x13, [sp, #( 6 * 16)] + ldp x10, x11, [sp, #( 5 * 16)] + ldp x8, x9, [sp, #( 4 * 16)] + ldp x6, x7, [sp, #( 3 * 16)] + ldp x4, x5, [sp, #( 2 * 16)] + ldp x2, x3, [sp, #( 1 * 16)] + ldp x0, x1, [sp, #( 0 * 16)] + add sp, sp, {SR_SIZE} + eret +", SR_SIZE = const core::mem::size_of::()); diff --git a/src/hypervisor_kernel/src/memory_hook.rs b/src/hypervisor_kernel/src/memory_hook.rs new file mode 100644 index 0000000..013f23f --- /dev/null +++ b/src/hypervisor_kernel/src/memory_hook.rs @@ -0,0 +1,201 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +/// +/// Memory Hook Handler +/// +use crate::StoredRegisters; + +#[allow(dead_code)] +pub enum LoadHookResult { + PassThrough, + Data(u64), +} + +#[allow(dead_code)] +pub enum StoreHookResult { + PassThrough, + AlternativeData(u64), + Cancel, +} + +pub type LoadAccessHandler = fn( + accessing_memory_address: usize, + stored_registers: &mut StoredRegisters, + access_size: u8, + is_64bit_register: bool, + is_sign_extend_required: bool, +) -> Result; + +pub type StoreAccessHandler = fn( + accessing_memory_address: usize, + stored_registers: &mut StoredRegisters, + access_size: u8, + data: u64, +) -> Result; + +#[derive(Clone, Copy)] +pub struct LoadAccessHandlerEntry { + target_address: usize, + range: usize, + handler: LoadAccessHandler, + //busy_flag: AtomicBool +} + +#[derive(Clone, Copy)] +pub struct StoreAccessHandlerEntry { + target_address: usize, + range: usize, + handler: StoreAccessHandler, + //busy_flag: AtomicBool +} + +impl LoadAccessHandlerEntry { + pub const fn new(target_address: usize, range: usize, handler: LoadAccessHandler) -> Self { + Self { + target_address, + range, + handler, + } + } + + pub const fn get_target_address(&self) -> usize { + self.target_address + } + + pub fn set_target_address(&mut self, address: usize) { + self.target_address = address; + } +} + +impl StoreAccessHandlerEntry { + pub const fn new(target_address: usize, range: usize, handler: StoreAccessHandler) -> Self { + Self { + target_address, + range, + handler, + } + } + + pub const fn get_target_address(&self) -> usize { + self.target_address + } + + pub fn set_target_address(&mut self, address: usize) { + self.target_address = address; + } +} + +pub static mut LOAD_HANDLER_LIST: [Option; 256] = [None; 256]; +static mut NUM_OF_LOAD_HANDLER_ENABLED_ENTRIES: usize = 0; +pub static mut STORE_HANDLER_LIST: [Option; 256] = [None; 256]; +static mut NUM_OF_STORE_HANDLER_ENABLED_ENTRIES: usize = 0; + +pub fn add_memory_load_hook_handler(entry: LoadAccessHandlerEntry) -> Result<(), ()> { + for e in unsafe { &mut LOAD_HANDLER_LIST } { + if e.is_none() { + *e = Some(entry); + unsafe { NUM_OF_LOAD_HANDLER_ENABLED_ENTRIES += 1 }; + return Ok(()); + } + } + return Err(()); +} + +pub fn add_memory_store_hook_handler(entry: StoreAccessHandlerEntry) -> Result<(), ()> { + for e in unsafe { &mut STORE_HANDLER_LIST } { + if e.is_none() { + *e = Some(entry); + unsafe { NUM_OF_STORE_HANDLER_ENABLED_ENTRIES += 1 }; + return Ok(()); + } + } + return Err(()); +} + +pub fn remove_memory_load_hook_handler(entry: LoadAccessHandlerEntry) -> Result<(), ()> { + for e in unsafe { &mut LOAD_HANDLER_LIST } { + if let Some(t_e) = e { + if t_e.target_address == entry.target_address && t_e.range == entry.range { + *e = None; + unsafe { NUM_OF_LOAD_HANDLER_ENABLED_ENTRIES -= 1 }; + return Ok(()); + } + } + } + return Err(()); +} + +pub fn remove_memory_store_hook_handler(entry: StoreAccessHandlerEntry) -> Result<(), ()> { + for e in unsafe { &mut STORE_HANDLER_LIST } { + if let Some(t_e) = e { + if t_e.target_address == entry.target_address && t_e.range == entry.range { + *e = None; + unsafe { NUM_OF_STORE_HANDLER_ENABLED_ENTRIES -= 1 }; + return Ok(()); + } + } + } + return Err(()); +} + +pub fn memory_load_hook_handler( + accessing_memory_address: usize, + stored_registers: &mut StoredRegisters, + access_size: u8, + is_64bit_register: bool, + is_sign_extend_required: bool, +) -> Result { + let mut num_of_check_entries = 0; + for e in unsafe { &LOAD_HANDLER_LIST } { + if let Some(handler_entry) = e { + if (handler_entry.target_address..(handler_entry.target_address + handler_entry.range)) + .contains(&accessing_memory_address) + { + return (handler_entry.handler)( + accessing_memory_address, + stored_registers, + access_size, + is_64bit_register, + is_sign_extend_required, + ); + } + num_of_check_entries += 1; + if num_of_check_entries == unsafe { NUM_OF_LOAD_HANDLER_ENABLED_ENTRIES } { + break; + } + } + } + return Ok(LoadHookResult::PassThrough); +} + +pub fn memory_store_hook_handler( + accessing_memory_address: usize, + stored_registers: &mut StoredRegisters, + access_size: u8, + data: u64, +) -> Result { + let mut num_of_check_entries = 0; + for e in unsafe { &STORE_HANDLER_LIST } { + if let Some(handler_entry) = e { + if (handler_entry.target_address..(handler_entry.target_address + handler_entry.range)) + .contains(&accessing_memory_address) + { + return (handler_entry.handler)( + accessing_memory_address, + stored_registers, + access_size, + data, + ); + } + num_of_check_entries += 1; + if num_of_check_entries == unsafe { NUM_OF_STORE_HANDLER_ENABLED_ENTRIES } { + break; + } + } + } + return Ok(StoreHookResult::PassThrough); +} diff --git a/src/hypervisor_kernel/src/multi_core.rs b/src/hypervisor_kernel/src/multi_core.rs index c11b6e0..57d7ef7 100644 --- a/src/hypervisor_kernel/src/multi_core.rs +++ b/src/hypervisor_kernel/src/multi_core.rs @@ -1,14 +1,21 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! MultiCore Handling Functions //! -use crate::{allocate_memory, StoredRegisters}; +use crate::{allocate_memory, handler_panic, StoredRegisters}; use crate::psci::{call_psci_function, PsciFunctionId, PsciReturnCode}; use common::cpu::convert_virtual_address_to_physical_address_el2_read; use common::STACK_PAGES; +use core::arch::asm; use core::sync::atomic::{AtomicU64, Ordering}; #[repr(C, align(16))] @@ -30,6 +37,23 @@ struct HypervisorRegisters { complete_flag: AtomicU64, } +static mut REGISTER_BUFFER: HypervisorRegisters = HypervisorRegisters { + stack_address: 0, + cnthctl_el2: 0, + cptr_el2: 0, + hcr_el2: 0, + vttbr_el2: 0, + ttbr0_el2: 0, + mair_el2: 0, + tcr_el2: 0, + vtcr_el2: 0, + sctlr_el2: 0, + vbar_el2: 0, + el1_entry_point: 0, + el1_context_id: 0, + complete_flag: AtomicU64::new(1), +}; + pub fn setup_new_cpu(regs: &mut StoredRegisters) { let stack_address: u64 = allocate_memory(STACK_PAGES).expect("Failed to alloc stack for new CPU.") as u64; @@ -56,32 +80,50 @@ pub fn setup_new_cpu(regs: &mut StoredRegisters) { asm!("mrs {:x}, vbar_el2", out(reg) vbar_el2); } - let hypervisor_registers = HypervisorRegisters { - stack_address, - cnthctl_el2, - cptr_el2, - hcr_el2, - vttbr_el2, - ttbr0_el2, - mair_el2, - tcr_el2, - vtcr_el2, - sctlr_el2, - vbar_el2, - el1_entry_point: regs.x2, - el1_context_id: regs.x3, - complete_flag: AtomicU64::new(0), - }; + /* Aquire REGISTER_BUFFER's lock */ + loop { + let mut current; + loop { + current = unsafe { REGISTER_BUFFER.complete_flag.load(Ordering::Relaxed) }; + if current != 0 { + break; + } + core::hint::spin_loop(); + } + if unsafe { + REGISTER_BUFFER + .complete_flag + .compare_exchange_weak(current, 0, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } { + break; + } + } + unsafe { + REGISTER_BUFFER.stack_address = stack_address; + REGISTER_BUFFER.cnthctl_el2 = cnthctl_el2; + REGISTER_BUFFER.cptr_el2 = cptr_el2; + REGISTER_BUFFER.hcr_el2 = hcr_el2; + REGISTER_BUFFER.vttbr_el2 = vttbr_el2; + REGISTER_BUFFER.ttbr0_el2 = ttbr0_el2; + REGISTER_BUFFER.mair_el2 = mair_el2; + REGISTER_BUFFER.tcr_el2 = tcr_el2; + REGISTER_BUFFER.vtcr_el2 = vtcr_el2; + REGISTER_BUFFER.sctlr_el2 = sctlr_el2; + REGISTER_BUFFER.vbar_el2 = vbar_el2; + REGISTER_BUFFER.el1_entry_point = regs.x2; + REGISTER_BUFFER.el1_context_id = regs.x3; + } let hypervisor_registers_real_address = convert_virtual_address_to_physical_address_el2_read( - &hypervisor_registers as *const _ as usize, + unsafe { ®ISTER_BUFFER } as *const _ as usize, ) .expect("Failed to convert virtual address to real address"); let cpu_boot_address_real_address = convert_virtual_address_to_physical_address_el2_read(cpu_boot as *const fn() as usize) .expect("Failed to convert virtual address to real address"); - println!("{:#X?}", hypervisor_registers); + pr_debug!("{:#X?}", hypervisor_registers); regs.x0 = call_psci_function( PsciFunctionId::CpuOn, @@ -90,17 +132,14 @@ pub fn setup_new_cpu(regs: &mut StoredRegisters) { hypervisor_registers_real_address as u64, ); if regs.x0 as i32 != PsciReturnCode::Success as i32 { - panic!( + handler_panic!( + regs, "Failed to on the cpu (MPIDR: {:#X}): {:?}", regs.x1, PsciReturnCode::try_from(regs.x0 as i32) ); } - println!("Wait for completing the initialization."); - while hypervisor_registers.complete_flag.load(Ordering::Relaxed) == 0 { - core::hint::spin_loop() - } - println!("The initialization completed."); + pr_debug!("The initialization completed."); } /* cpu_boot must use position-relative code */ diff --git a/src/hypervisor_kernel/src/paging.rs b/src/hypervisor_kernel/src/paging.rs new file mode 100644 index 0000000..76e9914 --- /dev/null +++ b/src/hypervisor_kernel/src/paging.rs @@ -0,0 +1,627 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +//! +//! Paging +//! + +use super::allocate_memory; + +use common::cpu::{ + flush_tlb_el2, flush_tlb_vmalls12e1, get_tcr_el2, get_ttbr0_el2, get_vtcr_el2, get_vttbr_el2, + set_tcr_el2, TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H, TCR_EL2_T0SZ_WITHOUT_E2H, VTCR_EL2_SL0, + VTCR_EL2_SL0_BITS_OFFSET, VTCR_EL2_T0SZ, VTCR_EL2_T0SZ_BITS_OFFSET, +}; +use common::paging::{ + calculate_number_of_concatenated_page_tables, create_attributes_for_stage_1, + create_attributes_for_stage_2, extract_output_address, + get_initial_page_table_level_and_bits_to_shift, + get_suitable_memory_attribute_index_from_mair_el2, is_block_descriptor, + is_descriptor_table_or_level_3_descriptor, table_level_to_table_shift, + MEMORY_PERMISSION_EXECUTABLE_BIT, MEMORY_PERMISSION_READABLE_BIT, + MEMORY_PERMISSION_WRITABLE_BIT, PAGE_DESCRIPTORS_CONTIGUOUS, PAGE_DESCRIPTORS_NT, + PAGE_TABLE_SIZE, TTBR, +}; +use common::{PAGE_SHIFT, PAGE_SIZE, STAGE_2_PAGE_SHIFT, STAGE_2_PAGE_SIZE}; + +/// Map physical Address Recursively +/// +/// permission: Bit0:Readable, Bit1: Writable, Bit2: Executable +fn map_address_recursive( + physical_address: &mut usize, + virtual_address: &mut usize, + num_of_remaining_pages: &mut usize, + table_address: usize, + table_level: i8, + permission: u8, + memory_attribute: u8, /* MemAttr */ + t0sz: u8, +) -> Result<(), ()> { + let shift_level = table_level_to_table_shift(PAGE_SHIFT, table_level); + let mut table_index = (*virtual_address >> shift_level) & 0x1FF; + + if table_level == 3 { + let current_table = unsafe { + &mut *(table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + let num_of_pages = if *num_of_remaining_pages + table_index > 512 { + 512 - table_index + } else { + *num_of_remaining_pages + }; + let attributes = create_attributes_for_stage_1(permission, memory_attribute, false); + + for index in table_index..(table_index + num_of_pages) { + current_table[index] = *physical_address as u64 | attributes; + *physical_address += PAGE_SIZE; + *virtual_address += PAGE_SIZE; + } + *num_of_remaining_pages -= num_of_pages; + return Ok(()); + } + let current_table = unsafe { + &mut *(table_address as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + + while *num_of_remaining_pages != 0 { + pr_debug!( + "{:#X}: Level{}'s Table Index: {:#X}", + *virtual_address, + table_level, + table_index + ); + if table_index >= 512 { + break; + } + let target_descriptor = &mut current_table[table_index]; + + if table_level > 1 + && (*physical_address & ((1usize << shift_level) - 1)) == 0 + && (*virtual_address & ((1usize << shift_level) - 1)) == 0 + && *num_of_remaining_pages >= 512usize.pow((3 - table_level) as u32) + { + pr_debug!( + "Creating BlockEntry: VA: {:#X}, PA: {:#X}, TableLevel: {}", + *virtual_address, + *physical_address, + table_level + ); + if is_descriptor_table_or_level_3_descriptor(*target_descriptor) { + pr_debug!( + "PageTable:({:#X}) will be deleted.", + extract_output_address(*target_descriptor, PAGE_SHIFT) + ); + /* TODO: free page table */ + } + let attributes = create_attributes_for_stage_1(permission, memory_attribute, true); + *target_descriptor = *physical_address as u64 | attributes; + *physical_address += 1 << shift_level; + *virtual_address += 1 << shift_level; + *num_of_remaining_pages -= 512usize.pow((3 - table_level) as u32); + } else { + let mut created_entry: Option = None; + + if !is_descriptor_table_or_level_3_descriptor(*target_descriptor) { + let allocated_table_address = + allocate_page_table_for_stage_1(table_level, t0sz, false)?; + + if is_block_descriptor(*target_descriptor) { + pr_debug!( + "Convert the block descriptor({:#b}) to table descriptor", + *target_descriptor + ); + + let mut block_physical_address = + extract_output_address(*target_descriptor, PAGE_SHIFT); + let mut descriptor_attribute = + *target_descriptor ^ (block_physical_address as u64); + let next_level_page = unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + + if table_level + 1 == 3 { + descriptor_attribute |= 0b11; + descriptor_attribute &= !PAGE_DESCRIPTORS_NT; + } + + for e in next_level_page { + *e = (block_physical_address as u64) | descriptor_attribute; + block_physical_address += 1 << (shift_level - 9); + } + } else { + /* set_mem */ + for e in unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + } { + *e = 0; + } + } + + /* TODO: 52bit OA support */ + created_entry = Some(allocated_table_address as u64 | 0b11); + pr_debug!("Allocated: {:#X}", allocated_table_address); + } + map_address_recursive( + physical_address, + virtual_address, + num_of_remaining_pages, + extract_output_address(created_entry.unwrap_or(*target_descriptor), PAGE_SHIFT), + table_level + 1, + permission, + memory_attribute, + t0sz, + )?; + + if let Some(new_descriptor) = created_entry { + *target_descriptor = new_descriptor; + } + } + table_index += 1; + } + return Ok(()); +} + +pub fn map_address( + mut physical_address: usize, + mut virtual_address: usize, + size: usize, + readable: bool, + writable: bool, + executable: bool, + is_device: bool, +) -> Result<(), ()> { + if (physical_address & ((1usize << PAGE_SHIFT) - 1)) != 0 { + println!("Physical Address is not aligned."); + return Err(()); + } + let aligned_size = if (size & ((1usize << PAGE_SHIFT) - 1)) != 0 { + (size & ((1usize << PAGE_SHIFT) - 1)) + PAGE_SIZE + } else { + size + }; + let mut num_of_needed_pages = aligned_size >> PAGE_SHIFT; + let tcr_el2 = get_tcr_el2(); + + let mut tcr_el2_t0sz = + ((tcr_el2 & TCR_EL2_T0SZ_WITHOUT_E2H) >> TCR_EL2_T0SZ_BITS_OFFSET_WITHOUT_E2H) as u32; + + let (table_level, _) = get_initial_page_table_level_and_bits_to_shift(tcr_el2); + pr_debug!("Initial Page Table Level: {}", table_level); + let min_t0sz = (virtual_address + size).leading_zeros(); + if min_t0sz < tcr_el2_t0sz { + pr_debug!( + "TCR_EL2::T0SZ will be changed to {} from {}, this may cause panic.", + min_t0sz, + tcr_el2_t0sz + ); + let new_tcr_el2 = (tcr_el2 ^ tcr_el2_t0sz as u64) | min_t0sz as u64; + let (new_table_level, _) = get_initial_page_table_level_and_bits_to_shift(new_tcr_el2); + if new_table_level != table_level { + panic!( + "Paging Table Level is to be changed. {} => {}", + table_level, new_table_level + ); + /* TODO: adjust process */ + } else { + set_tcr_el2(new_tcr_el2); + tcr_el2_t0sz = min_t0sz; + } + } + + map_address_recursive( + &mut physical_address, + &mut virtual_address, + &mut num_of_needed_pages, + TTBR::new(get_ttbr0_el2()).get_base_address(), + table_level, + (readable as u8) << MEMORY_PERMISSION_READABLE_BIT + | (writable as u8) << MEMORY_PERMISSION_WRITABLE_BIT + | (executable as u8) << MEMORY_PERMISSION_EXECUTABLE_BIT, + get_suitable_memory_attribute_index_from_mair_el2(is_device), + tcr_el2_t0sz as u8, + )?; + + if num_of_needed_pages != 0 { + println!( + "Failed to map address(remaining_pages:{} != 0", + num_of_needed_pages + ); + return Err(()); + } + flush_tlb_el2(); + pr_debug!( + "Mapped {:#X} Bytes({} Pages)", + aligned_size, + aligned_size >> PAGE_SHIFT + ); + return Ok(()); +} + +/// Map physical Address Recursively into Stage2 translation table +/// +/// permission: Bit0:Readable, Bit1: Writable, Bit2: Executable +fn map_address_recursive_stage2( + physical_address: &mut usize, + virtual_address: &mut usize, + num_of_remaining_pages: &mut usize, + table_address: usize, + table_level: i8, + is_unmap: bool, + permission: u8, + concatenated_tables: u8, + t0sz: u8, + is_dummy_page: bool, +) -> Result<(), ()> { + let shift_level = table_level_to_table_shift(STAGE_2_PAGE_SHIFT, table_level); + let mut table_index = + (*virtual_address >> shift_level) & (0x200 * (concatenated_tables as usize) - 1); + + if table_level == 3 { + let table_len = + (PAGE_TABLE_SIZE * (concatenated_tables as usize)) / core::mem::size_of::(); + + let current_table = + unsafe { core::slice::from_raw_parts_mut(table_address as *mut u64, table_len) }; + + let num_of_pages = if *num_of_remaining_pages + table_index > table_len { + table_len - table_index + } else { + *num_of_remaining_pages + }; + if STAGE_2_PAGE_SIZE == 0x1000 { + let contiguous_first_entry = &mut current_table[table_index & !0xF]; + *contiguous_first_entry &= !PAGE_DESCRIPTORS_CONTIGUOUS; + } + let attributes = create_attributes_for_stage_2(permission, is_dummy_page, is_unmap, false); + let end_index = table_index + num_of_pages; + for index in table_index..end_index { + if STAGE_2_PAGE_SIZE == 0x1000 + && (index & 0xF) == 0 + && !is_dummy_page + && (end_index - index) >= 16 + { + println!( + "Enable CONTIGUOUS_BIT(index: {:#X}, end_index: {:#X}", + index, end_index + ); + current_table[index] = + *physical_address as u64 | attributes | PAGE_DESCRIPTORS_CONTIGUOUS; + } else { + current_table[index] = *physical_address as u64 | attributes; + } + if !is_dummy_page { + *physical_address += STAGE_2_PAGE_SIZE; + } + *virtual_address += STAGE_2_PAGE_SIZE; + } + *num_of_remaining_pages -= num_of_pages; + return Ok(()); + } + let current_table = unsafe { + core::slice::from_raw_parts_mut( + table_address as *mut u64, + (PAGE_TABLE_SIZE * concatenated_tables as usize) / core::mem::size_of::(), + ) + }; + + while *num_of_remaining_pages != 0 { + pr_debug!( + "{:#X}: Level{}'s Table Index: {:#X}", + *virtual_address, + table_level, + table_index + ); + if table_index >= (512 * concatenated_tables as usize) { + break; + } + let target_descriptor = &mut current_table[table_index]; + if !is_dummy_page + && table_level > 1 + && (*physical_address & ((1usize << shift_level) - 1)) == 0 + && (*virtual_address & ((1usize << shift_level) - 1)) == 0 + && *num_of_remaining_pages >= 512usize.pow((3 - table_level) as u32) + { + pr_debug!( + "Creating BlockEntry: VA: {:#X}, PA: {:#X}, TableLevel: {}(Stage 2)", + *virtual_address, + *physical_address, + table_level + ); + if is_descriptor_table_or_level_3_descriptor(*target_descriptor) { + pr_debug!( + "PageTable:({:#X}) will be deleted.", + extract_output_address(*target_descriptor, STAGE_2_PAGE_SHIFT) + ); + /* TODO: free page table */ + } + let attributes = + create_attributes_for_stage_2(permission, is_dummy_page, is_unmap, true); + + *target_descriptor = *physical_address as u64 | attributes; + + *physical_address += 1 << shift_level; + *virtual_address += 1 << shift_level; + *num_of_remaining_pages -= 512usize.pow((3 - table_level) as u32); + } else { + let mut created_entry: Option = None; + + if *target_descriptor & 0b11 != 0b11 { + let allocated_table_address = + allocate_page_table_for_stage_2(table_level, t0sz, false, 1)?; + + if *target_descriptor & 0b11 == 0b01 { + pr_debug!( + "Convert the block descriptor({:#b}) to table descriptor", + *target_descriptor + ); + + let mut block_physical_address = + extract_output_address(*target_descriptor, STAGE_2_PAGE_SHIFT); + let mut descriptor_attribute = + *target_descriptor ^ (block_physical_address as u64); + let next_level_page = unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + }; + + if table_level + 1 == 3 { + descriptor_attribute |= 0b11; + descriptor_attribute &= !PAGE_DESCRIPTORS_NT; /* Clear nT Bit */ + } + + for e in next_level_page { + *e = (block_physical_address as u64) | descriptor_attribute; + block_physical_address += 1 << (shift_level - 9); + } + } else { + /* set_mem */ + for e in unsafe { + &mut *(allocated_table_address + as *mut [u64; PAGE_TABLE_SIZE / core::mem::size_of::()]) + } { + *e = 0; + } + } + + /* TODO: 52bit OA support */ + created_entry = Some(allocated_table_address as u64 | 0b11); + pr_debug!("Allocated: {:#X}", allocated_table_address); + } + map_address_recursive_stage2( + physical_address, + virtual_address, + num_of_remaining_pages, + extract_output_address( + created_entry.unwrap_or(*target_descriptor), + STAGE_2_PAGE_SHIFT, + ), + table_level + 1, + is_unmap, + permission, + 1, + t0sz, + is_dummy_page, + )?; + if let Some(d) = created_entry { + *target_descriptor = d; + } + } + table_index += 1; + } + flush_tlb_vmalls12e1(); + return Ok(()); +} + +#[allow(dead_code)] +pub fn map_dummy_page_into_vttbr_el2( + mut virtual_address: usize, + size: usize, + mut dummy_page: usize, /*4 KiB Page Physical Address*/ +) -> Result<(), ()> { + if (size & ((1usize << STAGE_2_PAGE_SHIFT) - 1)) != 0 { + println!("Size({:#X}) is not aligned.", size); + return Err(()); + } + let mut num_of_needed_pages = size >> STAGE_2_PAGE_SHIFT; + let vtcr_el2 = get_vtcr_el2(); + let vtcr_el2_sl0 = ((vtcr_el2 & VTCR_EL2_SL0) >> VTCR_EL2_SL0_BITS_OFFSET) as u8; + let vtcr_el2_t0sz = ((vtcr_el2 & VTCR_EL2_T0SZ) >> VTCR_EL2_T0SZ_BITS_OFFSET) as u8; + let initial_look_up_level: i8 = match vtcr_el2_sl0 { + 0b00 => 2, + 0b01 => 1, + 0b10 => 0, + 0b11 => 3, + _ => unreachable!(), + }; + + let original_dummy_page = dummy_page; + map_address_recursive_stage2( + &mut dummy_page, + &mut virtual_address, + &mut num_of_needed_pages, + TTBR::new(get_vttbr_el2()).get_base_address(), + initial_look_up_level, + false, + (1 << MEMORY_PERMISSION_READABLE_BIT) + | (1 << MEMORY_PERMISSION_WRITABLE_BIT) + | (1 << MEMORY_PERMISSION_EXECUTABLE_BIT), + calculate_number_of_concatenated_page_tables(vtcr_el2_t0sz, initial_look_up_level), + vtcr_el2_t0sz, + true, + )?; + + assert_eq!(num_of_needed_pages, 0); + assert_eq!(original_dummy_page, dummy_page); + return Ok(()); +} + +/// VTTBR_EL2の該当アドレス範囲をトラップできるようにする。 +/// 現在はInitial Table Levelは0までのみの対応 +pub fn add_memory_access_trap( + mut address: usize, + size: usize, + allow_read_access: bool, + allow_write_access: bool, +) -> Result<(), ()> { + if (size & ((1usize << STAGE_2_PAGE_SHIFT) - 1)) != 0 { + println!("Size({:#X}) is not aligned.", size); + return Err(()); + } + if allow_write_access && allow_read_access { + println!("Invalid access control."); + return Err(()); + } + let mut num_of_needed_pages = size >> STAGE_2_PAGE_SHIFT; + let vtcr_el2 = get_vtcr_el2(); + let vtcr_el2_sl0 = ((vtcr_el2 & VTCR_EL2_SL0) >> VTCR_EL2_SL0_BITS_OFFSET) as u8; + let vtcr_el2_t0sz = ((vtcr_el2 & VTCR_EL2_T0SZ) >> VTCR_EL2_T0SZ_BITS_OFFSET) as u8; + let initial_look_up_level: i8 = match vtcr_el2_sl0 { + 0b00 => 2, + 0b01 => 1, + 0b10 => 0, + 0b11 => 3, + _ => unreachable!(), + }; + + assert!(address < (1 << (64 - vtcr_el2_t0sz))); + + let mut physical_address = address; + map_address_recursive_stage2( + &mut physical_address, + &mut address, + &mut num_of_needed_pages, + TTBR::new(get_vttbr_el2()).get_base_address(), + initial_look_up_level, + false, + ((allow_read_access as u8) << MEMORY_PERMISSION_READABLE_BIT) + | ((allow_write_access as u8) << MEMORY_PERMISSION_WRITABLE_BIT), + calculate_number_of_concatenated_page_tables(vtcr_el2_t0sz, initial_look_up_level), + vtcr_el2_t0sz, + false, + )?; + assert_eq!(num_of_needed_pages, 0); + assert_eq!(physical_address, address); + pr_debug!("Unmapped {:#X} Bytes({} Pages)", size, size >> PAGE_SHIFT); + return Ok(()); +} + +/// VTTBR_EL2の該当アドレス範囲をトラップできないようにする。 +/// 現在はInitial Table Levelは0までのみの対応 +pub fn remove_memory_access_trap(mut address: usize, size: usize) -> Result<(), ()> { + if (size & ((1usize << STAGE_2_PAGE_SHIFT) - 1)) != 0 { + println!("Size({:#X}) is not aligned.", size); + return Err(()); + } + let mut num_of_needed_pages = size >> STAGE_2_PAGE_SHIFT; + let vtcr_el2 = get_vtcr_el2(); + let vtcr_el2_sl0 = ((vtcr_el2 & VTCR_EL2_SL0) >> VTCR_EL2_SL0_BITS_OFFSET) as u8; + let vtcr_el2_t0sz = ((vtcr_el2 & VTCR_EL2_T0SZ) >> VTCR_EL2_T0SZ_BITS_OFFSET) as u8; + let initial_look_up_level: i8 = match vtcr_el2_sl0 { + 0b00 => 2, + 0b01 => 1, + 0b10 => 0, + 0b11 => 3, + _ => unreachable!(), + }; + + assert!(address < (1 << (64 - vtcr_el2_t0sz))); + let mut physical_address = address; + + map_address_recursive_stage2( + &mut physical_address, + &mut address, + &mut num_of_needed_pages, + TTBR::new(get_vttbr_el2()).get_base_address(), + initial_look_up_level, + false, + (1 << MEMORY_PERMISSION_READABLE_BIT) + | (1 << MEMORY_PERMISSION_WRITABLE_BIT) + | (1 << MEMORY_PERMISSION_EXECUTABLE_BIT), + calculate_number_of_concatenated_page_tables(vtcr_el2_t0sz, initial_look_up_level), + vtcr_el2_t0sz, + false, + )?; + assert_eq!(num_of_needed_pages, 0); + pr_debug!("Unmapped {:#X} Bytes({} Pages)", size, size >> PAGE_SHIFT); + return Ok(()); +} + +/// Allocate page table for stage 1 with suitable address alignment +#[inline(always)] +fn allocate_page_table_for_stage_1( + look_up_level: i8, + t0sz: u8, + is_for_ttbr: bool, +) -> Result { + let table_address_alignment = if is_for_ttbr { + ((64 - ((PAGE_SHIFT - 3) as u8 * (4 - look_up_level) as u8) - t0sz).max(4)).min(12) + } else { + PAGE_SHIFT as u8 + }; + loop { + match allocate_memory(1) { + Ok(address) => { + if (address & ((1 << table_address_alignment) - 1)) != 0 { + pr_debug!( + "The table address is not alignment with {}, {:#X} will be wasted.", + table_address_alignment, + address + ); + /* TODO: アライメントを指定してメモリを確保できるようにし、無駄をなくす。 */ + } else { + return Ok(address); + } + } + Err(e) => { + println!("Failed to allocate memory for the paging table: {:?}", e); + return Err(e); + } + }; + } +} + +/// Allocate page table for stage 2 with suitable address alignment +#[inline(always)] +fn allocate_page_table_for_stage_2( + look_up_level: i8, + t0sz: u8, + is_for_ttbr: bool, + number_of_tables: u8, +) -> Result { + assert_ne!(number_of_tables, 0); + let table_address_alignment = if is_for_ttbr { + ((64 - ((PAGE_SHIFT - 3) as u8 * (4 - look_up_level) as u8) - t0sz).max(4)).min(12) + + (number_of_tables - 1) + } else { + assert_eq!(number_of_tables, 1); + STAGE_2_PAGE_SHIFT as u8 + }; + loop { + match allocate_memory(number_of_tables as usize) { + Ok(address) => { + if (address & ((1 << table_address_alignment) - 1)) != 0 { + pr_debug!( + "The table address is not alignment with {}, {:#X} will be wasted.", + table_address_alignment, + address + ); + /* TODO: アライメントを指定してメモリを確保できるようにし、無駄をなくす。 */ + if number_of_tables != 1 { + let _ = allocate_memory(1); + } + } else { + return Ok(address); + } + } + Err(e) => { + println!("Failed to allocate memory for the paging table: {:?}", e); + return Err(()); + } + }; + } +} diff --git a/src/hypervisor_kernel/src/panic.rs b/src/hypervisor_kernel/src/panic.rs index ffd5efd..aabbe80 100644 --- a/src/hypervisor_kernel/src/panic.rs +++ b/src/hypervisor_kernel/src/panic.rs @@ -1,9 +1,17 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Panic Handler //! use crate::serial_port::DEFAULT_SERIAL_PORT; +use common::cpu::halt_loop; + use core::panic; #[panic_handler] @@ -26,7 +34,5 @@ pub fn panic(info: &panic::PanicInfo) -> ! { ); println!("===== Dump complete ====="); - loop { - unsafe { asm!("wfi") } - } + halt_loop() } diff --git a/src/hypervisor_kernel/src/pci.rs b/src/hypervisor_kernel/src/pci.rs new file mode 100644 index 0000000..1f54aa4 --- /dev/null +++ b/src/hypervisor_kernel/src/pci.rs @@ -0,0 +1,84 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + +/// +/// PCI +/// +use crate::drivers; + +pub fn init_pci(ecam_address: usize, start_bus_number: u8, end_bus_number: u8) { + for bus in start_bus_number..=end_bus_number { + for device in 0..32 { + let vendor_id = get_configuration_space_data(ecam_address, bus, device, 0, 0, 2) as u16; + if vendor_id == 0xffff { + continue; + } + let device_id = get_configuration_space_data(ecam_address, bus, device, 0, 2, 2) as u16; + + println!( + "{:X}:{:X} VenderId: {:#X}, DeviceId: {:#X}", + bus, device, vendor_id, device_id + ); + /* TODO: 動的にハンドラ呼び出し */ + if vendor_id == drivers::i210::VENDOR_ID && device_id == drivers::i210::DEVICE_ID { + drivers::i210::setup_device(ecam_address, bus, device, 0); + } + for function in 1..8 { + let vendor_id = + get_configuration_space_data(ecam_address, bus, device, function, 0, 2) as u16; + if vendor_id == 0xffff { + continue; + } + println!( + " {:X}:{:X}:{:X} VenderId: {:#X}", + bus, device, function, vendor_id + ); + for i in 0..6 { + println!( + "BaseAddress{}: {:#X}", + i, + get_configuration_space_data( + ecam_address, + bus, + device, + function, + 0x10 + (i << 2), + 4 + ) + ); + } + } + } + } +} + +pub fn get_configuration_space_data( + base_address: usize, + bus: u8, + device: u8, + function: u8, + offset: usize, + size: u8, +) -> u32 { + let address = get_ecam_target_address(base_address, bus, device, function); + let aligned_offset = offset & !0b11; + let data = unsafe { *((address + aligned_offset) as *const u32) }; + let byte_offset = (offset & 0b11) as u8; + assert!(byte_offset + size <= 4); + return if size == 4 { + data + } else { + (data >> (byte_offset << 3)) & ((1 << (size << 3)) - 1) + }; +} + +pub fn get_ecam_target_address(base_address: usize, bus: u8, device: u8, function: u8) -> usize { + assert!(device < 32); + assert!(function < 8); + + base_address + + (((bus as usize) << 20) | ((device as usize) << 15) | ((function as usize) << 12)) +} diff --git a/src/hypervisor_kernel/src/psci.rs b/src/hypervisor_kernel/src/psci.rs index ba6ecdc..06ace45 100644 --- a/src/hypervisor_kernel/src/psci.rs +++ b/src/hypervisor_kernel/src/psci.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Power State Coordination Interface //! @@ -108,10 +114,10 @@ impl TryFrom for PsciReturnCode { } pub fn handle_psci_call(function_id: PsciFunctionId, stored_registers: &mut StoredRegisters) { - println!("PSCI Function Call: {:?}", function_id); + pr_debug!("PSCI Function Call: {:?}", function_id); if function_id == PsciFunctionId::CpuOn { - println!("CPU ON: MPIDR: {:#X}", stored_registers.x1); + pr_debug!("CPU ON: MPIDR: {:#X}", stored_registers.x1); setup_new_cpu(stored_registers); } else { secure_monitor_call( @@ -134,12 +140,6 @@ pub fn handle_psci_call(function_id: PsciFunctionId, stored_registers: &mut Stor &mut stored_registers.x16, &mut stored_registers.x17, ); - - if let Ok(error) = PsciReturnCode::try_from(stored_registers.x0 as i32) { - if error != PsciReturnCode::Success { - print!("Failed to Exec the PSCI function: {:?}", error); - } - } } } diff --git a/src/hypervisor_kernel/src/serial_port.rs b/src/hypervisor_kernel/src/serial_port.rs index a069ee3..1ca352b 100644 --- a/src/hypervisor_kernel/src/serial_port.rs +++ b/src/hypervisor_kernel/src/serial_port.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + mod arm_pl011; mod arm_sbsa_generic_uart; mod meson_gx_uart; @@ -152,5 +158,19 @@ macro_rules! print { #[macro_export] macro_rules! println { ($fmt:expr) => (print!(concat!($fmt,"\n"))); - ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"),$($arg)*)) + ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"),$($arg)*)); +} + +#[cfg(debug_assertions)] +#[macro_export] +macro_rules! pr_debug { + ($fmt:expr) => (print!(concat!($fmt,"\n"))); + ($fmt:expr, $($arg:tt)*) => (print!(concat!($fmt, "\n"),$($arg)*)); +} + +#[cfg(not(debug_assertions))] +#[macro_export] +macro_rules! pr_debug { + ($fmt:expr) => {}; + ($fmt:expr, $($arg:tt)*) => {}; } diff --git a/src/hypervisor_kernel/src/serial_port/arm_pl011.rs b/src/hypervisor_kernel/src/serial_port/arm_pl011.rs index 1d62acb..3a07017 100644 --- a/src/hypervisor_kernel/src/serial_port/arm_pl011.rs +++ b/src/hypervisor_kernel/src/serial_port/arm_pl011.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + use super::SerialPortDevice; use core::ptr; diff --git a/src/hypervisor_kernel/src/serial_port/arm_sbsa_generic_uart.rs b/src/hypervisor_kernel/src/serial_port/arm_sbsa_generic_uart.rs index 9e0b303..96f368e 100644 --- a/src/hypervisor_kernel/src/serial_port/arm_sbsa_generic_uart.rs +++ b/src/hypervisor_kernel/src/serial_port/arm_sbsa_generic_uart.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + use super::SerialPortDevice; use core::ptr; diff --git a/src/hypervisor_kernel/src/serial_port/meson_gx_uart.rs b/src/hypervisor_kernel/src/serial_port/meson_gx_uart.rs index 8caad1d..91217f7 100644 --- a/src/hypervisor_kernel/src/serial_port/meson_gx_uart.rs +++ b/src/hypervisor_kernel/src/serial_port/meson_gx_uart.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + use super::SerialPortDevice; use core::ptr; diff --git a/src/uefi/Cargo.toml b/src/uefi/Cargo.toml index a1c43a8..9feb49f 100644 --- a/src/uefi/Cargo.toml +++ b/src/uefi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "uefi" -version = "0.1.0" +version = "0.2.0" edition = "2021" [dependencies] diff --git a/src/uefi/Makefile b/src/uefi/Makefile index c957bdb..673508d 100644 --- a/src/uefi/Makefile +++ b/src/uefi/Makefile @@ -1,3 +1,9 @@ +# Copyright (c) 2022 RIKEN +# All rights reserved. +# +# This software is released under the MIT License. +# http://opensource.org/licenses/mit-license.php + CARGO = cargo clean: diff --git a/src/uefi/src/boot_service.rs b/src/uefi/src/boot_service.rs index 2740e92..2a44be4 100644 --- a/src/uefi/src/boot_service.rs +++ b/src/uefi/src/boot_service.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! UEFI Boot Services //! diff --git a/src/uefi/src/boot_service/memory_service.rs b/src/uefi/src/boot_service/memory_service.rs index d30e3b3..ed26006 100644 --- a/src/uefi/src/boot_service/memory_service.rs +++ b/src/uefi/src/boot_service/memory_service.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Memory Allocation Services of Boot Service //! diff --git a/src/uefi/src/file.rs b/src/uefi/src/file.rs index cd24404..70dd216 100644 --- a/src/uefi/src/file.rs +++ b/src/uefi/src/file.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! EFI Simple File System Protocol //! diff --git a/src/uefi/src/lib.rs b/src/uefi/src/lib.rs index 619ffbe..66273bc 100644 --- a/src/uefi/src/lib.rs +++ b/src/uefi/src/lib.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! UEFI //! diff --git a/src/uefi/src/loaded_image.rs b/src/uefi/src/loaded_image.rs index 1cc4858..9a7330a 100644 --- a/src/uefi/src/loaded_image.rs +++ b/src/uefi/src/loaded_image.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! Loaded Image //! diff --git a/src/uefi/src/output.rs b/src/uefi/src/output.rs index e77994b..bfc65aa 100644 --- a/src/uefi/src/output.rs +++ b/src/uefi/src/output.rs @@ -1,3 +1,9 @@ +// Copyright (c) 2022 RIKEN +// All rights reserved. +// +// This software is released under the MIT License. +// http://opensource.org/licenses/mit-license.php + //! //! EFI Output Protocol //!