diff --git a/arch/Kconfig b/arch/Kconfig index 5e3b96f414d497..cc9925e77747c9 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -125,6 +125,7 @@ config XTENSA select IRQ_OFFLOAD_NESTED if IRQ_OFFLOAD select ARCH_HAS_CODE_DATA_RELOCATION select ARCH_HAS_TIMING_FUNCTIONS + select ARCH_MEM_DOMAIN_DATA if USERSPACE imply ATOMIC_OPERATIONS_ARCH help Xtensa architecture diff --git a/arch/xtensa/CMakeLists.txt b/arch/xtensa/CMakeLists.txt index 133d74331d8e66..21de223d4ec201 100644 --- a/arch/xtensa/CMakeLists.txt +++ b/arch/xtensa/CMakeLists.txt @@ -1,4 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 +zephyr_syscall_header(${ZEPHYR_BASE}/include/zephyr/arch/xtensa/arch.h) set_property(GLOBAL PROPERTY PROPERTY_OUTPUT_FORMAT elf32-xtensa-le) add_subdirectory(core) + +if (CONFIG_XTENSA_INSECURE_USERSPACE) + message(WARNING " + This userspace implementation uses the window ABI this means that the kernel + will spill registers in behave of the userpsace. Use it carefully.") +endif() diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index a1517f17ed0976..326c7749e6b20f 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -113,8 +113,10 @@ config XTENSA_MMU bool "Xtensa MMU Support" default n select MMU + select ARCH_MEM_DOMAIN_SYNCHRONOUS_API if USERSPACE select XTENSA_SMALL_VECTOR_TABLE_ENTRY select KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK if XTENSA_RPO_CACHE + select CURRENT_THREAD_USE_NO_TLS if USERSPACE help Enable support for Xtensa Memory Management Unit. @@ -144,8 +146,18 @@ if XTENSA_MMU The bit shift number for the virtual address for Xtensa page table (PTEVADDR). + config XTENSA_MMU_NUM_L1_TABLES + int "Number of L1 page tables" + default 1 if !USERSPACE + default 4 + help + This option specifies the maximum number of traslation tables. + Translation tables are directly related to the number of + memory domains in the target, considering the kernel itself requires one. + config XTENSA_MMU_NUM_L2_TABLES int "Number of L2 page tables" + default 20 if USERSPACE default 10 help Each table can address up to 4MB memory address. @@ -159,6 +171,20 @@ if XTENSA_MMU endif # XTENSA_MMU +config XTENSA_SYSCALL_USE_HELPER + bool "Use userspace syscall helper" + default y if "$(ZEPHYR_TOOLCHAIN_VARIANT)" = "xcc-clang" + depends on USERSPACE + help + Use syscall helpers for passing more then 3 arguments. + This is a workaround for toolchains where they have + issue modeling register usage. + +config XTENSA_INSECURE_USERSPACE + bool + default y if USERSPACE + depends on XTENSA_MMU + endif # CPU_HAS_MMU endmenu diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index c6fffd4f5e181d..f3122c1a5504da 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -21,7 +21,9 @@ zephyr_library_sources_ifdef(CONFIG_XTENSA_ENABLE_BACKTRACE debug_helpers_asm.S) zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c) zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c) zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c) -zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU xtensa_mmu.c) +zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU ptables.c mmu.c) +zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S) +zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c) zephyr_library_sources_ifdef( CONFIG_KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK @@ -57,6 +59,7 @@ add_custom_command(OUTPUT ${CORE_ISA_DM} set(ZSR_H ${CMAKE_BINARY_DIR}/zephyr/include/generated/zsr.h) add_custom_command(OUTPUT ${ZSR_H} DEPENDS ${CORE_ISA_DM} COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/gen_zsr.py + $<$:--mmu> ${CORE_ISA_DM} ${ZSR_H}) add_custom_target(zsr_h DEPENDS ${ZSR_H}) add_dependencies(zephyr_interface zsr_h) diff --git a/arch/xtensa/core/README-MMU.txt b/arch/xtensa/core/README-MMU.txt new file mode 100644 index 00000000000000..499a251cdf2f2f --- /dev/null +++ b/arch/xtensa/core/README-MMU.txt @@ -0,0 +1,268 @@ +# Xtensa MMU Operation + +As with other elements of the architecture, paged virtual memory +management on Xtensa is somewhat unique. And there is similarly a +lack of introductory material available. This document is an attempt +to introduce the architecture at an overview/tutorial level, and to +describe Zephyr's specific implementation choices. + +## General TLB Operation + +The Xtensa MMU operates on top of a fairly conventional TLB cache. +The TLB stores virtual to physical translation for individual pages of +memory. It is partitioned into an automatically managed +4-way-set-associative bank of entries mapping 4k pages, and 3-6 +"special" ways storing mappings under OS control. Some of these are +for mapping pages larger than 4k, which Zephyr does not directly +support. A few are for bootstrap and initialization, and will be +discussed below. + +Like the L1 cache, the TLB is split into separate instruction and data +entries. Zephyr manages both as needed, but symmetrically. The +architecture technically supports separately-virtualized instruction +and data spaces, but the hardware page table refill mechanism (see +below) does not, and Zephyr's memory spaces are unified regardless. + +The TLB may be loaded with permissions and attributes controlling +cacheability, access control based on ring (i.e. the contents of the +RING field of the PS register) and togglable write and execute access. +Memory access, even with a matching TLB entry, may therefore create +Kernel/User exceptions as desired to enforce permissions choices on +userspace code. + +Live TLB entries are tagged with an 8-bit "ASID" value derived from +their ring field of the PTE that loaded them, via a simple translation +specified in the RASID special register. The intent is that each +non-kernel address space will get a separate ring 3 ASID set in RASID, +such that you can switch between them without a TLB flush. The ASID +value of ring zero is fixed at 1, it may not be changed. (An ASID +value of zero is used to tag an invalid/unmapped TLB entry at +initialization, but this mechanism isn't accessible to OS code except +in special circumstances, and in any case there is already an invalid +attribute value that can be used in a PTE). + +## Virtually-mapped Page Tables + +Xtensa has a unique (and, to someone exposed for the first time, +extremely confusing) "page table" format. The simplest was to begin +to explain this is just to describe the (quite simple) hardware +behavior: + +On a TLB miss, the hardware immediately does a single fetch (at ring 0 +privilege) from RAM by adding the "desired address right shifted by +10 bits with the bottom two bits set to zero" (i.e. the page frame +number in units of 4 bytes) to the value in the PTEVADDR special +register. If this load succeeds, then the word is treated as a PTE +with which to fill the TLB and use for a (restarted) memory access. +This is extremely simple (just one extra hardware state that does just +one thing the hardware can already do), and quite fast (only one +memory fetch vs. e.g. the 2-5 fetches required to walk a page table on +x86). + +This special "refill" fetch is otherwise identical to any other memory +access, meaning it too uses the TLB to translate from a virtual to +physical address. Which means that the page tables occupy a 4M region +of virtual, not physical, address space, in the same memory space +occupied by the running code. The 1024 pages in that range (not all +of which might be mapped in physical memory) are a linear array of +1048576 4-byte PTE entries, each describing a mapping for 4k of +virtual memory. Note especially that exactly one of those pages +contains the 1024 PTE entries for the 4M page table itself, pointed to +by PTEVADDR. + +Obviously, the page table memory being virtual means that the fetch +can fail: there are 1024 possible pages in a complete page table +covering all of memory, and the ~16 entry TLB clearly won't contain +entries mapping all of them. If we are missing a TLB entry for the +page translation we want (NOT for the original requested address, we +already know we're missing that TLB entry), the hardware has exactly +one more special trick: it throws a TLB Miss exception (there are two, +one each for instruction/data TLBs, but in Zephyr they operate +identically). + +The job of that exception handler is simply to ensure that the TLB has +an entry for the page table page we want. And the simplest way to do +that is to just load the faulting PTE as an address, which will then +go through the same refill process above. This second TLB fetch in +the exception handler may result in an invalid/inapplicable mapping +within the 4M page table region. This is an typical/expected runtime +fault, and simply indicates unmapped memory. The result is TLB miss +exception from within the TLB miss exception handler (i.e. while the +EXCM bit is set). This will produce a Double Exception fault, which +is handled by the OS identically to a general Kernel/User data access +prohibited exception. + +After the TLB refill exception, the original faulting instruction is +restarted, which retries the refill process, which succeeds in +fetching a new TLB entry, which is then used to service the original +memory access. (And may then result in yet another exception if it +turns out that the TLB entry doesn't permit the access requested, of +course.) + +## Special Cases + +The page-tables-specified-in-virtual-memory trick works very well in +practice. But it does have a chicken/egg problem with the initial +state. Because everything depends on state in the TLB, something +needs to tell the hardware how to find a physical address using the +TLB to begin the process. Here we exploit the separate +non-automatically-refilled TLB ways to store bootstrap records. + +First, note that the refill process to load a PTE requires that the 4M +space of PTE entries be resolvable by the TLB directly, without +requiring another refill. This 4M mapping is provided by a single +page of PTE entries (which itself lives in the 4M page table region!). +This page must always be in the TLB. + +Thankfully, for the data TLB Xtensa provides 3 special/non-refillable +ways (ways 7-9) with at least one 4k page mapping each. We can use +one of these to "pin" the top-level page table entry in place, +ensuring that a refill access will be able to find a PTE address. + +But now note that the load from that PTE address for the refill is +done in an exception handler. And running an exception handler +requires doing a fetch via the instruction TLB. And that obviously +means that the page(s) containing the exception handler must never +require a refill exception of its own. + +Ideally we would just pin the vector/handler page in the ITLB in the +same way we do for data, but somewhat inexplicably, Xtensa does not +provide 4k "pinnable" ways in the instruction TLB (frankly this seems +like a design flaw). + +Instead, we load ITLB entries for vector handlers via the refill +mechanism using the data TLB, and so need the refill mechanism for the +vector page to succeed always. The way to do this is to similarly pin +the page table page containing the (single) PTE for the vector page in +the data TLB, such that instruction fetches always find their TLB +mapping via refill, without requiring an exception. + +## Initialization + +Unlike most other architectures, Xtensa does not have a "disable" mode +for the MMU. Virtual address translation through the TLB is active at +all times. There therefore needs to be a mechanism for the CPU to +execute code before the OS is able to initialize a refillable page +table. + +The way Xtensa resolves this (on the hardware Zephyr supports, see the +note below) is to have an 8-entry set ("way 6") of 512M pages able to +cover all of memory. These 8 entries are initialized as valid, with +attributes specifying that they are accessible only to an ASID of 1 +(i.e. the fixed ring zero / kernel ASID), writable, executable, and +uncached. So at boot the CPU relies on these TLB entries to provide a +clean view of hardware memory. + +But that means that enabling page-level translation requires some +care, as the CPU will throw an exception ("multi hit") if a memory +access matches more than one live entry in the TLB. The +initialization algorithm is therefore: + +0. Start with a fully-initialized page table layout, including the + top-level "L1" page containing the mappings for the page table + itself. + +1. Ensure that the initialization routine does not cross a page + boundary (to prevent stray TLB refill exceptions), that it occupies + a separate 4k page than the exception vectors (which we must + temporarily double-map), and that it operates entirely in registers + (to avoid doing memory access at inopportune moments). + +2. Pin the L1 page table PTE into the data TLB. This creates a double + mapping condition, but it is safe as nothing will use it until we + start refilling. + +3. Pin the page table page containing the PTE for the TLB miss + exception handler into the data TLB. This will likewise not be + accessed until the double map condition is resolved. + +4. Set PTEVADDR appropriately. The CPU state to handle refill + exceptions is now complete, but cannot be used until we resolve the + double mappings. + +5. Disable the initial/way6 data TLB entries first, by setting them to + an ASID of zero. This is safe as the code being executed is not + doing data accesses yet (including refills), and will resolve the + double mapping conditions we created above. + +6. Disable the initial/way6 instruction TLBs second. The very next + instruction following the invalidation of the currently-executing + code page will then cause a TLB refill exception, which will work + normally because we just resolved the final double-map condition. + (Pedantic note: if the vector page and the currently-executing page + are in different 512M way6 pages, disable the mapping for the + exception handlers first so the trap from our current code can be + handled. Currently Zephyr doesn't handle this condition as in all + reasonable hardware these regions will be near each other) + +Note: there is a different variant of the Xtensa MMU architecture +where the way 5/6 pages are immutable, and specify a set of +unchangable mappings from the final 384M of memory to the bottom and +top of physical memory. The intent here would (presumably) be that +these would be used by the kernel for all physical memory and that the +remaining memory space would be used for virtual mappings. This +doesn't match Zephyr's architecture well, as we tend to assume +page-level control over physical memory (e.g. .text/.rodata is cached +but .data is not on SMP, etc...). And in any case we don't have any +such hardware to experiment with. But with a little address +translation we could support this. + +## ASID vs. Virtual Mapping + +The ASID mechanism in Xtensa works like other architectures, and is +intended to be used similarly. The intent of the design is that at +context switch time, you can simply change RADID and the page table +data, and leave any existing mappings in place in the TLB using the +old ASID value(s). So in the common case where you switch back, +nothing needs to be flushed. + +Unfortunately this runs afoul of the virtual mapping of the page +refill: data TLB entries storing the 4M page table mapping space are +stored at ASID 1 (ring 0), they can't change when the page tables +change! So this region naively would have to be flushed, which is +tantamount to flushing the entire TLB regardless (the TLB is much +smaller than the 1024-page PTE array). + +The resolution in Zephyr is to give each ASID its own PTEVADDR mapping +in virtual space, such that the page tables don't overlap. This is +expensive in virtual address space: assigning 4M of space to each of +the 256 ASIDs (actually 254 as 0 and 1 are never used by user access) +would take a full gigabyte of address space. Zephyr optimizes this a +bit by deriving a unique sequential ASID from the hardware address of +the statically allocated array of L1 page table pages. + +Note, obviously, that any change of the mappings within an ASID +(e.g. to re-use it for another memory domain, or just for any runtime +mapping change other than mapping previously-unmapped pages) still +requires a TLB flush, and always will. + +## SMP/Cache Interaction + +A final important note is that the hardware PTE refill fetch works +like any other CPU memory access, and in particular it is governed by +the cacheability attributes of the TLB entry through which it was +loaded. This means that if the page table entries are marked +cacheable, then the hardware TLB refill process will be downstream of +the L1 data cache on the CPU. If the physical memory storing page +tables has been accessed recently by the CPU (for a refill of another +page mapped within the same cache line, or to change the tables) then +the refill will be served from the data cache and not main memory. + +This may or may not be desirable depending on access patterns. It +lets the L1 data cache act as a "L2 TLB" for applications with a lot +of access variability. But it also means that the TLB entries end up +being stored twice in the same CPU, wasting transistors that could +presumably store other useful data. + +But it it also important to note that the L1 data cache on Xtensa is +incoherent! The cache being used for refill reflects the last access +on the current CPU only, and not of the underlying memory being +mapped. Page table changes in the data cache of one CPU will be +invisible to the data cache of another. There is no simple way of +notifying another CPU of changes to page mappings beyond doing +system-wide flushes on all cpus every time a memory domain is +modified. + +The result is that, when SMP is enabled, Zephyr must ensure that all +page table mappings in the system are set uncached. The OS makes no +attempt to bolt on a software coherence layer. diff --git a/arch/xtensa/core/crt1.S b/arch/xtensa/core/crt1.S index 0fa3ad230995c4..c616b0889d7198 100644 --- a/arch/xtensa/core/crt1.S +++ b/arch/xtensa/core/crt1.S @@ -24,10 +24,6 @@ .global __start .type z_cstart, @function -#ifdef CONFIG_XTENSA_MMU -.type z_xtensa_mmu_init, @function -#endif - /* Macros to abstract away ABI differences */ @@ -192,10 +188,6 @@ _start: #endif /* !XCHAL_HAVE_BOOTLOADER */ -#ifdef CONFIG_XTENSA_MMU - CALL z_xtensa_mmu_init -#endif - /* Enter C domain, never returns from here */ CALL z_cstart diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index e693937f99bfd5..6e939e243cd3f6 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -8,13 +8,9 @@ #include #include #include -#include -#if defined(CONFIG_XTENSA_ENABLE_BACKTRACE) -#if XCHAL_HAVE_WINDOWED #include -#endif -#endif #include +#include #include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); @@ -86,6 +82,8 @@ char *z_xtensa_exccause(unsigned int cause_code) case 63: /* i.e. z_except_reason */ return "zephyr exception"; + case 64: + return "kernel oops"; default: return "unknown/reserved"; } @@ -113,7 +111,6 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf) z_xtensa_backtrace_print(100, (int *)esf); #endif #endif - arch_irq_unlock(key); } @@ -140,3 +137,33 @@ FUNC_NORETURN void z_system_halt(unsigned int reason) CODE_UNREACHABLE; } #endif + +FUNC_NORETURN void arch_syscall_oops(void *ssf) +{ + ARG_UNUSED(ssf); + + xtensa_arch_kernel_oops(K_ERR_KERNEL_OOPS, ssf); + + CODE_UNREACHABLE; +} + +#ifdef CONFIG_USERSPACE +void z_impl_xtensa_user_fault(unsigned int reason) +{ + if ((_current->base.user_options & K_USER) != 0) { + if ((reason != K_ERR_KERNEL_OOPS) && + (reason != K_ERR_STACK_CHK_FAIL)) { + reason = K_ERR_KERNEL_OOPS; + } + } + xtensa_arch_except(reason); +} + +static void z_vrfy_xtensa_user_fault(unsigned int reason) +{ + z_impl_xtensa_user_fault(reason); +} + +#include + +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/core/gen_zsr.py b/arch/xtensa/core/gen_zsr.py index 8d052f4891d70f..0e3069a4c450d4 100755 --- a/arch/xtensa/core/gen_zsr.py +++ b/arch/xtensa/core/gen_zsr.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2022 Intel corporation # SPDX-License-Identifier: Apache-2.0 -import sys +import argparse import re # Scratch register allocator. Zephyr uses multiple Xtensa SRs as @@ -11,10 +11,26 @@ # -dM") core-isa.h file for the current architecture and assigns # registers to usages. -NEEDED = ("ALLOCA", "CPU", "FLUSH") +def parse_args(): + parser = argparse.ArgumentParser(allow_abbrev=False) -coreisa = sys.argv[1] -outfile = sys.argv[2] + parser.add_argument("--mmu", action="store_true", + help="Enable scratch registers for MMU usage") + parser.add_argument("coreisa", + help="Path to preprocessed core-isa.h") + parser.add_argument("outfile", + help="Output file") + + return parser.parse_args() + +args = parse_args() + +NEEDED = ["A0SAVE", "CPU", "FLUSH"] +if args.mmu: + NEEDED += ["MMU_0", "MMU_1", "DBLEXC"] + +coreisa = args.coreisa +outfile = args.outfile syms = {} diff --git a/arch/xtensa/core/include/xtensa_mmu_priv.h b/arch/xtensa/core/include/xtensa_mmu_priv.h index 7519c1b6010649..7b1030786f424d 100644 --- a/arch/xtensa/core/include/xtensa_mmu_priv.h +++ b/arch/xtensa/core/include/xtensa_mmu_priv.h @@ -18,18 +18,37 @@ #define Z_XTENSA_PTE_VPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_PPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_ATTR_MASK 0x0000000FU +#define Z_XTENSA_PTE_ATTR_CACHED_MASK 0x0000000CU #define Z_XTENSA_L1_MASK 0x3FF00000U #define Z_XTENSA_L2_MASK 0x3FFFFFU #define Z_XTENSA_PPN_SHIFT 12U #define Z_XTENSA_PTE_RING_MASK 0x00000030U +#define Z_XTENSA_PTE_RING_SHIFT 4U #define Z_XTENSA_PTE(paddr, ring, attr) \ (((paddr) & Z_XTENSA_PTE_PPN_MASK) | \ - (((ring) << 4) & Z_XTENSA_PTE_RING_MASK) | \ + (((ring) << Z_XTENSA_PTE_RING_SHIFT) & Z_XTENSA_PTE_RING_MASK) | \ ((attr) & Z_XTENSA_PTE_ATTR_MASK)) +#define Z_XTENSA_PTE_ATTR_GET(pte) \ + (pte) & Z_XTENSA_PTE_ATTR_MASK + +#define Z_XTENSA_PTE_ATTR_SET(pte, attr) \ + (((pte) & ~Z_XTENSA_PTE_ATTR_MASK) | (attr)) + +#define Z_XTENSA_PTE_RING_SET(pte, ring) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) | \ + ((ring) << Z_XTENSA_PTE_RING_SHIFT)) + +#define Z_XTENSA_PTE_RING_GET(pte) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) >> Z_XTENSA_PTE_RING_SHIFT) + +#define Z_XTENSA_PTE_ASID_GET(pte, rasid) \ + (((rasid) >> ((((pte) & Z_XTENSA_PTE_RING_MASK) \ + >> Z_XTENSA_PTE_RING_SHIFT) * 8)) & 0xFF) + #define Z_XTENSA_TLB_ENTRY(vaddr, way) \ (((vaddr) & Z_XTENSA_PTE_PPN_MASK) | (way)) @@ -38,11 +57,38 @@ (((vaddr) >> Z_XTENSA_PPN_SHIFT) & 0x03U)) #define Z_XTENSA_L2_POS(vaddr) \ - (((vaddr) & Z_XTENSA_L2_MASK) >> Z_XTENSA_PPN_SHIFT) + (((vaddr) & Z_XTENSA_L2_MASK) >> 12U) + +#define Z_XTENSA_L1_POS(vaddr) \ + ((vaddr) >> 22U) + +/* PTE attributes for entries in the L1 page table. Should never be + * writable, may be cached in non-SMP contexts only + */ +#if CONFIG_MP_MAX_NUM_CPUS == 1 +#define Z_XTENSA_PAGE_TABLE_ATTR Z_XTENSA_MMU_CACHED_WB +#else +#define Z_XTENSA_PAGE_TABLE_ATTR 0 +#endif + +/* This ASID is shared between all domains and kernel. */ +#define Z_XTENSA_MMU_SHARED_ASID 255 + +/* Fixed data TLB way to map the page table */ +#define Z_XTENSA_MMU_PTE_WAY 7 + +/* Fixed data TLB way to map the vecbase */ +#define Z_XTENSA_MMU_VECBASE_WAY 8 /* Kernel specific ASID. Ring field in the PTE */ #define Z_XTENSA_KERNEL_RING 0 +/* User specific ASID. Ring field in the PTE */ +#define Z_XTENSA_USER_RING 2 + +/* Ring value for MMU_SHARED_ASID */ +#define Z_XTENSA_SHARED_RING 3 + /* Number of data TLB ways [0-9] */ #define Z_XTENSA_DTLB_WAYS 10 @@ -86,15 +132,16 @@ * * PTE_ENTRY_ADDRESS = PTEVADDR + ((VADDR / 4096) * 4) */ -#define Z_XTENSA_PTE_ENTRY_VADDR(vaddr) \ - (Z_XTENSA_PTEVADDR + (((vaddr) / KB(4)) * 4)) +#define Z_XTENSA_PTE_ENTRY_VADDR(base, vaddr) \ + ((base) + (((vaddr) / KB(4)) * 4)) /* - * The address of the top level page where the page - * is located in the virtual address. + * Get asid for a given ring from rasid register. + * rasid contains four asid, one per ring. */ -#define Z_XTENSA_PAGE_TABLE_VADDR \ - Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR) + +#define Z_XTENSA_RASID_ASID_GET(rasid, ring) \ + (((rasid) >> ((ring) * 8)) & 0xff) static ALWAYS_INLINE void xtensa_rasid_set(uint32_t rasid) { @@ -110,6 +157,16 @@ static ALWAYS_INLINE uint32_t xtensa_rasid_get(void) return rasid; } +static ALWAYS_INLINE void xtensa_rasid_asid_set(uint8_t asid, uint8_t pos) +{ + uint32_t rasid = xtensa_rasid_get(); + + rasid = (rasid & ~(0xff << (pos * 8))) | ((uint32_t)asid << (pos * 8)); + + xtensa_rasid_set(rasid); +} + + static ALWAYS_INLINE void xtensa_itlb_entry_invalidate(uint32_t entry) { __asm__ volatile("iitlb %0\n\t" @@ -163,119 +220,32 @@ static ALWAYS_INLINE void xtensa_itlb_entry_write_sync(uint32_t pte, uint32_t en } /** - * @brief Invalidate all ITLB entries. + * @brief Invalidate all autorefill DTLB and ITLB entries. * - * This should be used carefully since all entries in the instruction TLB - * will be erased and the only way to find lookup a physical address will be - * through the page tables. - */ -static inline void xtensa_itlb_invalidate_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_ITLB_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_itlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} - -/** - * @brief Invalidate all DTLB entries. + * This should be used carefully since all refill entries in the data + * and instruction TLB. At least two pages, the current code page and + * the current stack, will be repopulated by this code as it returns. * - * This should be used carefully since all entries in the data TLB will be - * erased and the only way to find lookup a physical address will be through - * the page tables. + * This needs to be called in any circumstance where the mappings for + * a previously-used page table change. It does not need to be called + * on context switch, where ASID tagging isolates entries for us. */ -static inline void xtensa_dtlb_invalidate_sync(void) +static inline void xtensa_tlb_autorefill_invalidate(void) { - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_DTLB_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_dtlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} + uint8_t way, i, entries; -/** - * @brief Invalidates an autorefill DTLB entry. - * - * Invalidates the page table enrty that maps a given virtual address. - */ -static inline void xtensa_dtlb_autorefill_invalidate_sync(void *vaddr) -{ - uint8_t way; + entries = BIT(MAX(XCHAL_ITLB_ARF_ENTRIES_LOG2, + XCHAL_DTLB_ARF_ENTRIES_LOG2)); for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - xtensa_dtlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way)); - } - __asm__ volatile("dsync"); -} - -/** - * @brief Invalidates an autorefill ITLB entry. - * - * Invalidates the page table enrty that maps a given virtual address. - */ -static inline void xtensa_itlb_autorefill_invalidate_sync(void *vaddr) -{ - uint8_t way; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - xtensa_itlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way)); - } - __asm__ volatile("isync"); -} -/** - * @brief Invalidate all autorefill ITLB entries. - * - * This should be used carefully since all entries in the instruction TLB - * will be erased and the only way to find lookup a physical address will be - * through the page tables. - */ -static inline void xtensa_itlb_autorefill_invalidate_all_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_itlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} - -/** - * @brief Invalidate all autorefill DTLB entries. - * - * This should be used carefully since all entries in the data TLB will be - * erased and the only way to find lookup a physical address will be through - * the page tables. - */ -static inline void xtensa_dtlb_autorefill_invalidate_all_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { + for (i = 0; i < entries; i++) { uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_dtlb_entry_invalidate(entry); + xtensa_dtlb_entry_invalidate_sync(entry); + xtensa_itlb_entry_invalidate_sync(entry); } } - __asm__ volatile("isync"); } - /** * @brief Set the page tables. * @@ -288,6 +258,21 @@ static ALWAYS_INLINE void xtensa_ptevaddr_set(void *ptables) __asm__ volatile("wsr.ptevaddr %0" : : "a"((uint32_t)ptables)); } +/** + * @brief Get the current page tables. + * + * The page tables is obtained by reading ptevaddr address. + * + * @return ptables The page tables address (virtual address) + */ +static ALWAYS_INLINE void *xtensa_ptevaddr_get(void) +{ + uint32_t ptables; + + __asm__ volatile("rsr.ptevaddr %0" : "=a" (ptables)); + + return (void *)ptables; +} /* * The following functions are helpful when debugging. */ @@ -357,4 +342,8 @@ static inline void xtensa_dtlb_vaddr_invalidate(void *vaddr) } } +void xtensa_init_paging(uint32_t *l1_page); + +void xtensa_set_paging(uint32_t asid, uint32_t *l1_page); + #endif /* ZEPHYR_ARCH_XTENSA_XTENSA_MMU_PRIV_H_ */ diff --git a/arch/xtensa/core/mmu.c b/arch/xtensa/core/mmu.c new file mode 100644 index 00000000000000..24e47a42b9ded3 --- /dev/null +++ b/arch/xtensa/core/mmu.c @@ -0,0 +1,178 @@ +/* + * Copyright 2023 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include +#include + +#define ASID_INVALID 0 + +struct tlb_regs { + uint32_t rasid; + uint32_t ptevaddr; + uint32_t ptepin_as; + uint32_t ptepin_at; + uint32_t vecpin_as; + uint32_t vecpin_at; +}; + +static void compute_regs(uint32_t user_asid, uint32_t *l1_page, struct tlb_regs *regs) +{ + uint32_t vecbase = XTENSA_RSR("VECBASE"); + + __ASSERT_NO_MSG((((uint32_t)l1_page) & 0xfff) == 0); + __ASSERT_NO_MSG((user_asid == 0) || ((user_asid > 2) && + (user_asid < Z_XTENSA_MMU_SHARED_ASID))); + + /* We don't use ring 1, ring 0 ASID must be 1 */ + regs->rasid = (Z_XTENSA_MMU_SHARED_ASID << 24) | + (user_asid << 16) | 0x000201; + + /* Derive PTEVADDR from ASID so each domain gets its own PTE area */ + regs->ptevaddr = CONFIG_XTENSA_MMU_PTEVADDR + user_asid * 0x400000; + + /* The ptables code doesn't add the mapping for the l1 page itself */ + l1_page[Z_XTENSA_L1_POS(regs->ptevaddr)] = + (uint32_t)l1_page | Z_XTENSA_PAGE_TABLE_ATTR; + + regs->ptepin_at = (uint32_t)l1_page; + regs->ptepin_as = Z_XTENSA_PTE_ENTRY_VADDR(regs->ptevaddr, regs->ptevaddr) + | Z_XTENSA_MMU_PTE_WAY; + + /* Pin mapping for refilling the vector address into the ITLB + * (for handling TLB miss exceptions). Note: this is NOT an + * instruction TLB entry for the vector code itself, it's a + * DATA TLB entry for the page containing the vector mapping + * so the refill on instruction fetch can find it. The + * hardware doesn't have a 4k pinnable instruction TLB way, + * frustratingly. + */ + uint32_t vb_pte = l1_page[Z_XTENSA_L1_POS(vecbase)]; + + regs->vecpin_at = vb_pte; + regs->vecpin_as = Z_XTENSA_PTE_ENTRY_VADDR(regs->ptevaddr, vecbase) + | Z_XTENSA_MMU_VECBASE_WAY; +} + +/* Switch to a new page table. There are four items we have to set in + * the hardware: the PTE virtual address, the ring/ASID mapping + * register, and two pinned entries in the data TLB handling refills + * for the page tables and the vector handlers. + * + * These can be done in any order, provided that we ensure that no + * memory access which cause a TLB miss can happen during the process. + * This means that we must work entirely within registers in a single + * asm block. Also note that instruction fetches are memory accesses + * too, which means we cannot cross a page boundary which might reach + * a new page not in the TLB (a single jump to an aligned address that + * holds our five instructions is sufficient to guarantee that: I + * couldn't think of a way to do the alignment statically that also + * interoperated well with inline assembly). + */ +void xtensa_set_paging(uint32_t user_asid, uint32_t *l1_page) +{ + /* Optimization note: the registers computed here are pure + * functions of the two arguments. With a minor API tweak, + * they could be cached in e.g. a thread struct instead of + * being recomputed. This is called on context switch paths + * and is performance-sensitive. + */ + struct tlb_regs regs; + + compute_regs(user_asid, l1_page, ®s); + + __asm__ volatile("j 1f\n" + ".align 16\n" /* enough for 5 insns */ + "1:\n" + "wsr %0, PTEVADDR\n" + "wsr %1, RASID\n" + "wdtlb %2, %3\n" + "wdtlb %4, %5\n" + "isync" + :: "r"(regs.ptevaddr), "r"(regs.rasid), + "r"(regs.ptepin_at), "r"(regs.ptepin_as), + "r"(regs.vecpin_at), "r"(regs.vecpin_as)); +} + +/* This is effectively the same algorithm from xtensa_set_paging(), + * but it also disables the hardware-initialized 512M TLB entries in + * way 6 (because the hardware disallows duplicate TLB mappings). For + * instruction fetches this produces a critical ordering constraint: + * the instruction following the invalidation of ITLB entry mapping + * the current PC will by definition create a refill condition, which + * will (because the data TLB was invalidated) cause a refill + * exception. Therefore this step must be the very last one, once + * everything else is setup up and working, which includes the + * invalidation of the virtual PTEVADDR area so that the resulting + * refill can complete. + * + * Note that we can't guarantee that the compiler won't insert a data + * fetch from our stack memory after exit from the asm block (while it + * might be double-mapped), so we invalidate that data TLB inside the + * asm for correctness. The other 13 entries get invalidated in a C + * loop at the end. + */ +void xtensa_init_paging(uint32_t *l1_page) +{ + extern char z_xt_init_pc; /* defined in asm below */ + struct tlb_regs regs; + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + /* The incoherent cache can get into terrible trouble if it's + * allowed to cache PTEs differently across CPUs. We require + * that all page tables supplied by the OS have exclusively + * uncached mappings for page data, but can't do anything + * about earlier code/firmware. Dump the cache to be safe. + */ + sys_cache_data_flush_and_invd_all(); +#endif + + compute_regs(ASID_INVALID, l1_page, ®s); + + uint32_t idtlb_pte = (regs.ptevaddr & 0xe0000000) | XCHAL_SPANNING_WAY; + uint32_t idtlb_stk = (((uint32_t)®s) & ~0xfff) | XCHAL_SPANNING_WAY; + uint32_t iitlb_pc = (((uint32_t)&z_xt_init_pc) & ~0xfff) | XCHAL_SPANNING_WAY; + + /* Note: the jump is mostly pedantry, as it's almost + * inconceivable that a hardware memory region at boot is + * going to cross a 512M page boundary. But we need the entry + * symbol to get the address above, so the jump is here for + * symmetry with the set_paging() code. + */ + __asm__ volatile("j z_xt_init_pc\n" + ".align 32\n" /* room for 10 insns */ + ".globl z_xt_init_pc\n" + "z_xt_init_pc:\n" + "wsr %0, PTEVADDR\n" + "wsr %1, RASID\n" + "wdtlb %2, %3\n" + "wdtlb %4, %5\n" + "idtlb %6\n" /* invalidate pte */ + "idtlb %7\n" /* invalidate stk */ + "isync\n" + "iitlb %8\n" /* invalidate pc */ + "isync\n" /* <--- traps a ITLB miss */ + :: "r"(regs.ptevaddr), "r"(regs.rasid), + "r"(regs.ptepin_at), "r"(regs.ptepin_as), + "r"(regs.vecpin_at), "r"(regs.vecpin_as), + "r"(idtlb_pte), "r"(idtlb_stk), "r"(iitlb_pc)); + + /* Invalidate the remaining (unused by this function) + * initialization entries. Now we're flying free with our own + * page table. + */ + for (int i = 0; i < 8; i++) { + uint32_t ixtlb = (i * 0x2000000000) | XCHAL_SPANNING_WAY; + + if (ixtlb != iitlb_pc) { + __asm__ volatile("iitlb %0" :: "r"(ixtlb)); + } + if (ixtlb != idtlb_stk && ixtlb != idtlb_pte) { + __asm__ volatile("idtlb %0" :: "r"(ixtlb)); + } + } + __asm__ volatile("isync"); +} diff --git a/arch/xtensa/core/offsets/offsets.c b/arch/xtensa/core/offsets/offsets.c index 860a12eb408909..8d4022fd81ef05 100644 --- a/arch/xtensa/core/offsets/offsets.c +++ b/arch/xtensa/core/offsets/offsets.c @@ -5,6 +5,7 @@ #include #include +#include #include @@ -60,4 +61,10 @@ GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu14); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu15); #endif +#ifdef CONFIG_USERSPACE +GEN_OFFSET_SYM(_thread_arch_t, psp); +GEN_OFFSET_SYM(_thread_arch_t, ptables); +#endif + + GEN_ABS_SYM_END diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c new file mode 100644 index 00000000000000..17950114544f4b --- /dev/null +++ b/arch/xtensa/core/ptables.c @@ -0,0 +1,1095 @@ +/* + * Copyright (c) 2022 Intel Corporation + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* Skip TLB IPI when updating page tables. + * This allows us to send IPI only after the last + * changes of a series. + */ +#define OPTION_NO_TLB_IPI BIT(0) + +/* Level 1 contains page table entries + * necessary to map the page table itself. + */ +#define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U + +/* Size of level 1 page table. + */ +#define XTENSA_L1_PAGE_TABLE_SIZE (XTENSA_L1_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + +/* Level 2 contains page table entries + * necessary to map the page table itself. + */ +#define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U + +/* Size of level 2 page table. + */ +#define XTENSA_L2_PAGE_TABLE_SIZE (XTENSA_L2_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + +LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); + +BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, + "MMU_PAGE_SIZE value is invalid, only 4 kB pages are supported\n"); + +/* + * Level 1 page table has to be 4Kb to fit into one of the wired entries. + * All entries are initialized as INVALID, so an attempt to read an unmapped + * area will cause a double exception. + * + * Each memory domain contains its own l1 page table. The kernel l1 page table is + * located at the index 0. + */ +static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES] + __aligned(KB(4)); + + +/* + * That is an alias for the page tables set used by the kernel. + */ +uint32_t *z_xtensa_kernel_ptables = (uint32_t *)l1_page_table[0]; + +/* + * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one + * covering a 4Kb page. + */ +static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES] + __aligned(KB(4)); + +/* + * This additional variable tracks which l1 tables are in use. This is kept separated from + * the tables to keep alignment easier. + * + * @note: The first bit is set because it is used for the kernel page tables. + */ +static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES); + +/* + * This additional variable tracks which l2 tables are in use. This is kept separated from + * the tables to keep alignment easier. + */ +static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES); + +/* + * Protects xtensa_domain_list and serializes access to page tables. + */ +static struct k_spinlock xtensa_mmu_lock; + +#ifdef CONFIG_USERSPACE + +/* + * Each domain has its own ASID. ASID can go through 1 (kernel) to 255. + * When a TLB entry matches, the hw will check the ASID in the entry and finds + * the correspondent position in the RASID register. This position will then be + * compared with the current ring (CRING) to check the permission. + */ +static uint8_t asid_count = 3; + +/* + * List with all active and initialized memory domains. + */ +static sys_slist_t xtensa_domain_list; +#endif /* CONFIG_USERSPACE */ + +extern char _heap_end[]; +extern char _heap_start[]; +extern char __data_start[]; +extern char __data_end[]; +extern char _bss_start[]; +extern char _bss_end[]; + +/* + * Static definition of all code & data memory regions of the + * current Zephyr image. This information must be available & + * processed upon MMU initialization. + */ + +static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { + /* + * Mark the zephyr execution regions (data, bss, noinit, etc.) + * cacheable, read / write and non-executable + */ + { + /* This includes .data, .bss and various kobject sections. */ + .start = (uint32_t)_image_ram_start, + .end = (uint32_t)_image_ram_end, +#ifdef CONFIG_XTENSA_RPO_CACHE + .attrs = Z_XTENSA_MMU_W, +#else + .attrs = Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, +#endif + .name = "data", + }, +#if CONFIG_HEAP_MEM_POOL_SIZE > 0 + /* System heap memory */ + { + .start = (uint32_t)_heap_start, + .end = (uint32_t)_heap_end, +#ifdef CONFIG_XTENSA_RPO_CACHE + .attrs = Z_XTENSA_MMU_W, +#else + .attrs = Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, +#endif + .name = "heap", + }, +#endif + /* Mark text segment cacheable, read only and executable */ + { + .start = (uint32_t)__text_region_start, + .end = (uint32_t)__text_region_end, + .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, + .name = "text", + }, + /* Mark rodata segment cacheable, read only and non-executable */ + { + .start = (uint32_t)__rodata_region_start, + .end = (uint32_t)__rodata_region_end, + .attrs = Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, + .name = "rodata", + }, +}; + +static inline uint32_t *thread_page_tables_get(const struct k_thread *thread) +{ +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) != 0U) { + return thread->arch.ptables; + } +#endif + + return z_xtensa_kernel_ptables; +} + +/** + * @brief Check if the page table entry is illegal. + * + * @param[in] Page table entry. + */ +static inline bool is_pte_illegal(uint32_t pte) +{ + uint32_t attr = pte & Z_XTENSA_PTE_ATTR_MASK; + + /* + * The ISA manual states only 12 and 14 are illegal values. + * 13 and 15 are not. So we need to be specific than simply + * testing if bits 2 and 3 are set. + */ + return (attr == 12) || (attr == 14); +} + +/* + * @brief Initialize all page table entries to be illegal. + * + * @param[in] Pointer to page table. + * @param[in] Number of page table entries in the page table. + */ +static void init_page_table(uint32_t *ptable, size_t num_entries) +{ + int i; + + for (i = 0; i < num_entries; i++) { + ptable[i] = Z_XTENSA_MMU_ILLEGAL; + } +} + +static inline uint32_t *alloc_l2_table(void) +{ + uint16_t idx; + + for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L2_TABLES; idx++) { + if (!atomic_test_and_set_bit(l2_page_tables_track, idx)) { + return (uint32_t *)&l2_page_tables[idx]; + } + } + + return NULL; +} + +static void map_memory_range(const uint32_t start, const uint32_t end, + const uint32_t attrs, bool shared) +{ + uint32_t page, *table; + + for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { + uint32_t pte = Z_XTENSA_PTE(page, + shared ? Z_XTENSA_SHARED_RING : Z_XTENSA_KERNEL_RING, + attrs); + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + uint32_t l1_pos = Z_XTENSA_L1_POS(page); + + if (is_pte_illegal(z_xtensa_kernel_ptables[l1_pos])) { + table = alloc_l2_table(); + + __ASSERT(table != NULL, "There is no l2 page table available to " + "map 0x%08x\n", page); + + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + z_xtensa_kernel_ptables[l1_pos] = + Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + } + + table = (uint32_t *)(z_xtensa_kernel_ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table[l2_pos] = pte; + } +} + +static void map_memory(const uint32_t start, const uint32_t end, + const uint32_t attrs, bool shared) +{ + map_memory_range(start, end, attrs, shared); + +#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP + if (arch_xtensa_is_ptr_uncached((void *)start)) { + map_memory_range(POINTER_TO_UINT(z_soc_cached_ptr((void *)start)), + POINTER_TO_UINT(z_soc_cached_ptr((void *)end)), + attrs | Z_XTENSA_MMU_CACHED_WB, shared); + } else if (arch_xtensa_is_ptr_cached((void *)start)) { + map_memory_range(POINTER_TO_UINT(z_soc_uncached_ptr((void *)start)), + POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs, shared); + } +#endif +} + +static void xtensa_init_page_tables(void) +{ + volatile uint8_t entry; + + init_page_table(z_xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES); + atomic_set_bit(l1_page_table_track, 0); + + for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { + const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; + bool shared; + uint32_t attrs; + + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; + + map_memory(range->start, range->end, attrs, shared); + } + +/** + * GCC complains about usage of the SoC MMU range ARRAY_SIZE + * (xtensa_soc_mmu_ranges) as the default weak declaration is + * an empty array, and any access to its element is considered + * out of bound access. However, we have a number of element + * variable to guard against this (... if done correctly). + * Besides, this will almost be overridden by the SoC layer. + * So tell GCC to ignore this. + */ +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif + for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) { + const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry]; + bool shared; + uint32_t attrs; + + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; + + map_memory(range->start, range->end, attrs, shared); + } +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + /* Finally, the direct-mapped pages used in the page tables + * must be fixed up to use the same cache attribute (but these + * must be writable, obviously). They shouldn't be left at + * the default. + */ + map_memory_range((uint32_t) &l1_page_table[0], + (uint32_t) &l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES], + Z_XTENSA_PAGE_TABLE_ATTR | Z_XTENSA_MMU_W, false); + map_memory_range((uint32_t) &l2_page_tables[0], + (uint32_t) &l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES], + Z_XTENSA_PAGE_TABLE_ATTR | Z_XTENSA_MMU_W, false); + + sys_cache_data_flush_all(); +} + +__weak void arch_xtensa_mmu_post_init(bool is_core0) +{ + ARG_UNUSED(is_core0); +} + +void z_xtensa_mmu_init(void) +{ + if (_current_cpu->id == 0) { + /* This is normally done via arch_kernel_init() inside z_cstart(). + * However, before that is called, we go through the sys_init of + * INIT_LEVEL_EARLY, which is going to result in TLB misses. + * So setup whatever necessary so the exception handler can work + * properly. + */ + xtensa_init_page_tables(); + } + + xtensa_init_paging(z_xtensa_kernel_ptables); + + arch_xtensa_mmu_post_init(_current_cpu->id == 0); +} + +#ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES +/* Zephyr's linker scripts for Xtensa usually puts + * something before z_mapped_start (aka .text), + * i.e. vecbase, so that we need to reserve those + * space or else k_mem_map() would be mapping those, + * resulting in faults. + */ +__weak void arch_reserved_pages_update(void) +{ + uintptr_t page; + struct z_page_frame *pf; + int idx; + + for (page = CONFIG_SRAM_BASE_ADDRESS, idx = 0; + page < (uintptr_t)z_mapped_start; + page += CONFIG_MMU_PAGE_SIZE, idx++) { + pf = &z_page_frames[idx]; + + pf->flags |= Z_PAGE_FRAME_RESERVED; + } +} +#endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ + +static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, + uint32_t flags, bool is_user) +{ + uint32_t l1_pos = Z_XTENSA_L1_POS((uint32_t)vaddr); + uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); + uint32_t *table; + + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { + table = alloc_l2_table(); + + if (table == NULL) { + return false; + } + + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + l1_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + } + + table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table[l2_pos] = Z_XTENSA_PTE(phys, is_user ? Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING, + flags); + + sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); + xtensa_tlb_autorefill_invalidate(); + + return true; +} + +static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user) +{ + bool ret; + void *vaddr, *vaddr_uc; + uintptr_t paddr, paddr_uc; + uint32_t flags, flags_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + + if (arch_xtensa_is_ptr_cached((void *)pa)) { + paddr = pa; + paddr_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)pa); + } else { + paddr = (uintptr_t)arch_xtensa_cached_ptr((void *)pa); + paddr_uc = pa; + } + + flags_uc = (xtensa_flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + flags = flags_uc | Z_XTENSA_MMU_CACHED_WB; + } else { + vaddr = va; + paddr = pa; + flags = xtensa_flags; + } + + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", va); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc); + } + +#ifndef CONFIG_USERSPACE + ARG_UNUSED(ret); +#else + if (ret) { + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr, domain); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(domain->ptables, + (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr_uc, domain); + } + } + k_spin_unlock(&z_mem_domain_lock, key); + } +#endif /* CONFIG_USERSPACE */ +} + +void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) +{ + uint32_t va = (uint32_t)virt; + uint32_t pa = (uint32_t)phys; + uint32_t rem_size = (uint32_t)size; + uint32_t xtensa_flags = 0; + k_spinlock_key_t key; + bool is_user; + + if (size == 0) { + LOG_ERR("Cannot map physical memory at 0x%08X: invalid " + "zero size", (uint32_t)phys); + k_panic(); + } + + switch (flags & K_MEM_CACHE_MASK) { + + case K_MEM_CACHE_WB: + xtensa_flags |= Z_XTENSA_MMU_CACHED_WB; + break; + case K_MEM_CACHE_WT: + xtensa_flags |= Z_XTENSA_MMU_CACHED_WT; + break; + case K_MEM_CACHE_NONE: + __fallthrough; + default: + break; + } + + if ((flags & K_MEM_PERM_RW) == K_MEM_PERM_RW) { + xtensa_flags |= Z_XTENSA_MMU_W; + } + if ((flags & K_MEM_PERM_EXEC) == K_MEM_PERM_EXEC) { + xtensa_flags |= Z_XTENSA_MMU_X; + } + + is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER; + + key = k_spin_lock(&xtensa_mmu_lock); + + while (rem_size > 0) { + __arch_mem_map((void *)va, pa, xtensa_flags, is_user); + + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + va += KB(4); + pa += KB(4); + } + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + z_xtensa_mmu_tlb_ipi(); +#endif + + sys_cache_data_flush_and_invd_all(); + k_spin_unlock(&xtensa_mmu_lock, key); +} + +/** + * @return True if page is executable (thus need to invalidate ITLB), + * false if not. + */ +static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) +{ + uint32_t l1_pos = Z_XTENSA_L1_POS((uint32_t)vaddr); + uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); + uint32_t *l2_table; + uint32_t table_pos; + bool exec; + + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { + /* We shouldn't be unmapping an illegal entry. + * Return true so that we can invalidate ITLB too. + */ + return true; + } + + exec = l1_table[l1_pos] & Z_XTENSA_MMU_X; + + l2_table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + l2_table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) { + if (!is_pte_illegal(l2_table[l2_pos])) { + goto end; + } + } + + l1_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); + atomic_clear_bit(l2_page_tables_track, table_pos); + +end: + /* Need to invalidate L2 page table as it is no longer valid. */ + xtensa_tlb_autorefill_invalidate(); + return exec; +} + +static inline void __arch_mem_unmap(void *va) +{ + bool is_exec; + void *vaddr, *vaddr_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + } else { + vaddr = va; + } + + is_exec = l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr_uc); + } + +#ifdef CONFIG_USERSPACE + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr_uc); + } + } + k_spin_unlock(&z_mem_domain_lock, key); +#endif /* CONFIG_USERSPACE */ +} + +void arch_mem_unmap(void *addr, size_t size) +{ + uint32_t va = (uint32_t)addr; + uint32_t rem_size = (uint32_t)size; + k_spinlock_key_t key; + + if (addr == NULL) { + LOG_ERR("Cannot unmap NULL pointer"); + return; + } + + if (size == 0) { + LOG_ERR("Cannot unmap virtual memory with zero size"); + return; + } + + key = k_spin_lock(&xtensa_mmu_lock); + + while (rem_size > 0) { + __arch_mem_unmap((void *)va); + + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; + va += KB(4); + } + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + z_xtensa_mmu_tlb_ipi(); +#endif + + sys_cache_data_flush_and_invd_all(); + k_spin_unlock(&xtensa_mmu_lock, key); +} + +/* This should be implemented in the SoC layer. + * This weak version is here to avoid build errors. + */ +void __weak z_xtensa_mmu_tlb_ipi(void) +{ +} + +void z_xtensa_mmu_tlb_shootdown(void) +{ + unsigned int key; + + /* Need to lock interrupts to prevent any context + * switching until all the page tables are updated. + * Or else we would be switching to another thread + * and running that with incorrect page tables + * which would result in permission issues. + */ + key = arch_irq_lock(); + + K_SPINLOCK(&xtensa_mmu_lock) { + /* We don't have information on which page tables have changed, + * so we just invalidate the cache for all L1 page tables. + */ + sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table)); + sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + } + +#ifdef CONFIG_USERSPACE + struct k_thread *thread = _current_cpu->current; + + /* If current thread is a user thread, we need to see if it has + * been migrated to another memory domain as the L1 page table + * is different from the currently used one. + */ + if ((thread->base.user_options & K_USER) == K_USER) { + uint32_t ptevaddr_entry, ptevaddr, thread_ptables; + + /* Need to read the currently used L1 page table. + * We know that L1 page table is always mapped at way + * MMU_PTE_WAY, so we can skip the probing step by + * generating the query entry directly. + */ + ptevaddr_entry = (uint32_t)xtensa_ptevaddr_get() | Z_XTENSA_MMU_PTE_WAY; + ptevaddr = xtensa_dtlb_paddr_read(ptevaddr_entry); + + thread_ptables = (uint32_t)thread->arch.ptables; + + if (thread_ptables != ptevaddr) { + /* Need to remap the thread page tables if the ones + * indicated by the current thread are different + * than the current mapped page table. + */ + struct arch_mem_domain *domain = + &(thread->mem_domain_info.mem_domain->arch); + xtensa_set_paging(domain->asid, (uint32_t *)thread_ptables); + } + + } +#endif /* CONFIG_USERSPACE */ + + /* L2 are done via autofill, so invalidate autofill TLBs + * would refresh the L2 page tables. + * + * L1 will be refreshed during context switch so no need + * to do anything here. + */ + xtensa_tlb_autorefill_invalidate(); + + arch_irq_unlock(key); +} + +#ifdef CONFIG_USERSPACE + +static inline uint32_t *alloc_l1_table(void) +{ + uint16_t idx; + + for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) { + if (!atomic_test_and_set_bit(l1_page_table_track, idx)) { + return (uint32_t *)&l1_page_table[idx]; + } + } + + return NULL; +} + +static uint32_t *dup_table(uint32_t *source_table) +{ + uint16_t i, j; + uint32_t *dst_table = alloc_l1_table(); + + if (!dst_table) { + return NULL; + } + + for (i = 0; i < XTENSA_L1_PAGE_TABLE_ENTRIES; i++) { + uint32_t *l2_table, *src_l2_table; + + if (is_pte_illegal(source_table[i])) { + dst_table[i] = Z_XTENSA_MMU_ILLEGAL; + continue; + } + + src_l2_table = (uint32_t *)(source_table[i] & Z_XTENSA_PTE_PPN_MASK); + l2_table = alloc_l2_table(); + if (l2_table == NULL) { + goto err; + } + + for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) { + l2_table[j] = src_l2_table[j]; + } + + /* The page table is using kernel ASID because we don't + * user thread manipulate it. + */ + dst_table[i] = Z_XTENSA_PTE((uint32_t)l2_table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE); + } + + sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE); + + return dst_table; + +err: + /* TODO: Cleanup failed allocation*/ + return NULL; +} + +int arch_mem_domain_init(struct k_mem_domain *domain) +{ + uint32_t *ptables; + k_spinlock_key_t key; + int ret; + + /* + * For now, lets just assert if we have reached the maximum number + * of asid we assert. + */ + __ASSERT(asid_count < (Z_XTENSA_MMU_SHARED_ASID), "Reached maximum of ASID available"); + + key = k_spin_lock(&xtensa_mmu_lock); + ptables = dup_table(z_xtensa_kernel_ptables); + + if (ptables == NULL) { + ret = -ENOMEM; + goto err; + } + + domain->arch.ptables = ptables; + domain->arch.asid = ++asid_count; + + sys_slist_append(&xtensa_domain_list, &domain->arch.node); + + ret = 0; + +err: + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static int region_map_update(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags) +{ + int ret = 0; + + for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { + uint32_t *l2_table, pte; + uint32_t page = start + offset; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + /* Make sure we grab a fresh copy of L1 page table */ + sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0])); + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + pte = Z_XTENSA_PTE_RING_SET(l2_table[l2_pos], ring); + pte = Z_XTENSA_PTE_ATTR_SET(pte, flags); + + l2_table[l2_pos] = pte; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + xtensa_dtlb_vaddr_invalidate((void *)page); + } + + return ret; +} + +static inline int update_region(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags, + uint32_t option) +{ + int ret; + k_spinlock_key_t key; + + key = k_spin_lock(&xtensa_mmu_lock); + +#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP + uintptr_t va, va_uc; + uint32_t new_flags, new_flags_uc; + + if (arch_xtensa_is_ptr_cached((void *)start)) { + va = start; + va_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)start); + } else { + va = (uintptr_t)arch_xtensa_cached_ptr((void *)start); + va_uc = start; + } + + new_flags_uc = (flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + new_flags = new_flags_uc | Z_XTENSA_MMU_CACHED_WB; + + ret = region_map_update(ptables, va, size, ring, new_flags); + + if (ret == 0) { + ret = region_map_update(ptables, va_uc, size, ring, new_flags_uc); + } +#else + ret = region_map_update(ptables, start, size, ring, flags); +#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + if ((option & OPTION_NO_TLB_IPI) != OPTION_NO_TLB_IPI) { + z_xtensa_mmu_tlb_ipi(); + } +#endif + + sys_cache_data_flush_and_invd_all(); + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size, uint32_t option) +{ + return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W, option); +} + +void xtensa_user_stack_perms(struct k_thread *thread) +{ + (void)memset((void *)thread->stack_info.start, + (IS_ENABLED(CONFIG_INIT_STACKS)) ? 0xAA : 0x00, + thread->stack_info.size - thread->stack_info.delta); + + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, 0); +} + +int arch_mem_domain_max_partitions_get(void) +{ + return CONFIG_MAX_DOMAIN_PARTITIONS; +} + +int arch_mem_domain_partition_remove(struct k_mem_domain *domain, + uint32_t partition_id) +{ + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + /* Reset the partition's region back to defaults */ + return reset_region(domain->arch.ptables, partition->start, + partition->size, 0); +} + +int arch_mem_domain_partition_add(struct k_mem_domain *domain, + uint32_t partition_id) +{ + uint32_t ring = domain->arch.asid == 0 ? Z_XTENSA_KERNEL_RING : Z_XTENSA_USER_RING; + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + return update_region(domain->arch.ptables, partition->start, + partition->size, ring, partition->attr, 0); +} + +/* These APIs don't need to do anything */ +int arch_mem_domain_thread_add(struct k_thread *thread) +{ + int ret = 0; + bool is_user, is_migration; + uint32_t *old_ptables; + struct k_mem_domain *domain; + + old_ptables = thread->arch.ptables; + domain = thread->mem_domain_info.mem_domain; + thread->arch.ptables = domain->arch.ptables; + + is_user = (thread->base.user_options & K_USER) != 0; + is_migration = (old_ptables != NULL) && is_user; + + if (is_migration) { + /* Give access to the thread's stack in its new + * memory domain if it is migrating. + */ + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, + Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, + OPTION_NO_TLB_IPI); + /* and reset thread's stack permission in + * the old page tables. + */ + ret = reset_region(old_ptables, + thread->stack_info.start, + thread->stack_info.size, 0); + } + + /* Need to switch to new page tables if this is + * the current thread running. + */ + if (thread == _current_cpu->current) { + xtensa_set_paging(domain->arch.asid, thread->arch.ptables); + } + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + /* Need to tell other CPUs to switch to the new page table + * in case the thread is running on one of them. + * + * Note that there is no need to send TLB IPI if this is + * migration as it was sent above during reset_region(). + */ + if ((thread != _current_cpu->current) && !is_migration) { + z_xtensa_mmu_tlb_ipi(); + } +#endif + + return ret; +} + +int arch_mem_domain_thread_remove(struct k_thread *thread) +{ + struct k_mem_domain *domain = thread->mem_domain_info.mem_domain; + + if ((thread->base.user_options & K_USER) == 0) { + return 0; + } + + if ((thread->base.thread_state & _THREAD_DEAD) == 0) { + /* Thread is migrating to another memory domain and not + * exiting for good; we weren't called from + * z_thread_abort(). Resetting the stack region will + * take place in the forthcoming thread_add() call. + */ + return 0; + } + + /* Restore permissions on the thread's stack area since it is no + * longer a member of the domain. + * + * Note that, since every thread must have an associated memory + * domain, removing a thread from domain will be followed by + * adding it back to another. So there is no need to send TLB IPI + * at this point. + */ + return reset_region(domain->arch.ptables, + thread->stack_info.start, + thread->stack_info.size, OPTION_NO_TLB_IPI); +} + +static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write) +{ + uint8_t asid_ring; + uint32_t rasid, pte, *l2_table; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + + if (is_pte_illegal(ptables[l1_pos])) { + return false; + } + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + pte = l2_table[l2_pos]; + + if (is_pte_illegal(pte)) { + return false; + } + + asid_ring = 0; + rasid = xtensa_rasid_get(); + for (uint32_t i = 0; i < 4; i++) { + if (Z_XTENSA_PTE_ASID_GET(pte, rasid) == + Z_XTENSA_RASID_ASID_GET(rasid, i)) { + asid_ring = i; + break; + } + } + + if (ring > asid_ring) { + return false; + } + + if (write) { + return (Z_XTENSA_PTE_ATTR_GET((pte)) & Z_XTENSA_MMU_W) != 0; + } + + return true; +} + +int arch_buffer_validate(void *addr, size_t size, int write) +{ + int ret = 0; + uint8_t *virt; + size_t aligned_size; + const struct k_thread *thread = _current; + uint32_t *ptables = thread_page_tables_get(thread); + uint8_t ring = ((thread->base.user_options & K_USER) != 0) ? + Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING; + + /* addr/size arbitrary, fix this up into an aligned region */ + k_mem_region_align((uintptr_t *)&virt, &aligned_size, + (uintptr_t)addr, size, CONFIG_MMU_PAGE_SIZE); + + for (size_t offset = 0; offset < aligned_size; + offset += CONFIG_MMU_PAGE_SIZE) { + if (!page_validate(ptables, (uint32_t)(virt + offset), ring, write)) { + ret = -1; + break; + } + } + + return ret; +} + +void z_xtensa_swap_update_page_tables(struct k_thread *incoming) +{ + uint32_t *ptables = incoming->arch.ptables; + struct arch_mem_domain *domain = + &(incoming->mem_domain_info.mem_domain->arch); + + xtensa_set_paging(domain->asid, ptables); +} + +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/core/syscall_helper.c b/arch/xtensa/core/syscall_helper.c new file mode 100644 index 00000000000000..f8fb7ec903ec3e --- /dev/null +++ b/arch/xtensa/core/syscall_helper.c @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +uintptr_t xtensa_syscall_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +} diff --git a/arch/xtensa/core/userspace.S b/arch/xtensa/core/userspace.S new file mode 100644 index 00000000000000..1a000d75d57071 --- /dev/null +++ b/arch/xtensa/core/userspace.S @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2022, Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +/** + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ +.pushsection .text.z_xtensa_do_syscall, "ax" +.global z_xtensa_do_syscall +.align 4 +z_xtensa_do_syscall: + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + s32i a1, a0, ___xtensa_irq_bsa_t_scratch_OFFSET + s32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + s32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + rsr a2, ZSR_A0SAVE + s32i a2, a0, ___xtensa_irq_bsa_t_a0_OFFSET + rsr.ps a2 + movi a3, ~PS_OWB_MASK + and a2, a2, a3 + s32i a2, a0, ___xtensa_irq_bsa_t_ps_OFFSET + rsr.epc1 a2 + s32i a2, a0, ___xtensa_irq_bsa_t_pc_OFFSET + + movi a2, PS_WOE|PS_INTLEVEL(XCHAL_NMILEVEL) + rsr.ps a3 + or a3, a3, a2 + movi a2, ~(PS_EXCM | PS_RING_MASK) + and a3, a3, a2 + wsr.ps a3 + rsync + l32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + mov a1, a0 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET +#if XCHAL_HAVE_LOOPS + /* If the syscall instruction was the last instruction in the body of + * a zero-overhead loop, and the loop will execute again, decrement + * the loop count and resume execution at the head of the loop. + */ + rsr.lend a2 + addi a3, a3, 3 + bne a2, a3, end_loop + rsr.lcount a2 + beqz a2, end_loop + addi a2, a2, -1 + wsr.lcount a2 + rsr.lbeg a3 +end_loop: +#else + /* EPC1 (and now a3) contains the address that invoked syscall. + * We need to increment it to execute the next instruction when + * we return. The instruction size is 3 bytes, so lets just add it. + */ + addi a3, a3, 3 +#endif + s32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + ODD_REG_SAVE + + call0 xtensa_save_high_regs + + l32i a2, a1, 0 + l32i a2, a2, ___xtensa_irq_bsa_t_a2_OFFSET + movi a0, K_SYSCALL_LIMIT + bgeu a2, a0, _bad_syscall + +_id_ok: + /* Find the function handler for the given syscall id. */ + movi a3, _k_syscall_table + slli a2, a2, 2 + add a2, a2, a3 + l32i a2, a2, 0 + + /* Clear up the threadptr because it is used + * to check if a thread is running on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possibly running in user mode. + */ +#ifdef CONFIG_THREAD_LOCAL_STORAGE + movi a0, is_user_mode@tpoff + rur.THREADPTR a3 + add a0, a3, a0 + + movi a3, 0 + s32i a3, a0, 0 +#else + movi a0, 0 + wur.THREADPTR a0 +#endif + + /* Set syscall parameters. We have an initial call4 to set up the + * the stack and then a new call4 for the syscall function itself. + * So parameters should be put as if it was a call8. + */ + mov a10, a8 + mov a11, a9 + mov a8, a4 + mov a9, a5 + l32i a3, a1, 0 + l32i a7, a3, ___xtensa_irq_bsa_t_a3_OFFSET + + + /* Since we are unmasking EXCM, we need to set RING bits to kernel + * mode, otherwise we won't be able to run the exception handler in C. + */ + movi a0, PS_WOE|PS_CALLINC(0)|PS_UM|PS_INTLEVEL(0) + wsr.ps a0 + rsync + + call4 _syscall_call0 + + /* copy return value. Lets put it in the top of stack + * because registers will be clobbered in + * xtensa_restore_high_regs + */ + l32i a3, a1, 0 + s32i a6, a3, ___xtensa_irq_bsa_t_a2_OFFSET + + j _syscall_returned + +.align 4 +_syscall_call0: + /* We want an ENTRY to set a bit in windowstart */ + jx a2 + + +_syscall_returned: + call0 xtensa_restore_high_regs + + l32i a3, a1, ___xtensa_irq_bsa_t_sar_OFFSET + wsr a3, SAR +#if XCHAL_HAVE_LOOPS + l32i a3, a1, ___xtensa_irq_bsa_t_lbeg_OFFSET + wsr a3, LBEG + l32i a3, a1, ___xtensa_irq_bsa_t_lend_OFFSET + wsr a3, LEND + l32i a3, a1, ___xtensa_irq_bsa_t_lcount_OFFSET + wsr a3, LCOUNT +#endif +#if XCHAL_HAVE_S32C1I + l32i a3, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET + wsr a3, SCOMPARE1 +#endif + +#ifdef CONFIG_THREAD_LOCAL_STORAGE + l32i a3, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET + movi a0, is_user_mode@tpoff + add a0, a3, a0 + movi a3, 1 + s32i a3, a0, 0 +#else + rsr a3, ZSR_CPU + l32i a3, a3, ___cpu_t_current_OFFSET + wur.THREADPTR a3 +#endif + + l32i a3, a1, ___xtensa_irq_bsa_t_ps_OFFSET + wsr.ps a3 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + wsr.epc1 a3 + + l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET + l32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET + + l32i a1, a1, ___xtensa_irq_bsa_t_scratch_OFFSET + rsync + + rfe + +_bad_syscall: + movi a2, K_SYSCALL_BAD + j _id_ok + +.popsection + +/* FUNC_NORETURN void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + * void *p1, void *p2, void *p3, + * uint32_t stack_end, + * uint32_t stack_start) + * + * A one-way trip to userspace. + */ +.global z_xtensa_userspace_enter +.type z_xtensa_userspace_enter, @function +.align 4 +z_xtensa_userspace_enter: + /* Call entry to set a bit in the windowstart and + * do the rotation, but we are going to set our own + * stack. + */ + entry a1, 16 + + /* We have to switch to kernel stack before spill kernel data and + * erase user stack to avoid leak from previous context. + */ + mov a1, a7 /* stack start (low address) */ + addi a1, a1, -16 + + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + + addi a1, a1, -28 + s32i a0, a1, 24 + s32i a2, a1, 20 + s32i a3, a1, 16 + s32i a4, a1, 12 + s32i a5, a1, 8 + s32i a6, a1, 4 + s32i a7, a1, 0 + + l32i a6, a1, 24 + call4 xtensa_user_stack_perms + + l32i a6, a1, 24 + call4 z_xtensa_swap_update_page_tables + +#ifdef CONFIG_THREAD_LOCAL_STORAGE + rur.threadptr a3 + movi a0, is_user_mode@tpoff + add a0, a3, a0 + movi a3, 1 + s32i a3, a0, 0 +#else + rsr a3, ZSR_CPU + l32i a3, a3, ___cpu_t_current_OFFSET + wur.THREADPTR a3 +#endif + + /* Set now z_thread_entry parameters, we are simulating a call4 + * call, so parameters start at a6, a7, ... + */ + l32i a6, a1, 20 + l32i a7, a1, 16 + l32i a8, a1, 12 + l32i a9, a1, 8 + + /* stash user stack */ + l32i a0, a1, 4 + + addi a1, a1, 28 + + /* Go back to user stack */ + mov a1, a0 + + movi a0, z_thread_entry + wsr.epc2 a0 + + /* Configuring PS register. + * We have to set callinc as well, since the called + * function will do "entry" + */ + movi a0, PS_WOE|PS_CALLINC(1)|PS_UM|PS_RING(2) + wsr a0, EPS2 + + movi a0, 0 + + rfi 2 + +/* + * size_t arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg) + */ +.global arch_user_string_nlen +.type arch_user_string_nlen, @function +.align 4 +arch_user_string_nlen: + entry a1, 32 + + /* error value, set to -1. */ + movi a5, -1 + s32i a5, a4, 0 + + /* length count */ + xor a5, a5, a5 + + /* This code might page fault */ +strlen_loop: +.global z_xtensa_user_string_nlen_fault_start +z_xtensa_user_string_nlen_fault_start: + l8ui a6, a2, 0 /* Current char */ + +.global z_xtensa_user_string_nlen_fault_end +z_xtensa_user_string_nlen_fault_end: + beqz a6, strlen_done + addi a5, a5, 1 + addi a2, a2, 1 + beq a5, a3, strlen_done + j strlen_loop + +strlen_done: + /* Set return value */ + mov a2, a5 + + /* Set error value to 0 since we succeeded */ + movi a5, 0x0 + s32i a5, a4, 0 + +.global z_xtensa_user_string_nlen_fixup +z_xtensa_user_string_nlen_fixup: + retw diff --git a/arch/xtensa/core/window_vectors.S b/arch/xtensa/core/window_vectors.S index a63923cbd6fc2d..90eba495bde809 100644 --- a/arch/xtensa/core/window_vectors.S +++ b/arch/xtensa/core/window_vectors.S @@ -84,7 +84,7 @@ _WindowUnderflow4: /* Handle alloca exception generated by interruptee executing 'movsp'. * This uses space between the window vectors, so is essentially * "free". All interruptee's regs are intact except a0 which is saved - * in $ZSR_ALLOCA (assigned at build time, see gen_zsr.py for + * in $ZSR_A0SAVE (assigned at build time, see gen_zsr.py for * details), and PS.EXCM has been set by the exception hardware (can't * be interrupted). The fact the alloca exception was taken means the * registers associated with the base-save area have been spilled and @@ -102,7 +102,7 @@ _xt_alloca_exc: rsr a2, PS extui a3, a2, XCHAL_PS_OWB_SHIFT, XCHAL_PS_OWB_BITS xor a3, a3, a4 /* bits changed from old to current windowbase */ - rsr a4, ZSR_ALLOCA /* restore original a0 (now in a4) */ + rsr a4, ZSR_A0SAVE /* restore original a0 (now in a4) */ slli a3, a3, XCHAL_PS_OWB_SHIFT xor a2, a2, a3 /* flip changed bits in old window base */ wsr a2, PS /* update PS.OWB to new window base */ diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index d7ba88aaffc397..4f8af894ccb572 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -174,7 +174,8 @@ _restore_context: l32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET wsr a0, SCOMPARE1 #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) l32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET wur a0, THREADPTR #endif @@ -203,6 +204,20 @@ xtensa_arch_except_epc: ill retw +/* + * void xtensa_arch_kernel_oops(int reason_p, void *ssf); + * + * Simply to raise hardware exception for Kernel OOPS. + */ +.global xtensa_arch_kernel_oops +.global xtensa_arch_kernel_oops_epc +.align 4 +xtensa_arch_kernel_oops: + entry a1, 16 +xtensa_arch_kernel_oops_epc: + ill + retw + /* * void xtensa_switch(void *new, void **old_return); * @@ -258,6 +273,16 @@ noflush: l32i a3, a2, ___xtensa_irq_bsa_t_a3_OFFSET s32i a1, a3, 0 +#ifdef CONFIG_USERSPACE + /* Switch page tables */ + rsr a6, ZSR_CPU + l32i a6, a6, ___cpu_t_current_OFFSET + call4 z_xtensa_swap_update_page_tables + + l32i a2, a3, 0 + l32i a2, a2, 0 +#endif + /* Switch stack pointer and restore. The jump to * _restore_context does not return as such, but we arrange * for the restored "next" address to be immediately after for @@ -342,11 +367,14 @@ DEF_EXCINT XCHAL_DEBUGLEVEL, _handle_excint, xtensa_debugint_c .pushsection .UserExceptionVector.text, "ax" .global _Level1RealVector _Level1RealVector: - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_A0SAVE rsync rsr.exccause a0 #ifdef CONFIG_XTENSA_MMU beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_user +#ifdef CONFIG_USERSPACE + beqi a0, EXCCAUSE_SYSCALL, _syscall +#endif /* CONFIG_USERSPACE */ addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_user rsr.exccause a0 @@ -355,7 +383,7 @@ _Level1RealVector: j _xt_alloca_exc _not_alloca: - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE j _Level1Vector #ifdef CONFIG_XTENSA_MMU _handle_tlb_miss_user: @@ -374,8 +402,13 @@ _handle_tlb_miss_user: */ rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE rfe +#ifdef CONFIG_USERSPACE +_syscall: + rsr a0, ZSR_A0SAVE + j z_xtensa_do_syscall +#endif /* CONFIG_USERSPACE */ #endif /* CONFIG_XTENSA_MMU */ .popsection @@ -391,12 +424,12 @@ _handle_tlb_miss_user: .global _KernelExceptionVector _KernelExceptionVector: #ifdef CONFIG_XTENSA_MMU - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_A0SAVE rsr.exccause a0 beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_kernel addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_kernel - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE #endif j _Level1Vector #ifdef CONFIG_XTENSA_MMU @@ -410,7 +443,7 @@ _handle_tlb_miss_kernel: */ rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE rfe #endif .popsection @@ -420,15 +453,18 @@ _handle_tlb_miss_kernel: .global _DoubleExceptionVector _DoubleExceptionVector: #ifdef CONFIG_XTENSA_MMU - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_DBLEXC rsync rsr.exccause a0 addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_dblexc - rsr a0, ZSR_ALLOCA -#endif + rsr a0, ZSR_DBLEXC + + j _Level1Vector +#else + #if defined(CONFIG_SIMULATOR_XTENSA) || defined(XT_SIMULATOR) 1: /* Tell simulator to stop executing here, instead of trying to do @@ -446,6 +482,7 @@ _DoubleExceptionVector: 1: #endif j 1b +#endif /* CONFIG_XTENSA_MMU */ #ifdef CONFIG_XTENSA_MMU _handle_tlb_miss_dblexc: @@ -459,7 +496,7 @@ _handle_tlb_miss_dblexc: rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_DBLEXC rfde #endif .popsection diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index c2371ac8f55d6a..88783c178c6426 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -15,10 +15,28 @@ #include #include #include +#include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); extern char xtensa_arch_except_epc[]; +extern char xtensa_arch_kernel_oops_epc[]; + +#ifdef CONFIG_USERSPACE +Z_EXC_DECLARE(z_xtensa_user_string_nlen); + +static const struct z_exc_handle exceptions[] = { + Z_EXC_HANDLE(z_xtensa_user_string_nlen) +}; + +#ifdef CONFIG_THREAD_LOCAL_STORAGE +/* + * Per-thread (TLS) variable indicating whether execution is in user mode. + */ +__thread uint32_t is_user_mode; +#endif + +#endif /* CONFIG_USERSPACE */ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, void (*entry)(void *, void *, void *), @@ -26,6 +44,13 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, { void *ret; _xtensa_irq_stack_frame_a11_t *frame; +#ifdef CONFIG_USERSPACE + struct z_xtensa_thread_stack_header *header = + (struct z_xtensa_thread_stack_header *)thread->stack_obj; + + thread->arch.psp = header->privilege_stack + + sizeof(header->privilege_stack); +#endif /* Not-a-cpu ID Ensures that the first time this is run, the * stack will be invalidated. That covers the edge case of @@ -48,11 +73,23 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, (void)memset(frame, 0, bsasz); - frame->bsa.pc = (uintptr_t)z_thread_entry; frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1); +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) == K_USER) { + frame->bsa.pc = (uintptr_t)arch_user_mode_enter; + } else { + frame->bsa.pc = (uintptr_t)z_thread_entry; + } +#else + frame->bsa.pc = (uintptr_t)z_thread_entry; +#endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR +#ifdef CONFIG_THREAD_LOCAL_STORAGE frame->bsa.threadptr = thread->tls; +#elif CONFIG_USERSPACE + frame->bsa.threadptr = (uintptr_t)((thread->base.user_options & K_USER) ? thread : NULL); +#endif #endif /* Arguments to z_thread_entry(). Remember these start at A6, @@ -174,6 +211,40 @@ static inline unsigned int get_bits(int offset, int num_bits, unsigned int val) return val & mask; } +static void print_fatal_exception(void *print_stack, int cause, + bool is_dblexc, uint32_t depc) +{ + void *pc; + uint32_t ps, vaddr; + _xtensa_irq_bsa_t *bsa = (void *)*(int **)print_stack; + + ps = bsa->ps; + pc = (void *)bsa->pc; + + __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); + + LOG_ERR(" ** FATAL EXCEPTION%s", (is_dblexc ? " (DOUBLE)" : "")); + LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)", + arch_curr_cpu()->id, cause, + z_xtensa_exccause(cause)); + LOG_ERR(" ** PC %p VADDR %p", pc, (void *)vaddr); + + if (is_dblexc) { + LOG_ERR(" ** DEPC %p", (void *)depc); + } + +#ifdef CONFIG_USERSPACE + LOG_ERR(" ** THREADPTR %p", (void *)bsa->threadptr); +#endif /* CONFIG_USERSPACE */ + + LOG_ERR(" ** PS %p", (void *)bsa->ps); + LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", + get_bits(0, 4, ps), get_bits(4, 1, ps), + get_bits(5, 1, ps), get_bits(6, 2, ps), + get_bits(18, 1, ps), + get_bits(8, 4, ps), get_bits(16, 2, ps)); +} + static ALWAYS_INLINE void usage_stop(void) { #ifdef CONFIG_SCHED_THREAD_USAGE @@ -268,40 +339,32 @@ static inline DEF_INT_C_HANDLER(1) */ void *xtensa_excint1_c(int *interrupted_stack) { - int cause, vaddr; + int cause; _xtensa_irq_bsa_t *bsa = (void *)*(int **)interrupted_stack; bool is_fatal_error = false; + bool is_dblexc = false; uint32_t ps; - void *pc; + void *pc, *print_stack = (void *)interrupted_stack; + uint32_t depc = 0; __asm__ volatile("rsr.exccause %0" : "=r"(cause)); #ifdef CONFIG_XTENSA_MMU - /* TLB miss exception comes through level 1 interrupt also. - * We need to preserve execution context after we have handled - * the TLB miss, so we cannot unconditionally unmask interrupts. - * For other cause, we can unmask interrupts so this would act - * the same as if there is no MMU. - */ - switch (cause) { - case EXCCAUSE_ITLB_MISS: - /* Instruction TLB miss */ - __fallthrough; - case EXCCAUSE_DTLB_MISS: - /* Data TLB miss */ + __asm__ volatile("rsr.depc %0" : "=r"(depc)); - /* Do not unmask interrupt while handling TLB misses. */ - break; - default: - /* For others, we can unmask interrupts. */ - bsa->ps &= ~PS_INTLEVEL_MASK; - break; - } + is_dblexc = (depc != 0U); #endif /* CONFIG_XTENSA_MMU */ switch (cause) { case EXCCAUSE_LEVEL1_INTERRUPT: - return xtensa_int1_c(interrupted_stack); + if (!is_dblexc) { + return xtensa_int1_c(interrupted_stack); + } + break; +#ifndef CONFIG_USERSPACE + /* Syscalls are handled earlier in assembly if MMU is enabled. + * So we don't need this here. + */ case EXCCAUSE_SYSCALL: /* Just report it to the console for now */ LOG_ERR(" ** SYSCALL PS %p PC %p", @@ -314,46 +377,29 @@ void *xtensa_excint1_c(int *interrupted_stack) */ bsa->pc += 3; break; -#ifdef CONFIG_XTENSA_MMU - case EXCCAUSE_ITLB_MISS: - /* Instruction TLB miss */ - __fallthrough; - case EXCCAUSE_DTLB_MISS: - /* Data TLB miss */ - - /** - * The way it works is, when we try to access an address - * that is not mapped, we will have a miss. The HW then - * will try to get the correspondent memory in the page - * table. As the page table is not mapped in memory we will - * have a second miss, which will trigger an exception. - * In the exception (here) what we do is to exploit this - * hardware capability just trying to load the page table - * (not mapped address), which will cause a miss, but then - * the hardware will automatically map it again from - * the page table. This time it will work since the page - * necessary to map the page table itself are wired map. - */ - __asm__ volatile("wsr a0, " ZSR_EXTRA0_STR "\n\t" - "rsr.ptevaddr a0\n\t" - "l32i a0, a0, 0\n\t" - "rsr a0, " ZSR_EXTRA0_STR "\n\t" - "rsync" - : : : "a0", "memory"); - - /* Since we are dealing with TLB misses, we will probably not - * want to switch to another thread. - */ - return interrupted_stack; -#endif /* CONFIG_XTENSA_MMU */ +#endif /* !CONFIG_USERSPACE */ default: ps = bsa->ps; pc = (void *)bsa->pc; - __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); +#ifdef CONFIG_USERSPACE + /* If the faulting address is from one of the known + * exceptions that should not be fatal, return to + * the fixup address. + */ + for (int i = 0; i < ARRAY_SIZE(exceptions); i++) { + if ((pc >= exceptions[i].start) && + (pc < exceptions[i].end)) { + bsa->pc = (uintptr_t)exceptions[i].fixup; + + goto fixup_out; + } + } +#endif /* CONFIG_USERSPACE */ /* Default for exception */ int reason = K_ERR_CPU_EXCEPTION; + is_fatal_error = true; /* We need to distinguish between an ill in xtensa_arch_except, * e.g for k_panic, and any other ill. For exceptions caused by @@ -363,25 +409,32 @@ void *xtensa_excint1_c(int *interrupted_stack) * We assign EXCCAUSE the unused, reserved code 63; this may be * problematic if the app or new boards also decide to repurpose * this code. + * + * Another intentionally ill is from xtensa_arch_kernel_oops. + * Kernel OOPS has to be explicity raised so we can simply + * set the reason and continue. */ - if ((pc == (void *) &xtensa_arch_except_epc) && (cause == 0)) { - cause = 63; - __asm__ volatile("wsr.exccause %0" : : "r"(cause)); - reason = bsa->a2; + if (cause == EXCCAUSE_ILLEGAL) { + if (pc == (void *)&xtensa_arch_except_epc) { + cause = 63; + __asm__ volatile("wsr.exccause %0" : : "r"(cause)); + reason = bsa->a2; + } else if (pc == (void *)&xtensa_arch_kernel_oops_epc) { + cause = 64; /* kernel oops */ + reason = K_ERR_KERNEL_OOPS; + + /* A3 contains the second argument to + * xtensa_arch_kernel_oops(reason, ssf) + * where ssf is the stack frame causing + * the kernel oops. + */ + print_stack = (void *)bsa->a3; + } } - LOG_ERR(" ** FATAL EXCEPTION"); - LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)", - arch_curr_cpu()->id, cause, - z_xtensa_exccause(cause)); - LOG_ERR(" ** PC %p VADDR %p", - pc, (void *)vaddr); - LOG_ERR(" ** PS %p", (void *)bsa->ps); - LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", - get_bits(0, 4, ps), get_bits(4, 1, ps), - get_bits(5, 1, ps), get_bits(6, 2, ps), - get_bits(18, 1, ps), - get_bits(8, 4, ps), get_bits(16, 2, ps)); + if (reason != K_ERR_KERNEL_OOPS) { + print_fatal_exception(print_stack, cause, is_dblexc, depc); + } /* FIXME: legacy xtensa port reported "HW" exception * for all unhandled exceptions, which seems incorrect @@ -389,25 +442,25 @@ void *xtensa_excint1_c(int *interrupted_stack) * up. */ z_xtensa_fatal_error(reason, - (void *)interrupted_stack); + (void *)print_stack); break; } - +#ifdef CONFIG_XTENSA_MMU switch (cause) { - case EXCCAUSE_SYSCALL: case EXCCAUSE_LEVEL1_INTERRUPT: - case EXCCAUSE_ALLOCA: - case EXCCAUSE_ITLB_MISS: - case EXCCAUSE_DTLB_MISS: +#ifndef CONFIG_USERSPACE + case EXCCAUSE_SYSCALL: +#endif /* !CONFIG_USERSPACE */ is_fatal_error = false; break; default: is_fatal_error = true; break; } +#endif /* CONFIG_XTENSA_MMU */ - if (is_fatal_error) { + if (is_dblexc || is_fatal_error) { uint32_t ignore; /* We are going to manipulate _current_cpu->nested manually. @@ -436,6 +489,16 @@ void *xtensa_excint1_c(int *interrupted_stack) _current_cpu->nested = 1; } +#ifdef CONFIG_XTENSA_MMU +#ifdef CONFIG_USERSPACE +fixup_out: +#endif + if (is_dblexc) { + __asm__ volatile("wsr.depc %0" : : "r"(0)); + } +#endif /* CONFIG_XTENSA_MMU */ + + return return_to(interrupted_stack); } @@ -471,3 +534,24 @@ void arch_spin_relax(void) #undef NOP1 } #endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */ + +#ifdef CONFIG_USERSPACE +FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3) +{ + struct k_thread *current = _current; + size_t stack_end; + + /* Transition will reset stack pointer to initial, discarding + * any old context since this is a one-way operation + */ + stack_end = Z_STACK_PTR_ALIGN(current->stack_info.start + + current->stack_info.size - + current->stack_info.delta); + + z_xtensa_userspace_enter(user_entry, p1, p2, p3, + stack_end, current->stack_info.start); + + CODE_UNREACHABLE; +} +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c deleted file mode 100644 index e50c51a3848f55..00000000000000 --- a/arch/xtensa/core/xtensa_mmu.c +++ /dev/null @@ -1,505 +0,0 @@ -/* - * Copyright (c) 2022 Intel Corporation - * SPDX-License-Identifier: Apache-2.0 - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* Fixed data TLB way to map the page table */ -#define MMU_PTE_WAY 7 - -/* Fixed data TLB way to map VECBASE */ -#define MMU_VECBASE_WAY 8 - -/* Level 1 contains page table entries - * necessary to map the page table itself. - */ -#define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U - -/* Level 2 contains page table entries - * necessary to map the page table itself. - */ -#define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U - -LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); - -BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, - "MMU_PAGE_SIZE value is invalid, only 4 kB pages are supported\n"); - -/* - * Level 1 page table has to be 4Kb to fit into one of the wired entries. - * All entries are initialized as INVALID, so an attempt to read an unmapped - * area will cause a double exception. - */ -uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); - -/* - * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one - * covering a 4Kb page. - */ -static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES] - __aligned(KB(4)); - -/* - * This additional variable tracks which l2 tables are in use. This is kept separated from - * the tables to keep alignment easier. - */ -static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES); - -extern char _heap_end[]; -extern char _heap_start[]; -extern char __data_start[]; -extern char __data_end[]; -extern char _bss_start[]; -extern char _bss_end[]; - -/* - * Static definition of all code & data memory regions of the - * current Zephyr image. This information must be available & - * processed upon MMU initialization. - */ - -static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { - /* - * Mark the zephyr execution regions (data, bss, noinit, etc.) - * cacheable, read / write and non-executable - */ - { - /* This includes .data, .bss and various kobject sections. */ - .start = (uint32_t)_image_ram_start, - .end = (uint32_t)_image_ram_end, -#ifdef CONFIG_XTENSA_RPO_CACHE - .attrs = Z_XTENSA_MMU_W, -#else - .attrs = Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, -#endif - .name = "data", - }, - /* System heap memory */ - { - .start = (uint32_t)_heap_start, - .end = (uint32_t)_heap_end, -#ifdef CONFIG_XTENSA_RPO_CACHE - .attrs = Z_XTENSA_MMU_W, -#else - .attrs = Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, -#endif - .name = "heap", - }, - /* Mark text segment cacheable, read only and executable */ - { - .start = (uint32_t)__text_region_start, - .end = (uint32_t)__text_region_end, - .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB, - .name = "text", - }, - /* Mark rodata segment cacheable, read only and non-executable */ - { - .start = (uint32_t)__rodata_region_start, - .end = (uint32_t)__rodata_region_end, - .attrs = Z_XTENSA_MMU_CACHED_WB, - .name = "rodata", - }, -}; - -static inline uint32_t *alloc_l2_table(void) -{ - uint16_t idx; - - for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L2_TABLES; idx++) { - if (!atomic_test_and_set_bit(l2_page_tables_track, idx)) { - return (uint32_t *)&l2_page_tables[idx]; - } - } - - return NULL; -} - -static void map_memory_range(const uint32_t start, const uint32_t end, - const uint32_t attrs) -{ - uint32_t page, *table; - - for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { - uint32_t pte = Z_XTENSA_PTE(page, Z_XTENSA_KERNEL_RING, attrs); - uint32_t l2_pos = Z_XTENSA_L2_POS(page); - uint32_t l1_pos = page >> 22; - - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { - table = alloc_l2_table(); - - __ASSERT(table != NULL, "There is no l2 page table available to " - "map 0x%08x\n", page); - - l1_page_table[l1_pos] = - Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); - } - - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = pte; - } -} - -static void map_memory(const uint32_t start, const uint32_t end, - const uint32_t attrs) -{ - map_memory_range(start, end, attrs); - -#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP - if (arch_xtensa_is_ptr_uncached((void *)start)) { - map_memory_range(POINTER_TO_UINT(z_soc_cached_ptr((void *)start)), - POINTER_TO_UINT(z_soc_cached_ptr((void *)end)), - attrs | Z_XTENSA_MMU_CACHED_WB); - } else if (arch_xtensa_is_ptr_cached((void *)start)) { - map_memory_range(POINTER_TO_UINT(z_soc_uncached_ptr((void *)start)), - POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs); - } -#endif -} - -static void xtensa_init_page_tables(void) -{ - volatile uint8_t entry; - uint32_t page; - - for (page = 0; page < XTENSA_L1_PAGE_TABLE_ENTRIES; page++) { - l1_page_table[page] = Z_XTENSA_MMU_ILLEGAL; - } - - for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { - const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; - - map_memory(range->start, range->end, range->attrs); - } - -/** - * GCC complains about usage of the SoC MMU range ARRAY_SIZE - * (xtensa_soc_mmu_ranges) as the default weak declaration is - * an empty array, and any access to its element is considered - * out of bound access. However, we have a number of element - * variable to guard against this (... if done correctly). - * Besides, this will almost be overridden by the SoC layer. - * So tell GCC to ignore this. - */ -#if defined(__GNUC__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Warray-bounds" -#endif - for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) { - const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry]; - - map_memory(range->start, range->end, range->attrs); - } -#if defined(__GNUC__) -#pragma GCC diagnostic pop -#endif - - sys_cache_data_flush_all(); -} - -__weak void arch_xtensa_mmu_post_init(bool is_core0) -{ - ARG_UNUSED(is_core0); -} - -static void xtensa_mmu_init(bool is_core0) -{ - volatile uint8_t entry; - uint32_t ps, vecbase; - - if (is_core0) { - /* This is normally done via arch_kernel_init() inside z_cstart(). - * However, before that is called, we go through the sys_init of - * INIT_LEVEL_EARLY, which is going to result in TLB misses. - * So setup whatever necessary so the exception handler can work - * properly. - */ - z_xtensa_kernel_init(); - xtensa_init_page_tables(); - } - - /* Set the page table location in the virtual address */ - xtensa_ptevaddr_set((void *)Z_XTENSA_PTEVADDR); - - /* Next step is to invalidate the tlb entry that contains the top level - * page table. This way we don't cause a multi hit exception. - */ - xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, 6)); - xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, 6)); - - /* We are not using a flat table page, so we need to map - * only the top level page table (which maps the page table itself). - * - * Lets use one of the wired entry, so we never have tlb miss for - * the top level table. - */ - xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)l1_page_table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, MMU_PTE_WAY)); - - /* Before invalidate the text region in the TLB entry 6, we need to - * map the exception vector into one of the wired entries to avoid - * a page miss for the exception. - */ - __asm__ volatile("rsr.vecbase %0" : "=r"(vecbase)); - - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY( - Z_XTENSA_PTEVADDR + MB(4), 3)); - - xtensa_dtlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY( - Z_XTENSA_PTEVADDR + MB(4), 3)); - - /* Temporarily uses KernelExceptionVector for level 1 interrupts - * handling. This is due to UserExceptionVector needing to jump to - * _Level1Vector. The jump ('j') instruction offset is incorrect - * when we move VECBASE below. - */ - __asm__ volatile("rsr.ps %0" : "=r"(ps)); - ps &= ~PS_UM; - __asm__ volatile("wsr.ps %0; rsync" :: "a"(ps)); - - __asm__ volatile("wsr.vecbase %0; rsync\n\t" - :: "a"(Z_XTENSA_PTEVADDR + MB(4))); - - - /* Finally, lets invalidate all entries in way 6 as the page tables - * should have already mapped the regions we care about for boot. - */ - for (entry = 0; entry < BIT(XCHAL_ITLB_ARF_ENTRIES_LOG2); entry++) { - __asm__ volatile("iitlb %[idx]\n\t" - "isync" - :: [idx] "a"((entry << 29) | 6)); - } - - for (entry = 0; entry < BIT(XCHAL_DTLB_ARF_ENTRIES_LOG2); entry++) { - __asm__ volatile("idtlb %[idx]\n\t" - "dsync" - :: [idx] "a"((entry << 29) | 6)); - } - - /* Map VECBASE to a fixed data TLB */ - xtensa_dtlb_entry_write( - Z_XTENSA_PTE((uint32_t)vecbase, - Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_CACHED_WB), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, MMU_VECBASE_WAY)); - - /* - * Pre-load TLB for vecbase so exception handling won't result - * in TLB miss during boot, and that we can handle single - * TLB misses. - */ - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_AUTOFILL_TLB_ENTRY(vecbase)); - - /* To finish, just restore vecbase and invalidate TLB entries - * used to map the relocated vecbase. - */ - __asm__ volatile("wsr.vecbase %0; rsync\n\t" - :: "a"(vecbase)); - - /* Restore PS_UM so that level 1 interrupt handling will go to - * UserExceptionVector. - */ - __asm__ volatile("rsr.ps %0" : "=r"(ps)); - ps |= PS_UM; - __asm__ volatile("wsr.ps %0; rsync" :: "a"(ps)); - - xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); - xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); - - arch_xtensa_mmu_post_init(is_core0); -} - -void z_xtensa_mmu_init(void) -{ - xtensa_mmu_init(true); -} - -void z_xtensa_mmu_smp_init(void) -{ - xtensa_mmu_init(false); -} - -#ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES -/* Zephyr's linker scripts for Xtensa usually puts - * something before z_mapped_start (aka .text), - * i.e. vecbase, so that we need to reserve those - * space or else k_mem_map() would be mapping those, - * resulting in faults. - */ -__weak void arch_reserved_pages_update(void) -{ - uintptr_t page; - struct z_page_frame *pf; - int idx; - - for (page = CONFIG_SRAM_BASE_ADDRESS, idx = 0; - page < (uintptr_t)z_mapped_start; - page += CONFIG_MMU_PAGE_SIZE, idx++) { - pf = &z_page_frames[idx]; - - pf->flags |= Z_PAGE_FRAME_RESERVED; - } -} -#endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ - -static bool l2_page_table_map(void *vaddr, uintptr_t phys, uint32_t flags) -{ - uint32_t l1_pos = (uint32_t)vaddr >> 22; - uint32_t pte = Z_XTENSA_PTE(phys, Z_XTENSA_KERNEL_RING, flags); - uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); - uint32_t *table; - - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { - table = alloc_l2_table(); - - if (table == NULL) { - return false; - } - - l1_page_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); - } - - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = pte; - - if ((flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { - xtensa_itlb_vaddr_invalidate(vaddr); - } - xtensa_dtlb_vaddr_invalidate(vaddr); - return true; -} - -void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) -{ - uint32_t va = (uint32_t)virt; - uint32_t pa = (uint32_t)phys; - uint32_t rem_size = (uint32_t)size; - uint32_t xtensa_flags = 0; - int key; - - if (size == 0) { - LOG_ERR("Cannot map physical memory at 0x%08X: invalid " - "zero size", (uint32_t)phys); - k_panic(); - } - - switch (flags & K_MEM_CACHE_MASK) { - - case K_MEM_CACHE_WB: - xtensa_flags |= Z_XTENSA_MMU_CACHED_WB; - break; - case K_MEM_CACHE_WT: - xtensa_flags |= Z_XTENSA_MMU_CACHED_WT; - break; - case K_MEM_CACHE_NONE: - __fallthrough; - default: - break; - } - - if ((flags & K_MEM_PERM_RW) == K_MEM_PERM_RW) { - xtensa_flags |= Z_XTENSA_MMU_W; - } - if ((flags & K_MEM_PERM_EXEC) == K_MEM_PERM_EXEC) { - xtensa_flags |= Z_XTENSA_MMU_X; - } - - key = arch_irq_lock(); - - while (rem_size > 0) { - bool ret = l2_page_table_map((void *)va, pa, xtensa_flags); - - ARG_UNUSED(ret); - __ASSERT(ret, "Virtual address (%u) already mapped", (uint32_t)virt); - rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; - va += KB(4); - pa += KB(4); - } - - arch_irq_unlock(key); -} - -static void l2_page_table_unmap(void *vaddr) -{ - uint32_t l1_pos = (uint32_t)vaddr >> 22; - uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); - uint32_t *table; - uint32_t table_pos; - bool exec; - - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { - return; - } - - exec = l1_page_table[l1_pos] & Z_XTENSA_MMU_X; - - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; - - for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) { - if (table[l2_pos] != Z_XTENSA_MMU_ILLEGAL) { - goto end; - } - } - - l1_page_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; - table_pos = (table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); - atomic_clear_bit(l2_page_tables_track, table_pos); - - /* Need to invalidate L2 page table as it is no longer valid. */ - xtensa_dtlb_vaddr_invalidate((void *)table); - -end: - if (exec) { - xtensa_itlb_vaddr_invalidate(vaddr); - } - xtensa_dtlb_vaddr_invalidate(vaddr); -} - -void arch_mem_unmap(void *addr, size_t size) -{ - uint32_t va = (uint32_t)addr; - uint32_t rem_size = (uint32_t)size; - int key; - - if (addr == NULL) { - LOG_ERR("Cannot unmap NULL pointer"); - return; - } - - if (size == 0) { - LOG_ERR("Cannot unmap virtual memory with zero size"); - return; - } - - key = arch_irq_lock(); - - while (rem_size > 0) { - l2_page_table_unmap((void *)va); - rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; - va += KB(4); - } - - arch_irq_unlock(key); -} diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index 6427b306e62319..080b81d72bf952 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -25,7 +25,7 @@ extern void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf); K_KERNEL_STACK_ARRAY_DECLARE(z_interrupt_stacks, CONFIG_MP_MAX_NUM_CPUS, CONFIG_ISR_STACK_SIZE); -static ALWAYS_INLINE void z_xtensa_kernel_init(void) +static ALWAYS_INLINE void arch_kernel_init(void) { _cpu_t *cpu0 = &_kernel.cpus[0]; @@ -51,20 +51,28 @@ static ALWAYS_INLINE void z_xtensa_kernel_init(void) * win. */ XTENSA_WSR(ZSR_CPU_STR, cpu0); -} -static ALWAYS_INLINE void arch_kernel_init(void) -{ -#ifndef CONFIG_XTENSA_MMU - /* This is called in z_xtensa_mmu_init() before z_cstart() - * so we do not need to call it again. +#ifdef CONFIG_INIT_STACKS + char *stack_start = Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[0]); + size_t stack_sz = K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[0]); + char *stack_end = stack_start + stack_sz; + + uint32_t sp; + + __asm__ volatile("mov %0, sp" : "=a"(sp)); + + /* Only clear the interrupt stack if the current stack pointer + * is not within the interrupt stack. Or else we would be + * wiping the in-use stack. */ - z_xtensa_kernel_init(); + if (((uintptr_t)sp < (uintptr_t)stack_start) || + ((uintptr_t)sp >= (uintptr_t)stack_end)) { + memset(stack_start, 0xAA, stack_sz); + } #endif -#ifdef CONFIG_INIT_STACKS - memset(Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[0]), 0xAA, - K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[0])); +#ifdef CONFIG_XTENSA_MMU + z_xtensa_mmu_init(); #endif } @@ -180,6 +188,13 @@ static inline bool arch_is_in_isr(void) return arch_curr_cpu()->nested != 0U; } +#ifdef CONFIG_USERSPACE +extern void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3, + uintptr_t stack_end, + uintptr_t stack_start); +#endif /* CONFIG_USERSPACE */ + #ifdef __cplusplus } #endif diff --git a/arch/xtensa/include/offsets_short_arch.h b/arch/xtensa/include/offsets_short_arch.h index 34a4a5842cf753..f19750dc0ac8b9 100644 --- a/arch/xtensa/include/offsets_short_arch.h +++ b/arch/xtensa/include/offsets_short_arch.h @@ -2,4 +2,18 @@ * Copyright (c) 2021 Intel Corporation * SPDX-License-Identifier: Apache-2.0 */ -/* Empty File */ +#ifndef ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ +#define ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ + +#define _thread_offset_to_flags \ + (___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET) + +#ifdef CONFIG_USERSPACE +#define _thread_offset_to_psp \ + (___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET) + +#define _thread_offset_to_ptables \ + (___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET) +#endif /* CONFIG_USERSPACE */ + +#endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ */ diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index f691dbc6cad9b1..416a83453a2d0e 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -176,7 +176,8 @@ rsr.SCOMPARE1 a0 s32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) rur.THREADPTR a0 s32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET #endif @@ -409,6 +410,16 @@ _xstack_returned_\@: l32i a2, a1, 0 l32i a2, a2, ___xtensa_irq_bsa_t_scratch_OFFSET +#if XCHAL_HAVE_THREADPTR && defined(CONFIG_USERSPACE) + /* Clear up the threadptr because it is used + * to check if a thread is runnig on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possbly running in user mode. + */ + movi.n a0, 0 + wur.THREADPTR a0 +#endif /* XCHAL_HAVE_THREADPTR && CONFIG_USERSPACE */ + /* There's a gotcha with level 1 handlers: the INTLEVEL field * gets left at zero and not set like high priority interrupts * do. That works fine for exceptions, but for L1 interrupts, @@ -496,6 +507,8 @@ _do_call_\@: * spills to the right place. */ beq a6, a1, _restore_\@ + +#ifndef CONFIG_USERSPACE l32i a1, a1, 0 l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF @@ -505,7 +518,37 @@ _do_call_\@: */ SPILL_ALL_WINDOWS #endif + + /* Restore A1 stack pointer from "next" handle. */ mov a1, a6 +#else + /* With userspace, we cannot simply restore A1 stack pointer + * at this pointer because we need to swap page tables to + * the incoming thread, and we do not want to call that + * function with thread's stack. So we stash the new stack + * pointer into A2 first, then move it to A1 after we have + * swapped the page table. + */ + mov a2, a6 + + /* Need to switch page tables because the "next" handle + * returned above is not the same handle as we started + * with. This means we are being restored to another + * thread. + */ + rsr a6, ZSR_CPU + l32i a6, a6, ___cpu_t_current_OFFSET + + call4 z_xtensa_swap_update_page_tables + l32i a1, a1, 0 + l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET + addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF + + SPILL_ALL_WINDOWS + + /* Moved stashed stack pointer to A1 to restore stack. */ + mov a1, a2 +#endif _restore_\@: j _restore_context @@ -546,31 +589,6 @@ _Level\LVL\()VectorHelper : .global _Level\LVL\()Vector _Level\LVL\()Vector: #endif -#ifdef CONFIG_XTENSA_MMU - wsr.ZSR_EXTRA0 a2 - wsr.ZSR_EXTRA1 a3 - rsync - - /* Calculations below will clobber registers used. - * So we make a copy of the stack pointer to avoid - * changing it. - */ - mov a3, a1 - - CALC_PTEVADDR_BASE a2 - - /* Preload PTE entry page of current stack. */ - PRELOAD_PTEVADDR a3, a2 - - /* Preload PTE entry page of new stack, where - * it will be used later (in EXCINT_HANDLER above). - */ - rsr.ZSR_CPU a3 - PRELOAD_PTEVADDR a3, a2 - - rsr.ZSR_EXTRA1 a3 - rsr.ZSR_EXTRA0 a2 -#endif /* CONFIG_XTENSA_MMU */ addi a1, a1, -___xtensa_irq_bsa_t_SIZEOF s32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET s32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET diff --git a/drivers/console/xtensa_sim_console.c b/drivers/console/xtensa_sim_console.c index 16e48eeee484fd..316162ddc0c1af 100644 --- a/drivers/console/xtensa_sim_console.c +++ b/drivers/console/xtensa_sim_console.c @@ -13,7 +13,7 @@ * @param c Character to output * @return The character passed as input. */ -static int console_out(int c) +int arch_printk_char_out(int c) { char buf[16]; @@ -54,8 +54,8 @@ extern void __printk_hook_install(int (*fn)(int)); */ static void xt_sim_console_hook_install(void) { - __stdout_hook_install(console_out); - __printk_hook_install(console_out); + __stdout_hook_install(arch_printk_char_out); + __printk_hook_install(arch_printk_char_out); } /** diff --git a/include/zephyr/arch/syscall.h b/include/zephyr/arch/syscall.h index b657717e3d4bc6..5b41561b681907 100644 --- a/include/zephyr/arch/syscall.h +++ b/include/zephyr/arch/syscall.h @@ -23,6 +23,8 @@ #include #elif defined(CONFIG_RISCV) #include +#elif defined(CONFIG_XTENSA) +#include #endif #endif /* ZEPHYR_INCLUDE_ARCH_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 2d13615a753ac9..d486bb4d7b6212 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -23,21 +23,18 @@ #include #include #include +#include #include #include #include #include #include #include +#include +#include #include -#ifdef CONFIG_KERNEL_COHERENCE -#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE -#else -#define ARCH_STACK_PTR_ALIGN 16 -#endif - /* Xtensa GPRs are often designated by two different names */ #define sys_define_gpr_with_alias(name1, name2) union { uint32_t name1, name2; } @@ -47,13 +44,43 @@ extern "C" { #endif +struct arch_mem_domain { +#ifdef CONFIG_XTENSA_MMU + uint32_t *ptables __aligned(CONFIG_MMU_PAGE_SIZE); + uint8_t asid; + bool dirty; +#endif + sys_snode_t node; +}; + extern void xtensa_arch_except(int reason_p); +extern void xtensa_arch_kernel_oops(int reason_p, void *ssf); + +#ifdef CONFIG_USERSPACE + +#define ARCH_EXCEPT(reason_p) do { \ + if (k_is_user_context()) { \ + arch_syscall_invoke1(reason_p, \ + K_SYSCALL_XTENSA_USER_FAULT); \ + } else { \ + xtensa_arch_except(reason_p); \ + } \ + CODE_UNREACHABLE; \ +} while (false) + +#else #define ARCH_EXCEPT(reason_p) do { \ xtensa_arch_except(reason_p); \ CODE_UNREACHABLE; \ } while (false) +#endif + +__syscall void xtensa_user_fault(unsigned int reason); + +#include + /* internal routine documented in C file, needed by IRQ_CONNECT() macro */ extern void z_irq_priority_set(uint32_t irq, uint32_t prio, uint32_t flags); @@ -243,7 +270,33 @@ static inline void *arch_xtensa_uncached_ptr(void __sparse_cache *ptr) FOR_EACH(_SET_ONE_TLB, (;), 0, 1, 2, 3, 4, 5, 6, 7); \ } while (0) -#endif +#else /* CONFIG_XTENSA_RPO_CACHE */ + +static inline bool arch_xtensa_is_ptr_cached(void *ptr) +{ + ARG_UNUSED(ptr); + + return false; +} + +static inline bool arch_xtensa_is_ptr_uncached(void *ptr) +{ + ARG_UNUSED(ptr); + + return false; +} + +static inline void *arch_xtensa_cached_ptr(void *ptr) +{ + return ptr; +} + +static inline void *arch_xtensa_uncached_ptr(void *ptr) +{ + return ptr; +} + +#endif /* CONFIG_XTENSA_RPO_CACHE */ #ifdef CONFIG_XTENSA_MMU extern void arch_xtensa_mmu_post_init(bool is_core0); diff --git a/include/zephyr/arch/xtensa/syscall.h b/include/zephyr/arch/xtensa/syscall.h new file mode 100644 index 00000000000000..b8b0bea8cdbce9 --- /dev/null +++ b/include/zephyr/arch/xtensa/syscall.h @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief Xtensa specific syscall header + * + * This header contains the Xtensa specific syscall interface. It is + * included by the syscall interface architecture-abstraction header + * (include/arch/syscall.h) + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ +#define ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ + +#ifdef CONFIG_USERSPACE +#ifndef _ASMLANGUAGE + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER +uintptr_t xtensa_syscall_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id); + +#define SYSINL ALWAYS_INLINE +#else +#define SYSINL inline +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ + +/** + * We are following Linux Xtensa syscall ABI: + * + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ + +static SYSINL uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, arg5, arg6, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static SYSINL uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, arg5, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static SYSINL uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, 0, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static SYSINL uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, arg2, arg3, 0, 0, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static SYSINL uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, arg2, 0, 0, 0, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3) + : "memory"); + + return a2; +#endif +} + +static SYSINL uintptr_t arch_syscall_invoke1(uintptr_t arg1, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(arg1, 0, 0, 0, 0, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6) + : "memory"); + + return a2; +#endif +} + +static SYSINL uintptr_t arch_syscall_invoke0(uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return xtensa_syscall_helper(0, 0, 0, 0, 0, 0, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2) + : "memory"); + + return a2; +#endif +} + +/* + * There is no easy (or generic) way to figure out if a thread is runnining + * in un-privileged mode. Reading the currrent ring (PS.CRING) is a privileged + * instruction and not thread local storage is not available in xcc. + */ +static inline bool arch_is_user_context(void) +{ + uint32_t thread; + + __asm__ volatile( + "rur.THREADPTR %0\n\t" + : "=a" (thread) + ); +#ifdef CONFIG_THREAD_LOCAL_STORAGE + extern __thread uint32_t is_user_mode; + + if (!thread) { + return false; + } + + return is_user_mode != 0; +#else + return !!thread; +#endif +} + +#undef SYSINL + +#ifdef __cplusplus +} +#endif + +#endif /* _ASMLANGUAGE */ +#endif /* CONFIG_USERSPACE */ +#endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/thread.h b/include/zephyr/arch/xtensa/thread.h index 4ec5da1ea2c048..2bebe2722bcb2d 100644 --- a/include/zephyr/arch/xtensa/thread.h +++ b/include/zephyr/arch/xtensa/thread.h @@ -7,6 +7,7 @@ #ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ #define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ +#include #ifndef _ASMLANGUAGE /* Xtensa doesn't use these structs, but Zephyr core requires they be @@ -22,6 +23,14 @@ typedef struct _callee_saved _callee_saved_t; struct _thread_arch { uint32_t last_cpu; +#ifdef CONFIG_USERSPACE + uint32_t *ptables; + + /* Initial privilege mode stack pointer when doing a system call. + * Un-set for surpervisor threads. + */ + uint8_t *psp; +#endif }; typedef struct _thread_arch _thread_arch_t; diff --git a/include/zephyr/arch/xtensa/thread_stack.h b/include/zephyr/arch/xtensa/thread_stack.h new file mode 100644 index 00000000000000..eaa160ccf1fdd4 --- /dev/null +++ b/include/zephyr/arch/xtensa/thread_stack.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_STACK_H_ +#define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_STACK_H_ + +#include +#include + +#ifdef CONFIG_KERNEL_COHERENCE +#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE +#else +#define ARCH_STACK_PTR_ALIGN 16 +#endif + + +#if CONFIG_USERSPACE +#define Z_XTENSA_STACK_BASE_ALIGN CONFIG_MMU_PAGE_SIZE +#define Z_XTENSA_STACK_SIZE_ALIGN CONFIG_MMU_PAGE_SIZE +#else +#define Z_XTENSA_STACK_BASE_ALIGN ARCH_STACK_PTR_ALIGN +#define Z_XTENSA_STACK_SIZE_ALIGN ARCH_STACK_PTR_ALIGN +#endif + +/* + * + * High memory addresses + * + * +-------------------+ <- thread.stack_info.start + thread.stack_info.size + * | TLS | + * +-------------------+ <- initial sp (computable with thread.stack_info.delta) + * | | + * | Thread stack | + * | | + * +-------------------+ <- thread.stack_info.start + * | Privileged stack | } CONFIG_MMU_PAGE_SIZE + * +-------------------+ <- thread.stack_obj + * + * Low Memory addresses + */ + +#ifndef _ASMLANGUAGE + +/* thread stack */ +#ifdef CONFIG_XTENSA_MMU +struct z_xtensa_thread_stack_header { + char privilege_stack[CONFIG_MMU_PAGE_SIZE]; +} __packed __aligned(Z_XTENSA_STACK_BASE_ALIGN); + +#define ARCH_THREAD_STACK_RESERVED \ + sizeof(struct z_xtensa_thread_stack_header) +#endif /* CONFIG_XTENSA_MMU */ + +#define ARCH_THREAD_STACK_OBJ_ALIGN(size) Z_XTENSA_STACK_BASE_ALIGN +#define ARCH_THREAD_STACK_SIZE_ADJUST(size) \ + ROUND_UP((size), Z_XTENSA_STACK_SIZE_ALIGN) + +/* kernel stack */ +#define ARCH_KERNEL_STACK_RESERVED 0 +#define ARCH_KERNEL_STACK_OBJ_ALIGN ARCH_STACK_PTR_ALIGN + +#endif /* _ASMLANGUAGE */ + +#endif diff --git a/include/zephyr/arch/xtensa/xtensa_mmu.h b/include/zephyr/arch/xtensa/xtensa_mmu.h index d03f876af30b11..aa00f935a8a4f0 100644 --- a/include/zephyr/arch/xtensa/xtensa_mmu.h +++ b/include/zephyr/arch/xtensa/xtensa_mmu.h @@ -9,8 +9,40 @@ #define Z_XTENSA_MMU_X BIT(0) #define Z_XTENSA_MMU_W BIT(1) +#define Z_XTENSA_MMU_XW (BIT(1) | BIT(0)) + #define Z_XTENSA_MMU_CACHED_WB BIT(2) #define Z_XTENSA_MMU_CACHED_WT BIT(3) + +#define K_MEM_PARTITION_IS_EXECUTABLE(attr) (((attr) & Z_XTENSA_MMU_X) != 0) +#define K_MEM_PARTITION_IS_WRITABLE(attr) (((attr) & Z_XENSA_MMU_W) != 0) + +/* Read-Write access permission attributes */ +#define K_MEM_PARTITION_P_RW_U_RW ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_W}) +#define K_MEM_PARTITION_P_RW_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_RO ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_NA_U_NA ((k_mem_partition_attr_t) \ + {0}) + +/* Execution-allowed attributes */ +#define K_MEM_PARTITION_P_RX_U_RX ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_X}) + +/* + * This BIT tells the mapping code whether the uncached pointer should + * be shared between all threads. That is not used in the HW, it is + * just for the implementation. + * + * The pte mapping this memory will use an ASID that is set in the + * ring 4 spot in RASID. + */ +#define Z_XTENSA_MMU_MAP_SHARED BIT(30) + #define Z_XTENSA_MMU_ILLEGAL (BIT(3) | BIT(2)) /* Struct used to map a memory region */ @@ -21,11 +53,31 @@ struct xtensa_mmu_range { const uint32_t attrs; }; +typedef uint32_t k_mem_partition_attr_t; + extern const struct xtensa_mmu_range xtensa_soc_mmu_ranges[]; extern int xtensa_soc_mmu_ranges_num; void z_xtensa_mmu_init(void); -void z_xtensa_mmu_smp_init(void); +/** + * @brief Tell other processors to flush TLBs. + * + * This sends IPI to other processors to telling them to + * invalidate cache to page tables and flush TLBs. This is + * needed when one processor is updating page tables that + * may affect threads running on other processors. + * + * @note This needs to be implemented in the SoC layer. + */ +void z_xtensa_mmu_tlb_ipi(void); + +/** + * @brief Invalidate cache to page tables and flush TLBs. + * + * This invalidates cache to all page tables and flush TLBs + * as they may have been modified by other processors. + */ +void z_xtensa_mmu_tlb_shootdown(void); #endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_XTENSA_MMU_H */ diff --git a/include/zephyr/kernel.h b/include/zephyr/kernel.h index fac7df01904fb6..d2b912f6d46e13 100644 --- a/include/zephyr/kernel.h +++ b/include/zephyr/kernel.h @@ -596,7 +596,8 @@ __syscall k_tid_t k_sched_current_thread_query(void); __attribute_const__ static inline k_tid_t k_current_get(void) { -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS + /* Thread-local cache of current thread ID, set in z_thread_entry() */ extern __thread k_tid_t z_tls_current; diff --git a/kernel/Kconfig b/kernel/Kconfig index 6280e80a400b4b..4a72d76da67054 100644 --- a/kernel/Kconfig +++ b/kernel/Kconfig @@ -288,6 +288,20 @@ config ERRNO_IN_TLS Use thread local storage to store errno instead of storing it in the kernel thread struct. This avoids a syscall if userspace is enabled. +config CURRENT_THREAD_USE_NO_TLS + bool + help + Hidden symbol to not use thread local storage to store current + thread. + +config CURRENT_THREAD_USE_TLS + bool "Store current thread in thread local storage (TLS)" + depends on THREAD_LOCAL_STORAGE && !CURRENT_THREAD_USE_NO_TLS + default y + help + Use thread local storage to store the current thread. This avoids a + syscall if userspace is enabled. + choice SCHED_ALGORITHM prompt "Scheduler priority queue algorithm" default SCHED_DUMB diff --git a/lib/os/thread_entry.c b/lib/os/thread_entry.c index 89eb2fe7b4b32f..ed6ca142d99f7c 100644 --- a/lib/os/thread_entry.c +++ b/lib/os/thread_entry.c @@ -12,7 +12,7 @@ */ #include -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS #include __thread k_tid_t z_tls_current; @@ -35,7 +35,7 @@ extern __thread volatile uintptr_t __stack_chk_guard; FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry, void *p1, void *p2, void *p3) { -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS z_tls_current = k_sched_current_thread_query(); #endif #ifdef CONFIG_STACK_CANARIES_TLS diff --git a/soc/xtensa/dc233c/Kconfig.soc b/soc/xtensa/dc233c/Kconfig.soc index 89b814da4d2bd9..617eaa13849b16 100644 --- a/soc/xtensa/dc233c/Kconfig.soc +++ b/soc/xtensa/dc233c/Kconfig.soc @@ -9,3 +9,4 @@ config SOC_XTENSA_DC233C select ARCH_HAS_THREAD_LOCAL_STORAGE select CPU_HAS_MMU select ARCH_HAS_RESERVED_PAGE_FRAMES if XTENSA_MMU + select ARCH_HAS_USERSPACE if XTENSA_MMU diff --git a/soc/xtensa/dc233c/include/xtensa-dc233c.ld b/soc/xtensa/dc233c/include/xtensa-dc233c.ld index b165016fa6a1da..9c120e1b9d8622 100644 --- a/soc/xtensa/dc233c/include/xtensa-dc233c.ld +++ b/soc/xtensa/dc233c/include/xtensa-dc233c.ld @@ -20,7 +20,7 @@ #include #define RAMABLE_REGION RAM :sram0_phdr -#define ROMABLE_REGION rom0_seg :rom0_phdr +#define ROMABLE_REGION RAM :sram0_phdr #ifdef CONFIG_MMU #define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); @@ -287,6 +287,9 @@ SECTIONS _DoubleExceptionVector_text_end = ABSOLUTE(.); } >sram0_18_seg :sram0_18_phdr +#define LIB_OBJ_FUNC_IN_SECT(library, obj_file, func) \ + *##library##:##obj_file##(.literal.##func .text.##func) \ + #ifdef CONFIG_XTENSA_MMU .vec_helpers : { @@ -301,48 +304,45 @@ SECTIONS * TLB multi-hit exception. */ - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.literal) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.text) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram.text) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram0.text) + *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.literal .text) + *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram.text .iram0.text) *libarch__xtensa__core.a:window_vectors.S.obj(.iram.text) - *libarch__xtensa__core.a:xtensa-asm2.c.obj(.literal.*) - *libarch__xtensa__core.a:xtensa-asm2.c.obj(.text.*) - - *libarch__xtensa__core.a:fatal.c.obj(.literal.*) - *libarch__xtensa__core.a:fatal.c.obj(.text.*) - - *libarch__xtensa__core.a:crt1.S.obj(.literal) - *libarch__xtensa__core.a:crt1.S.obj(.text) + *libarch__xtensa__core.a:crt1.S.obj(.literal .text) - *libarch__xtensa__core.a:cpu_idle.c.obj(.literal.*) - *libarch__xtensa__core.a:cpu_idle.c.obj(.text.*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa-asm2.c.obj,*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,fatal.c.obj,*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,cpu_idle.c.obj,*) *(.text.arch_is_in_isr) /* To support backtracing */ - *libarch__xtensa__core.a:xtensa_backtrace.c.obj(.literal.*) - *libarch__xtensa__core.a:xtensa_backtrace.c.obj(.text.*) - *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1.literal) - *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa_backtrace.c.obj,*) - *libkernel.a:fatal.c.obj(.literal.*) - *libkernel.a:fatal.c.obj(.text.*) + *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1.literal .iram1) + + /* Userspace related stuff */ + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,userspace.S.obj,z_xtensa_do_syscall) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,ptables.c.obj,z_xtensa_swap_update_page_tables) /* Below are to speed up execution by avoiding TLB misses * on frequently used functions. + * + * There is almost 1MB space (due to TLB pinning) so we can + * be generous. */ - *libkernel.a:sched.c.obj(.literal.*) - *libkernel.a:sched.c.obj(.text.*) - *libkernel.a:timeout.c.obj(.literal.*) - *libkernel.a:timeout.c.obj(.text.*) - - *libdrivers__console.a:(.literal.*) - *libdrivers__console.a:(.text.*) - *libdrivers__timer.a:(.literal.*) - *libdrivers__timer.a:(.text.*) + LIB_OBJ_FUNC_IN_SECT(libkernel.a,,*) + + LIB_OBJ_FUNC_IN_SECT(libdrivers__console.a,,*) + LIB_OBJ_FUNC_IN_SECT(libdrivers__timer.a,,*) + + *(.literal.z_vrfy_* .text.z_vrfy_*) + *(.literal.z_mrsh_* .text.z_mrsh_*) + *(.literal.z_impl_* .text.z_impl_*) + *(.literal.z_obj_* .text.z_obj_*) + + *(.literal.k_sys_fatal_error_handler .text.k_sys_fatal_error_handler) } >vec_helpers :vec_helpers_phdr #endif /* CONFIG_XTENSA_MMU */ @@ -380,7 +380,7 @@ SECTIONS _text_end = ABSOLUTE(.); _etext = .; - } >RAM :sram0_phdr + } >RAMABLE_REGION __text_region_end = .; .rodata : HDR_MMU_PAGE_ALIGN @@ -394,7 +394,16 @@ SECTIONS . = ALIGN(4); #include #include + } >RAMABLE_REGION + +#include + +#include +#include + + .rodata_end : ALIGN(4) + { . = ALIGN(4); /* this table MUST be 4-byte aligned */ _bss_table_start = ABSOLUTE(.); LONG(_bss_start) @@ -404,19 +413,26 @@ SECTIONS MMU_PAGE_ALIGN __rodata_region_end = ABSOLUTE(.); - } >RAM :sram0_phdr + } >RAMABLE_REGION -#include +#ifdef CONFIG_USERSPACE +#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN +#define APP_SHARED_ALIGN MMU_PAGE_ALIGN -#include +#include -#include - -#include + _image_ram_start = _app_smem_start; + _app_smem_size = _app_smem_end - _app_smem_start; + _app_smem_num_words = _app_smem_size >> 2; + _app_smem_rom_start = LOADADDR(_APP_SMEM_SECTION_NAME); + _app_smem_num_words = _app_smem_size >> 2; +#endif /* CONFIG_USERSPACE */ .data : HDR_MMU_PAGE_ALIGN { +#ifndef CONFIG_USERSPACE _image_ram_start = ABSOLUTE(.); +#endif __data_start = ABSOLUTE(.); *(.data) *(.data.*) @@ -438,7 +454,9 @@ SECTIONS MMU_PAGE_ALIGN __data_end = ABSOLUTE(.); - } >RAM :sram0_phdr + } >RAMABLE_REGION + +#include #include diff --git a/soc/xtensa/dc233c/mmu.c b/soc/xtensa/dc233c/mmu.c index 4b8d6b154b84c6..8decc952bf2ac6 100644 --- a/soc/xtensa/dc233c/mmu.c +++ b/soc/xtensa/dc233c/mmu.c @@ -18,7 +18,7 @@ const struct xtensa_mmu_range xtensa_soc_mmu_ranges[] = { { .start = (uint32_t)XCHAL_VECBASE_RESET_VADDR, .end = (uint32_t)CONFIG_SRAM_OFFSET, - .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "vecbase", }, { @@ -35,32 +35,3 @@ const struct xtensa_mmu_range xtensa_soc_mmu_ranges[] = { }; int xtensa_soc_mmu_ranges_num = ARRAY_SIZE(xtensa_soc_mmu_ranges); - -void arch_xtensa_mmu_post_init(bool is_core0) -{ - uint32_t vecbase; - - ARG_UNUSED(is_core0); - - __asm__ volatile("rsr.vecbase %0" : "=r"(vecbase)); - - /* Invalidate any autorefill instr TLBs of VECBASE so we can map it - * permanently below. - */ - xtensa_itlb_vaddr_invalidate((void *)vecbase); - - /* Map VECBASE permanently in instr TLB way 4 so we will always have - * access to exception handlers. Each way 4 TLB covers 1MB (unless - * ITLBCFG has been changed before this, which should not have - * happened). - * - * Note that we don't want to map the first 1MB in data TLB as - * we want to keep page 0 (0x00000000) unmapped to catch null pointer - * de-references. - */ - vecbase = ROUND_DOWN(vecbase, MB(1)); - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, 4)); -} diff --git a/tests/kernel/mem_protect/mem_protect/src/mem_protect.h b/tests/kernel/mem_protect/mem_protect/src/mem_protect.h index ef0ba48a519048..4d5400bb1d6bbf 100644 --- a/tests/kernel/mem_protect/mem_protect/src/mem_protect.h +++ b/tests/kernel/mem_protect/mem_protect/src/mem_protect.h @@ -67,6 +67,8 @@ static inline void set_fault_valid(bool valid) #else #define MEM_REGION_ALLOC (4) #endif +#elif defined(CONFIG_XTENSA) +#define MEM_REGION_ALLOC (4096) #else #error "Test suite not compatible for the given architecture" #endif diff --git a/tests/kernel/mem_protect/userspace/src/main.c b/tests/kernel/mem_protect/userspace/src/main.c index 1c0e7d3ef21010..bcf504102db87c 100644 --- a/tests/kernel/mem_protect/userspace/src/main.c +++ b/tests/kernel/mem_protect/userspace/src/main.c @@ -20,6 +20,11 @@ #include "test_syscall.h" #include /* for z_libc_partition */ +#if defined(CONFIG_XTENSA) +#include +#include +#endif + #if defined(CONFIG_ARC) #include #endif @@ -178,6 +183,12 @@ ZTEST_USER(userspace, test_write_control) set_fault(K_ERR_CPU_EXCEPTION); __asm__ volatile("csrr %0, mstatus" : "=r" (status)); +#elif defined(CONFIG_XTENSA) + unsigned int ps; + + set_fault(K_ERR_CPU_EXCEPTION); + + __asm__ volatile("rsr.ps %0" : "=r" (ps)); #else #error "Not implemented for this architecture" zassert_unreachable("Write to control register did not fault"); @@ -245,6 +256,24 @@ ZTEST_USER(userspace, test_disable_mmu_mpu) */ csr_write(pmpaddr3, LLONG_MAX); csr_write(pmpcfg0, (PMP_R|PMP_W|PMP_X|PMP_NAPOT) << 24); +#elif defined(CONFIG_XTENSA) + set_fault(K_ERR_CPU_EXCEPTION); + + /* Reset way 6 to do identity mapping. + * Complier would complain addr going out of range if we + * simply do addr = i * 0x20000000 inside the loop. So + * we do increment instead. + */ + uint32_t addr = 0U; + + for (int i = 0; i < 8; i++) { + uint32_t attr = addr | Z_XTENSA_MMU_XW; + + __asm__ volatile("wdtlb %0, %1; witlb %0, %1" + :: "r"(attr), "r"(addr)); + + addr += 0x20000000; + } #else #error "Not implemented for this architecture" #endif @@ -381,7 +410,8 @@ ZTEST_USER(userspace, test_read_priv_stack) s[0] = 0; priv_stack_ptr = (char *)&s[0] - size; -#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || defined(CONFIG_ARM64) +#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || \ + defined(CONFIG_ARM64) || defined(CONFIG_XTENSA) /* priv_stack_ptr set by test_main() */ #else #error "Not implemented for this architecture" @@ -405,7 +435,8 @@ ZTEST_USER(userspace, test_write_priv_stack) s[0] = 0; priv_stack_ptr = (char *)&s[0] - size; -#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || defined(CONFIG_ARM64) +#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || \ + defined(CONFIG_ARM64) || defined(CONFIG_XTENSA) /* priv_stack_ptr set by test_main() */ #else #error "Not implemented for this architecture"