From cb1d121b2a7fc1ed6c6184b99decb397dbb9b540 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 24 Aug 2023 12:20:50 -0700 Subject: [PATCH 01/33] xtensa: mmu: implement cached/uncached ptr funcs if !RPO_CACHE This implements the following functions when CONFIG_XTENSA_RPO_CACHE is false: * arch_xtensa_is_ptr_cached() returns false * arch_xtensa_is_ptr_uncached() returns false * arch_xtensa_cached_ptr() returns unmodified pointer * arch_xtensa_uncached_ptr() returns unmodified pointer Signed-off-by: Daniel Leung --- include/zephyr/arch/xtensa/arch.h | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 2d13615a753ac9..0838e68ee5a64d 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -243,7 +243,33 @@ static inline void *arch_xtensa_uncached_ptr(void __sparse_cache *ptr) FOR_EACH(_SET_ONE_TLB, (;), 0, 1, 2, 3, 4, 5, 6, 7); \ } while (0) -#endif +#else /* CONFIG_XTENSA_RPO_CACHE */ + +static inline bool arch_xtensa_is_ptr_cached(void *ptr) +{ + ARG_UNUSED(ptr); + + return false; +} + +static inline bool arch_xtensa_is_ptr_uncached(void *ptr) +{ + ARG_UNUSED(ptr); + + return false; +} + +static inline void *arch_xtensa_cached_ptr(void *ptr) +{ + return ptr; +} + +static inline void *arch_xtensa_uncached_ptr(void *ptr) +{ + return ptr; +} + +#endif /* CONFIG_XTENSA_RPO_CACHE */ #ifdef CONFIG_XTENSA_MMU extern void arch_xtensa_mmu_post_init(bool is_core0); From b3c2528807f318d48b4d548fc4c10241072622d1 Mon Sep 17 00:00:00 2001 From: Andy Ross Date: Mon, 21 Aug 2023 16:54:04 -0700 Subject: [PATCH 02/33] arch/xtensa: Rename "ALLOCA" ZSR to "A0SAVE" This register alias was originally introduced to allow A0 to be used as a scratch register when handling exceptions from MOVSP instructions. (It replaced some upstream code from Cadence that hard-coded EXCSAVE1). Now the MMU code is now using too, and for exactly the same purpose. Calling it "ALLOCA" is only confusing. Rename it to make it clear what it's doing. Signed-off-by: Andy Ross --- arch/xtensa/core/gen_zsr.py | 2 +- arch/xtensa/core/window_vectors.S | 4 ++-- arch/xtensa/core/xtensa-asm2-util.S | 18 +++++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/arch/xtensa/core/gen_zsr.py b/arch/xtensa/core/gen_zsr.py index 8d052f4891d70f..574542a4578ed1 100755 --- a/arch/xtensa/core/gen_zsr.py +++ b/arch/xtensa/core/gen_zsr.py @@ -11,7 +11,7 @@ # -dM") core-isa.h file for the current architecture and assigns # registers to usages. -NEEDED = ("ALLOCA", "CPU", "FLUSH") +NEEDED = ("A0SAVE", "CPU", "FLUSH") coreisa = sys.argv[1] outfile = sys.argv[2] diff --git a/arch/xtensa/core/window_vectors.S b/arch/xtensa/core/window_vectors.S index a63923cbd6fc2d..90eba495bde809 100644 --- a/arch/xtensa/core/window_vectors.S +++ b/arch/xtensa/core/window_vectors.S @@ -84,7 +84,7 @@ _WindowUnderflow4: /* Handle alloca exception generated by interruptee executing 'movsp'. * This uses space between the window vectors, so is essentially * "free". All interruptee's regs are intact except a0 which is saved - * in $ZSR_ALLOCA (assigned at build time, see gen_zsr.py for + * in $ZSR_A0SAVE (assigned at build time, see gen_zsr.py for * details), and PS.EXCM has been set by the exception hardware (can't * be interrupted). The fact the alloca exception was taken means the * registers associated with the base-save area have been spilled and @@ -102,7 +102,7 @@ _xt_alloca_exc: rsr a2, PS extui a3, a2, XCHAL_PS_OWB_SHIFT, XCHAL_PS_OWB_BITS xor a3, a3, a4 /* bits changed from old to current windowbase */ - rsr a4, ZSR_ALLOCA /* restore original a0 (now in a4) */ + rsr a4, ZSR_A0SAVE /* restore original a0 (now in a4) */ slli a3, a3, XCHAL_PS_OWB_SHIFT xor a2, a2, a3 /* flip changed bits in old window base */ wsr a2, PS /* update PS.OWB to new window base */ diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index d7ba88aaffc397..bc526ae0165e24 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -342,7 +342,7 @@ DEF_EXCINT XCHAL_DEBUGLEVEL, _handle_excint, xtensa_debugint_c .pushsection .UserExceptionVector.text, "ax" .global _Level1RealVector _Level1RealVector: - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_A0SAVE rsync rsr.exccause a0 #ifdef CONFIG_XTENSA_MMU @@ -355,7 +355,7 @@ _Level1RealVector: j _xt_alloca_exc _not_alloca: - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE j _Level1Vector #ifdef CONFIG_XTENSA_MMU _handle_tlb_miss_user: @@ -374,7 +374,7 @@ _handle_tlb_miss_user: */ rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE rfe #endif /* CONFIG_XTENSA_MMU */ .popsection @@ -391,12 +391,12 @@ _handle_tlb_miss_user: .global _KernelExceptionVector _KernelExceptionVector: #ifdef CONFIG_XTENSA_MMU - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_A0SAVE rsr.exccause a0 beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_kernel addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_kernel - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE #endif j _Level1Vector #ifdef CONFIG_XTENSA_MMU @@ -410,7 +410,7 @@ _handle_tlb_miss_kernel: */ rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE rfe #endif .popsection @@ -420,14 +420,14 @@ _handle_tlb_miss_kernel: .global _DoubleExceptionVector _DoubleExceptionVector: #ifdef CONFIG_XTENSA_MMU - wsr a0, ZSR_ALLOCA + wsr a0, ZSR_A0SAVE rsync rsr.exccause a0 addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_dblexc - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE #endif #if defined(CONFIG_SIMULATOR_XTENSA) || defined(XT_SIMULATOR) 1: @@ -459,7 +459,7 @@ _handle_tlb_miss_dblexc: rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_ALLOCA + rsr a0, ZSR_A0SAVE rfde #endif .popsection From 83768732564ac2e52eb6b6d89598e2075366abae Mon Sep 17 00:00:00 2001 From: Andy Ross Date: Sun, 27 Aug 2023 09:03:20 -0700 Subject: [PATCH 03/33] drivers/console: xtensa_sim_console: implement arch_printk_char_out() This is an older driver and didn't support the weak arch_printk_char_out() hook, which is a link-time symbol that allows logging to work from the first instruction. Some drivers can't do that because they need an initialization step, but this one works great. Signed-off-by: Andy Ross --- drivers/console/xtensa_sim_console.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/console/xtensa_sim_console.c b/drivers/console/xtensa_sim_console.c index 16e48eeee484fd..316162ddc0c1af 100644 --- a/drivers/console/xtensa_sim_console.c +++ b/drivers/console/xtensa_sim_console.c @@ -13,7 +13,7 @@ * @param c Character to output * @return The character passed as input. */ -static int console_out(int c) +int arch_printk_char_out(int c) { char buf[16]; @@ -54,8 +54,8 @@ extern void __printk_hook_install(int (*fn)(int)); */ static void xt_sim_console_hook_install(void) { - __stdout_hook_install(console_out); - __printk_hook_install(console_out); + __stdout_hook_install(arch_printk_char_out); + __printk_hook_install(arch_printk_char_out); } /** From 079bc92c6229238d89010b8646609c69cbc21568 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Fri, 17 Nov 2023 13:54:36 -0800 Subject: [PATCH 04/33] xtensa: mmu: Simplify autorefill TLB helpers Replace all autorefill helpers with only one that invalidates both, DTLB and ITLB, since that is what is really needed. Signed-off-by: Flavio Ceolin --- arch/xtensa/core/include/xtensa_mmu_priv.h | 115 +++------------------ 1 file changed, 14 insertions(+), 101 deletions(-) diff --git a/arch/xtensa/core/include/xtensa_mmu_priv.h b/arch/xtensa/core/include/xtensa_mmu_priv.h index 7519c1b6010649..dcfb9deefc0d54 100644 --- a/arch/xtensa/core/include/xtensa_mmu_priv.h +++ b/arch/xtensa/core/include/xtensa_mmu_priv.h @@ -163,119 +163,32 @@ static ALWAYS_INLINE void xtensa_itlb_entry_write_sync(uint32_t pte, uint32_t en } /** - * @brief Invalidate all ITLB entries. + * @brief Invalidate all autorefill DTLB and ITLB entries. * - * This should be used carefully since all entries in the instruction TLB - * will be erased and the only way to find lookup a physical address will be - * through the page tables. - */ -static inline void xtensa_itlb_invalidate_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_ITLB_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_itlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} - -/** - * @brief Invalidate all DTLB entries. + * This should be used carefully since all refill entries in the data + * and instruction TLB. At least two pages, the current code page and + * the current stack, will be repopulated by this code as it returns. * - * This should be used carefully since all entries in the data TLB will be - * erased and the only way to find lookup a physical address will be through - * the page tables. + * This needs to be called in any circumstance where the mappings for + * a previously-used page table change. It does not need to be called + * on context switch, where ASID tagging isolates entries for us. */ -static inline void xtensa_dtlb_invalidate_sync(void) +static inline void xtensa_tlb_autorefill_invalidate(void) { - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_DTLB_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_dtlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} + uint8_t way, i, entries; -/** - * @brief Invalidates an autorefill DTLB entry. - * - * Invalidates the page table enrty that maps a given virtual address. - */ -static inline void xtensa_dtlb_autorefill_invalidate_sync(void *vaddr) -{ - uint8_t way; + entries = BIT(MAX(XCHAL_ITLB_ARF_ENTRIES_LOG2, + XCHAL_DTLB_ARF_ENTRIES_LOG2)); for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - xtensa_dtlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way)); - } - __asm__ volatile("dsync"); -} - -/** - * @brief Invalidates an autorefill ITLB entry. - * - * Invalidates the page table enrty that maps a given virtual address. - */ -static inline void xtensa_itlb_autorefill_invalidate_sync(void *vaddr) -{ - uint8_t way; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - xtensa_itlb_entry_invalidate(Z_XTENSA_TLB_ENTRY((uint32_t)vaddr, way)); - } - __asm__ volatile("isync"); -} -/** - * @brief Invalidate all autorefill ITLB entries. - * - * This should be used carefully since all entries in the instruction TLB - * will be erased and the only way to find lookup a physical address will be - * through the page tables. - */ -static inline void xtensa_itlb_autorefill_invalidate_all_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { + for (i = 0; i < entries; i++) { uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_itlb_entry_invalidate(entry); + xtensa_dtlb_entry_invalidate_sync(entry); + xtensa_itlb_entry_invalidate_sync(entry); } } - __asm__ volatile("isync"); } -/** - * @brief Invalidate all autorefill DTLB entries. - * - * This should be used carefully since all entries in the data TLB will be - * erased and the only way to find lookup a physical address will be through - * the page tables. - */ -static inline void xtensa_dtlb_autorefill_invalidate_all_sync(void) -{ - uint8_t way, i; - - for (way = 0; way < Z_XTENSA_TLB_AUTOREFILL_WAYS; way++) { - for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { - uint32_t entry = way + (i << Z_XTENSA_PPN_SHIFT); - - xtensa_dtlb_entry_invalidate(entry); - } - } - __asm__ volatile("isync"); -} - - /** * @brief Set the page tables. * From 97fdcbac3f4ac40d52828c466c9cf7a985d9d279 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Wed, 1 Nov 2023 22:55:43 -0700 Subject: [PATCH 05/33] xtensa: mmu: Simplify initialization Simplify the logic around xtensa_mmu_init. - Do not have a different path to init part of kernel - Call xtensa_mmu_init from C Signed-off-by: Flavio Ceolin --- arch/xtensa/core/crt1.S | 8 -------- arch/xtensa/core/xtensa_mmu.c | 17 +++-------------- arch/xtensa/include/kernel_arch_func.h | 16 +++++----------- include/zephyr/arch/xtensa/xtensa_mmu.h | 2 -- 4 files changed, 8 insertions(+), 35 deletions(-) diff --git a/arch/xtensa/core/crt1.S b/arch/xtensa/core/crt1.S index 0fa3ad230995c4..c616b0889d7198 100644 --- a/arch/xtensa/core/crt1.S +++ b/arch/xtensa/core/crt1.S @@ -24,10 +24,6 @@ .global __start .type z_cstart, @function -#ifdef CONFIG_XTENSA_MMU -.type z_xtensa_mmu_init, @function -#endif - /* Macros to abstract away ABI differences */ @@ -192,10 +188,6 @@ _start: #endif /* !XCHAL_HAVE_BOOTLOADER */ -#ifdef CONFIG_XTENSA_MMU - CALL z_xtensa_mmu_init -#endif - /* Enter C domain, never returns from here */ CALL z_cstart diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index e50c51a3848f55..18ce5f22f63466 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -213,19 +213,18 @@ __weak void arch_xtensa_mmu_post_init(bool is_core0) ARG_UNUSED(is_core0); } -static void xtensa_mmu_init(bool is_core0) +void z_xtensa_mmu_init(void) { volatile uint8_t entry; uint32_t ps, vecbase; - if (is_core0) { + if (_current_cpu->id == 0) { /* This is normally done via arch_kernel_init() inside z_cstart(). * However, before that is called, we go through the sys_init of * INIT_LEVEL_EARLY, which is going to result in TLB misses. * So setup whatever necessary so the exception handler can work * properly. */ - z_xtensa_kernel_init(); xtensa_init_page_tables(); } @@ -326,17 +325,7 @@ static void xtensa_mmu_init(bool is_core0) xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); - arch_xtensa_mmu_post_init(is_core0); -} - -void z_xtensa_mmu_init(void) -{ - xtensa_mmu_init(true); -} - -void z_xtensa_mmu_smp_init(void) -{ - xtensa_mmu_init(false); + arch_xtensa_mmu_post_init(_current_cpu->id == 0); } #ifdef CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index 6427b306e62319..2256f72f64544d 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -25,7 +25,7 @@ extern void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf); K_KERNEL_STACK_ARRAY_DECLARE(z_interrupt_stacks, CONFIG_MP_MAX_NUM_CPUS, CONFIG_ISR_STACK_SIZE); -static ALWAYS_INLINE void z_xtensa_kernel_init(void) +static ALWAYS_INLINE void arch_kernel_init(void) { _cpu_t *cpu0 = &_kernel.cpus[0]; @@ -51,21 +51,15 @@ static ALWAYS_INLINE void z_xtensa_kernel_init(void) * win. */ XTENSA_WSR(ZSR_CPU_STR, cpu0); -} - -static ALWAYS_INLINE void arch_kernel_init(void) -{ -#ifndef CONFIG_XTENSA_MMU - /* This is called in z_xtensa_mmu_init() before z_cstart() - * so we do not need to call it again. - */ - z_xtensa_kernel_init(); -#endif #ifdef CONFIG_INIT_STACKS memset(Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[0]), 0xAA, K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[0])); #endif + +#ifdef CONFIG_XTENSA_MMU + z_xtensa_mmu_init(); +#endif } void xtensa_switch(void *switch_to, void **switched_from); diff --git a/include/zephyr/arch/xtensa/xtensa_mmu.h b/include/zephyr/arch/xtensa/xtensa_mmu.h index d03f876af30b11..2ee3dc6c9ab8ee 100644 --- a/include/zephyr/arch/xtensa/xtensa_mmu.h +++ b/include/zephyr/arch/xtensa/xtensa_mmu.h @@ -26,6 +26,4 @@ extern int xtensa_soc_mmu_ranges_num; void z_xtensa_mmu_init(void); -void z_xtensa_mmu_smp_init(void); - #endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_XTENSA_MMU_H */ From a3628b803308a316dcc709667d5cf1e0621961fa Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Wed, 14 Dec 2022 00:35:36 -0800 Subject: [PATCH 06/33] xtensa: Enable userspace Userspace support for Xtensa architecture using Xtensa MMU. Some considerations: - Syscalls are not inline functions like in other architectures because some compiler issues when using multiple registers to pass parameters to the syscall. So here we have a function call so we can use registers as we need. - TLS is not supported by xcc in xtensa and reading PS register is a privileged instruction. So, we have to use threadptr to know if a thread is an user mode thread. Signed-off-by: Flavio Ceolin Signed-off-by: Daniel Leung --- arch/Kconfig | 1 + arch/xtensa/Kconfig | 20 + arch/xtensa/core/CMakeLists.txt | 2 + arch/xtensa/core/fatal.c | 9 + arch/xtensa/core/include/xtensa_mmu_priv.h | 83 ++- arch/xtensa/core/offsets/offsets.c | 7 + arch/xtensa/core/syscall_helper.c | 124 ++++ arch/xtensa/core/userspace.S | 302 ++++++++ arch/xtensa/core/xtensa-asm2-util.S | 21 +- arch/xtensa/core/xtensa-asm2.c | 44 +- arch/xtensa/core/xtensa_mmu.c | 772 +++++++++++++++++++-- arch/xtensa/include/kernel_arch_func.h | 7 + arch/xtensa/include/offsets_short_arch.h | 16 +- arch/xtensa/include/xtensa-asm2-s.h | 13 +- include/zephyr/arch/syscall.h | 2 + include/zephyr/arch/xtensa/arch.h | 10 + include/zephyr/arch/xtensa/syscall.h | 238 +++++++ include/zephyr/arch/xtensa/thread.h | 9 + include/zephyr/arch/xtensa/xtensa_mmu.h | 34 + 19 files changed, 1646 insertions(+), 68 deletions(-) create mode 100644 arch/xtensa/core/syscall_helper.c create mode 100644 arch/xtensa/core/userspace.S create mode 100644 include/zephyr/arch/xtensa/syscall.h diff --git a/arch/Kconfig b/arch/Kconfig index 5e3b96f414d497..cc9925e77747c9 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -125,6 +125,7 @@ config XTENSA select IRQ_OFFLOAD_NESTED if IRQ_OFFLOAD select ARCH_HAS_CODE_DATA_RELOCATION select ARCH_HAS_TIMING_FUNCTIONS + select ARCH_MEM_DOMAIN_DATA if USERSPACE imply ATOMIC_OPERATIONS_ARCH help Xtensa architecture diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index a1517f17ed0976..c875aa3097230b 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -113,6 +113,7 @@ config XTENSA_MMU bool "Xtensa MMU Support" default n select MMU + select ARCH_MEM_DOMAIN_SYNCHRONOUS_API if USERSPACE select XTENSA_SMALL_VECTOR_TABLE_ENTRY select KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK if XTENSA_RPO_CACHE help @@ -144,8 +145,18 @@ if XTENSA_MMU The bit shift number for the virtual address for Xtensa page table (PTEVADDR). + config XTENSA_MMU_NUM_L1_TABLES + int "Number of L1 page tables" + default 1 if !USERSPACE + default 4 + help + This option specifies the maximum number of traslation tables. + Translation tables are directly related to the number of + memory domains in the target, considering the kernel itself requires one. + config XTENSA_MMU_NUM_L2_TABLES int "Number of L2 page tables" + default 20 if USERSPACE default 10 help Each table can address up to 4MB memory address. @@ -159,6 +170,15 @@ if XTENSA_MMU endif # XTENSA_MMU +config XTENSA_SYSCALL_USE_HELPER + bool "Use userspace syscall helper" + default y if "$(ZEPHYR_TOOLCHAIN_VARIANT)" = "xcc-clang" + depends on USERSPACE + help + Use syscall helpers for passing more then 3 arguments. + This is a workaround for toolchains where they have + issue modeling register usage. + endif # CPU_HAS_MMU endmenu diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index c6fffd4f5e181d..8a23b65b9a9b16 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -22,6 +22,8 @@ zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c) zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c) zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c) zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU xtensa_mmu.c) +zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S) +zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c) zephyr_library_sources_ifdef( CONFIG_KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index e693937f99bfd5..3117ebc4a56d70 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -15,6 +15,7 @@ #endif #endif #include +#include #include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); @@ -120,6 +121,14 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf) z_fatal_error(reason, esf); } +#ifdef CONFIG_USERSPACE +Z_EXC_DECLARE(z_xtensa_user_string_nlen); + +static const struct z_exc_handle exceptions[] = { + Z_EXC_HANDLE(z_xtensa_user_string_nlen) +}; +#endif /* CONFIG_USERSPACE */ + #ifdef XT_SIMULATOR void exit(int return_code) { diff --git a/arch/xtensa/core/include/xtensa_mmu_priv.h b/arch/xtensa/core/include/xtensa_mmu_priv.h index dcfb9deefc0d54..cf72c92138373c 100644 --- a/arch/xtensa/core/include/xtensa_mmu_priv.h +++ b/arch/xtensa/core/include/xtensa_mmu_priv.h @@ -18,18 +18,37 @@ #define Z_XTENSA_PTE_VPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_PPN_MASK 0xFFFFF000U #define Z_XTENSA_PTE_ATTR_MASK 0x0000000FU +#define Z_XTENSA_PTE_ATTR_CACHED_MASK 0x0000000CU #define Z_XTENSA_L1_MASK 0x3FF00000U #define Z_XTENSA_L2_MASK 0x3FFFFFU #define Z_XTENSA_PPN_SHIFT 12U #define Z_XTENSA_PTE_RING_MASK 0x00000030U +#define Z_XTENSA_PTE_RING_SHIFT 4U #define Z_XTENSA_PTE(paddr, ring, attr) \ (((paddr) & Z_XTENSA_PTE_PPN_MASK) | \ - (((ring) << 4) & Z_XTENSA_PTE_RING_MASK) | \ + (((ring) << Z_XTENSA_PTE_RING_SHIFT) & Z_XTENSA_PTE_RING_MASK) | \ ((attr) & Z_XTENSA_PTE_ATTR_MASK)) +#define Z_XTENSA_PTE_ATTR_GET(pte) \ + (pte) & Z_XTENSA_PTE_ATTR_MASK + +#define Z_XTENSA_PTE_ATTR_SET(pte, attr) \ + (((pte) & ~Z_XTENSA_PTE_ATTR_MASK) | (attr)) + +#define Z_XTENSA_PTE_RING_SET(pte, ring) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) | \ + ((ring) << Z_XTENSA_PTE_RING_SHIFT)) + +#define Z_XTENSA_PTE_RING_GET(pte) \ + (((pte) & ~Z_XTENSA_PTE_RING_MASK) >> Z_XTENSA_PTE_RING_SHIFT) + +#define Z_XTENSA_PTE_ASID_GET(pte, rasid) \ + (((rasid) >> ((((pte) & Z_XTENSA_PTE_RING_MASK) \ + >> Z_XTENSA_PTE_RING_SHIFT) * 8)) & 0xFF) + #define Z_XTENSA_TLB_ENTRY(vaddr, way) \ (((vaddr) & Z_XTENSA_PTE_PPN_MASK) | (way)) @@ -38,11 +57,38 @@ (((vaddr) >> Z_XTENSA_PPN_SHIFT) & 0x03U)) #define Z_XTENSA_L2_POS(vaddr) \ - (((vaddr) & Z_XTENSA_L2_MASK) >> Z_XTENSA_PPN_SHIFT) + (((vaddr) & Z_XTENSA_L2_MASK) >> 12U) + +#define Z_XTENSA_L1_POS(vaddr) \ + ((vaddr) >> 22U) + +/* PTE attributes for entries in the L1 page table. Should never be + * writable, may be cached in non-SMP contexts only + */ +#if CONFIG_MP_MAX_NUM_CPUS == 1 +#define Z_XTENSA_PAGE_TABLE_ATTR Z_XTENSA_MMU_CACHED_WB +#else +#define Z_XTENSA_PAGE_TABLE_ATTR 0 +#endif + +/* This ASID is shared between all domains and kernel. */ +#define Z_XTENSA_MMU_SHARED_ASID 255 + +/* Fixed data TLB way to map the page table */ +#define Z_XTENSA_MMU_PTE_WAY 7 + +/* Fixed data TLB way to map the vecbase */ +#define Z_XTENSA_MMU_VECBASE_WAY 8 /* Kernel specific ASID. Ring field in the PTE */ #define Z_XTENSA_KERNEL_RING 0 +/* User specific ASID. Ring field in the PTE */ +#define Z_XTENSA_USER_RING 2 + +/* Ring value for MMU_SHARED_ASID */ +#define Z_XTENSA_SHARED_RING 3 + /* Number of data TLB ways [0-9] */ #define Z_XTENSA_DTLB_WAYS 10 @@ -96,6 +142,14 @@ #define Z_XTENSA_PAGE_TABLE_VADDR \ Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR) +/* + * Get asid for a given ring from rasid register. + * rasid contains four asid, one per ring. + */ + +#define Z_XTENSA_RASID_ASID_GET(rasid, ring) \ + (((rasid) >> ((ring) * 8)) & 0xff) + static ALWAYS_INLINE void xtensa_rasid_set(uint32_t rasid) { __asm__ volatile("wsr %0, rasid\n\t" @@ -110,6 +164,16 @@ static ALWAYS_INLINE uint32_t xtensa_rasid_get(void) return rasid; } +static ALWAYS_INLINE void xtensa_rasid_asid_set(uint8_t asid, uint8_t pos) +{ + uint32_t rasid = xtensa_rasid_get(); + + rasid = (rasid & ~(0xff << (pos * 8))) | ((uint32_t)asid << (pos * 8)); + + xtensa_rasid_set(rasid); +} + + static ALWAYS_INLINE void xtensa_itlb_entry_invalidate(uint32_t entry) { __asm__ volatile("iitlb %0\n\t" @@ -201,6 +265,21 @@ static ALWAYS_INLINE void xtensa_ptevaddr_set(void *ptables) __asm__ volatile("wsr.ptevaddr %0" : : "a"((uint32_t)ptables)); } +/** + * @brief Get the current page tables. + * + * The page tables is obtained by reading ptevaddr address. + * + * @return ptables The page tables address (virtual address) + */ +static ALWAYS_INLINE void *xtensa_ptevaddr_get(void) +{ + uint32_t ptables; + + __asm__ volatile("rsr.ptevaddr %0" : "=a" (ptables)); + + return (void *)ptables; +} /* * The following functions are helpful when debugging. */ diff --git a/arch/xtensa/core/offsets/offsets.c b/arch/xtensa/core/offsets/offsets.c index 860a12eb408909..8d4022fd81ef05 100644 --- a/arch/xtensa/core/offsets/offsets.c +++ b/arch/xtensa/core/offsets/offsets.c @@ -5,6 +5,7 @@ #include #include +#include #include @@ -60,4 +61,10 @@ GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu14); GEN_OFFSET_SYM(_xtensa_irq_bsa_t, fpu15); #endif +#ifdef CONFIG_USERSPACE +GEN_OFFSET_SYM(_thread_arch_t, psp); +GEN_OFFSET_SYM(_thread_arch_t, ptables); +#endif + + GEN_ABS_SYM_END diff --git a/arch/xtensa/core/syscall_helper.c b/arch/xtensa/core/syscall_helper.c new file mode 100644 index 00000000000000..fbbdf1041910d5 --- /dev/null +++ b/arch/xtensa/core/syscall_helper.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6) + : "memory"); + + return a2; +} + +uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id) +{ + register uintptr_t a2 __asm__("%a2") = call_id; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2) + : "memory"); + + return a2; +} diff --git a/arch/xtensa/core/userspace.S b/arch/xtensa/core/userspace.S new file mode 100644 index 00000000000000..b649014fd52bec --- /dev/null +++ b/arch/xtensa/core/userspace.S @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2022, Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +/** + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ +.pushsection .text.z_xtensa_do_syscall, "ax" +.global z_xtensa_do_syscall +.align 4 +z_xtensa_do_syscall: + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + s32i a1, a0, ___xtensa_irq_bsa_t_scratch_OFFSET + s32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + s32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + rsr a2, ZSR_A0SAVE + s32i a2, a0, ___xtensa_irq_bsa_t_a0_OFFSET + rsr.ps a2 + movi a3, ~PS_OWB_MASK + and a2, a2, a3 + s32i a2, a0, ___xtensa_irq_bsa_t_ps_OFFSET + rsr.epc1 a2 + s32i a2, a0, ___xtensa_irq_bsa_t_pc_OFFSET + + movi a2, PS_WOE|PS_INTLEVEL(XCHAL_NMILEVEL) + rsr.ps a3 + or a3, a3, a2 + movi a2, ~(PS_EXCM | PS_RING_MASK) + and a3, a3, a2 + wsr.ps a3 + rsync + l32i a2, a0, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a0, ___xtensa_irq_bsa_t_a3_OFFSET + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + l32i a0, a0, _thread_offset_to_psp + addi a0, a0, -___xtensa_irq_bsa_t_SIZEOF + + mov a1, a0 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET +#if XCHAL_HAVE_LOOPS + /* If the syscall instruction was the last instruction in the body of + * a zero-overhead loop, and the loop will execute again, decrement + * the loop count and resume execution at the head of the loop. + */ + rsr.lend a2 + addi a3, a3, 3 + bne a2, a3, end_loop + rsr.lcount a2 + beqz a2, end_loop + addi a2, a2, -1 + wsr.lcount a2 + rsr.lbeg a3 +end_loop: +#else + /* EPC1 (and now a3) contains the address that invoked syscall. + * We need to increment it to execute the next instruction when + * we return. The instruction size is 3 bytes, so lets just add it. + */ + addi a3, a3, 3 +#endif + s32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + ODD_REG_SAVE + + call0 xtensa_save_high_regs + + l32i a2, a1, 0 + l32i a2, a2, ___xtensa_irq_bsa_t_a2_OFFSET + movi a0, K_SYSCALL_LIMIT + bgeu a2, a0, _bad_syscall + +_id_ok: + /* Find the function handler for the given syscall id. */ + movi a3, _k_syscall_table + slli a2, a2, 2 + add a2, a2, a3 + l32i a2, a2, 0 + + /* Clear up the threadptr because it is used + * to check if a thread is running on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possibly running in user mode. + */ + movi a0, 0 + wur.THREADPTR a0 + + /* Set syscall parameters. We have an initial call4 to set up the + * the stack and then a new call4 for the syscall function itself. + * So parameters should be put as if it was a call8. + */ + mov a10, a8 + mov a11, a9 + mov a8, a4 + mov a9, a5 + l32i a3, a1, 0 + l32i a7, a3, ___xtensa_irq_bsa_t_a3_OFFSET + + + /* Since we are unmasking EXCM, we need to set RING bits to kernel + * mode, otherwise we won't be able to run the exception handler in C. + */ + movi a0, PS_WOE|PS_CALLINC(0)|PS_UM|PS_INTLEVEL(0) + wsr.ps a0 + rsync + + call4 _syscall_call0 + + /* copy return value. Lets put it in the top of stack + * because registers will be clobbered in + * xtensa_restore_high_regs + */ + l32i a3, a1, 0 + s32i a6, a3, ___xtensa_irq_bsa_t_a2_OFFSET + + j _syscall_returned + +.align 4 +_syscall_call0: + /* We want an ENTRY to set a bit in windowstart */ + jx a2 + + +_syscall_returned: + call0 xtensa_restore_high_regs + + l32i a3, a1, ___xtensa_irq_bsa_t_sar_OFFSET + wsr a3, SAR +#if XCHAL_HAVE_LOOPS + l32i a3, a1, ___xtensa_irq_bsa_t_lbeg_OFFSET + wsr a3, LBEG + l32i a3, a1, ___xtensa_irq_bsa_t_lend_OFFSET + wsr a3, LEND + l32i a3, a1, ___xtensa_irq_bsa_t_lcount_OFFSET + wsr a3, LCOUNT +#endif +#if XCHAL_HAVE_S32C1I + l32i a3, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET + wsr a3, SCOMPARE1 +#endif + + rsr a3, ZSR_CPU + l32i a3, a3, ___cpu_t_current_OFFSET + wur.THREADPTR a3 + + l32i a3, a1, ___xtensa_irq_bsa_t_ps_OFFSET + wsr.ps a3 + + l32i a3, a1, ___xtensa_irq_bsa_t_pc_OFFSET + wsr.epc1 a3 + + l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET + l32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET + l32i a3, a1, ___xtensa_irq_bsa_t_a3_OFFSET + + l32i a1, a1, ___xtensa_irq_bsa_t_scratch_OFFSET + rsync + + rfe + +_bad_syscall: + movi a2, K_SYSCALL_BAD + j _id_ok + +.popsection + +/* FUNC_NORETURN void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + * void *p1, void *p2, void *p3, + * uint32_t stack_end, + * uint32_t stack_start) + * + * A one-way trip to userspace. + */ +.global z_xtensa_userspace_enter +.type z_xtensa_userspace_enter, @function +.align 4 +z_xtensa_userspace_enter: + /* Call entry to set a bit in the windowstart and + * do the rotation, but we are going to set our own + * stack. + */ + entry a1, 16 + + /* We have to switch to kernel stack before spill kernel data and + * erase user stack to avoid leak from previous context. + */ + mov a1, a7 /* stack start (low address) */ + addi a1, a1, -16 + + SPILL_ALL_WINDOWS + + rsr a0, ZSR_CPU + l32i a0, a0, ___cpu_t_current_OFFSET + + addi a1, a1, -28 + s32i a0, a1, 24 + s32i a2, a1, 20 + s32i a3, a1, 16 + s32i a4, a1, 12 + s32i a5, a1, 8 + s32i a6, a1, 4 + s32i a7, a1, 0 + + l32i a6, a1, 24 + call4 xtensa_user_stack_perms + + l32i a6, a1, 24 + call4 z_xtensa_swap_update_page_tables + + /* Set threadptr with the thread address, we are going to user mode. */ + l32i a0, a1, 24 + wur.THREADPTR a0 + + /* Set now z_thread_entry parameters, we are simulating a call4 + * call, so parameters start at a6, a7, ... + */ + l32i a6, a1, 20 + l32i a7, a1, 16 + l32i a8, a1, 12 + l32i a9, a1, 8 + + /* stash user stack */ + l32i a0, a1, 4 + + addi a1, a1, 28 + + /* Go back to user stack */ + mov a1, a0 + + movi a0, z_thread_entry + wsr.epc2 a0 + + /* Configuring PS register. + * We have to set callinc as well, since the called + * function will do "entry" + */ + movi a0, PS_WOE|PS_CALLINC(1)|PS_UM|PS_RING(2) + wsr a0, EPS2 + + movi a0, 0 + + rfi 2 + +/* + * size_t arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg) + */ +.global arch_user_string_nlen +.type arch_user_string_nlen, @function +.align 4 +arch_user_string_nlen: + entry a1, 32 + + /* error value, set to -1. */ + movi a5, -1 + s32i a5, a4, 0 + + /* length count */ + xor a5, a5, a5 + + /* This code might page fault */ +strlen_loop: +.global z_xtensa_user_string_nlen_fault_start +z_xtensa_user_string_nlen_fault_start: + l8ui a6, a2, 0 /* Current char */ + +.global z_xtensa_user_string_nlen_fault_end +z_xtensa_user_string_nlen_fault_end: + beqz a6, strlen_done + addi a5, a5, 1 + addi a2, a2, 1 + beq a5, a3, strlen_done + j strlen_loop + +strlen_done: + /* Set return value */ + mov a2, a5 + + /* Set error value to 0 since we succeeded */ + movi a5, 0x0 + s32i a5, a4, 0 + +.global z_xtensa_user_string_nlen_fixup +z_xtensa_user_string_nlen_fixup: + retw diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index bc526ae0165e24..c108a09dee4814 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -174,7 +174,8 @@ _restore_context: l32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET wsr a0, SCOMPARE1 #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) l32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET wur a0, THREADPTR #endif @@ -258,6 +259,16 @@ noflush: l32i a3, a2, ___xtensa_irq_bsa_t_a3_OFFSET s32i a1, a3, 0 +#ifdef CONFIG_USERSPACE + /* Switch page tables */ + rsr a6, ZSR_CPU + l32i a6, a6, ___cpu_t_current_OFFSET + call4 z_xtensa_swap_update_page_tables + + l32i a2, a3, 0 + l32i a2, a2, 0 +#endif + /* Switch stack pointer and restore. The jump to * _restore_context does not return as such, but we arrange * for the restored "next" address to be immediately after for @@ -347,6 +358,9 @@ _Level1RealVector: rsr.exccause a0 #ifdef CONFIG_XTENSA_MMU beqi a0, EXCCAUSE_ITLB_MISS, _handle_tlb_miss_user +#ifdef CONFIG_USERSPACE + beqi a0, EXCCAUSE_SYSCALL, _syscall +#endif /* CONFIG_USERSPACE */ addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_user rsr.exccause a0 @@ -376,6 +390,11 @@ _handle_tlb_miss_user: l32i a0, a0, 0 rsr a0, ZSR_A0SAVE rfe +#ifdef CONFIG_USERSPACE +_syscall: + rsr a0, ZSR_A0SAVE + j z_xtensa_do_syscall +#endif /* CONFIG_USERSPACE */ #endif /* CONFIG_XTENSA_MMU */ .popsection diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index c2371ac8f55d6a..1e8c92759e42fb 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -26,6 +26,13 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, { void *ret; _xtensa_irq_stack_frame_a11_t *frame; +#ifdef CONFIG_USERSPACE + struct z_xtensa_thread_stack_header *header = + (struct z_xtensa_thread_stack_header *)thread->stack_obj; + + thread->arch.psp = header->privilege_stack + + sizeof(header->privilege_stack); +#endif /* Not-a-cpu ID Ensures that the first time this is run, the * stack will be invalidated. That covers the edge case of @@ -48,11 +55,23 @@ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, (void)memset(frame, 0, bsasz); - frame->bsa.pc = (uintptr_t)z_thread_entry; frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1); +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) == K_USER) { + frame->bsa.pc = (uintptr_t)arch_user_mode_enter; + } else { + frame->bsa.pc = (uintptr_t)z_thread_entry; + } +#else + frame->bsa.pc = (uintptr_t)z_thread_entry; +#endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR +#ifdef CONFIG_THREAD_LOCAL_STORAGE frame->bsa.threadptr = thread->tls; +#elif CONFIG_USERSPACE + frame->bsa.threadptr = (uintptr_t)((thread->base.user_options & K_USER) ? thread : NULL); +#endif #endif /* Arguments to z_thread_entry(). Remember these start at A6, @@ -471,3 +490,24 @@ void arch_spin_relax(void) #undef NOP1 } #endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */ + +#ifdef CONFIG_USERSPACE +FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3) +{ + struct k_thread *current = _current; + size_t stack_end; + + /* Transition will reset stack pointer to initial, discarding + * any old context since this is a one-way operation + */ + stack_end = Z_STACK_PTR_ALIGN(current->stack_info.start + + current->stack_info.size - + current->stack_info.delta); + + z_xtensa_userspace_enter(user_entry, p1, p2, p3, + stack_end, current->stack_info.start); + + CODE_UNREACHABLE; +} +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 18ce5f22f63466..915a26357eb894 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -4,6 +4,7 @@ */ #include #include +#include #include #include #include @@ -15,22 +16,24 @@ #include #include -/* Fixed data TLB way to map the page table */ -#define MMU_PTE_WAY 7 - -/* Fixed data TLB way to map VECBASE */ -#define MMU_VECBASE_WAY 8 - /* Level 1 contains page table entries * necessary to map the page table itself. */ #define XTENSA_L1_PAGE_TABLE_ENTRIES 1024U +/* Size of level 1 page table. + */ +#define XTENSA_L1_PAGE_TABLE_SIZE (XTENSA_L1_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + /* Level 2 contains page table entries * necessary to map the page table itself. */ #define XTENSA_L2_PAGE_TABLE_ENTRIES 1024U +/* Size of level 2 page table. + */ +#define XTENSA_L2_PAGE_TABLE_SIZE (XTENSA_L2_PAGE_TABLE_ENTRIES * sizeof(uint32_t)) + LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, @@ -40,8 +43,18 @@ BUILD_ASSERT(CONFIG_MMU_PAGE_SIZE == 0x1000, * Level 1 page table has to be 4Kb to fit into one of the wired entries. * All entries are initialized as INVALID, so an attempt to read an unmapped * area will cause a double exception. + * + * Each memory domain contains its own l1 page table. The kernel l1 page table is + * located at the index 0. */ -uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); +static uint32_t l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES][XTENSA_L1_PAGE_TABLE_ENTRIES] + __aligned(KB(4)); + + +/* + * That is an alias for the page tables set used by the kernel. + */ +uint32_t *z_xtensa_kernel_ptables = (uint32_t *)l1_page_table[0]; /* * Each table in the level 2 maps a 4Mb memory range. It consists of 1024 entries each one @@ -50,12 +63,41 @@ uint32_t l1_page_table[XTENSA_L1_PAGE_TABLE_ENTRIES] __aligned(KB(4)); static uint32_t l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES][XTENSA_L2_PAGE_TABLE_ENTRIES] __aligned(KB(4)); +/* + * This additional variable tracks which l1 tables are in use. This is kept separated from + * the tables to keep alignment easier. + * + * @note: The first bit is set because it is used for the kernel page tables. + */ +static ATOMIC_DEFINE(l1_page_table_track, CONFIG_XTENSA_MMU_NUM_L1_TABLES); + /* * This additional variable tracks which l2 tables are in use. This is kept separated from * the tables to keep alignment easier. */ static ATOMIC_DEFINE(l2_page_tables_track, CONFIG_XTENSA_MMU_NUM_L2_TABLES); +/* + * Protects xtensa_domain_list and serializes access to page tables. + */ +static struct k_spinlock xtensa_mmu_lock; + +#ifdef CONFIG_USERSPACE + +/* + * Each domain has its own ASID. ASID can go through 1 (kernel) to 255. + * When a TLB entry matches, the hw will check the ASID in the entry and finds + * the correspondent position in the RASID register. This position will then be + * compared with the current ring (CRING) to check the permission. + */ +static uint8_t asid_count = 3; + +/* + * List with all active and initialized memory domains. + */ +static sys_slist_t xtensa_domain_list; +#endif /* CONFIG_USERSPACE */ + extern char _heap_end[]; extern char _heap_start[]; extern char __data_start[]; @@ -100,18 +142,61 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { { .start = (uint32_t)__text_region_start, .end = (uint32_t)__text_region_end, - .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "text", }, /* Mark rodata segment cacheable, read only and non-executable */ { .start = (uint32_t)__rodata_region_start, .end = (uint32_t)__rodata_region_end, - .attrs = Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "rodata", }, }; +static inline uint32_t *thread_page_tables_get(const struct k_thread *thread) +{ +#ifdef CONFIG_USERSPACE + if ((thread->base.user_options & K_USER) != 0U) { + return thread->arch.ptables; + } +#endif + + return z_xtensa_kernel_ptables; +} + +/** + * @brief Check if the page table entry is illegal. + * + * @param[in] Page table entry. + */ +static inline bool is_pte_illegal(uint32_t pte) +{ + uint32_t attr = pte & Z_XTENSA_PTE_ATTR_MASK; + + /* + * The ISA manual states only 12 and 14 are illegal values. + * 13 and 15 are not. So we need to be specific than simply + * testing if bits 2 and 3 are set. + */ + return (attr == 12) || (attr == 14); +} + +/* + * @brief Initialize all page table entries to be illegal. + * + * @param[in] Pointer to page table. + * @param[in] Number of page table entries in the page table. + */ +static void init_page_table(uint32_t *ptable, size_t num_entries) +{ + int i; + + for (i = 0; i < num_entries; i++) { + ptable[i] = Z_XTENSA_MMU_ILLEGAL; + } +} + static inline uint32_t *alloc_l2_table(void) { uint16_t idx; @@ -125,45 +210,86 @@ static inline uint32_t *alloc_l2_table(void) return NULL; } +/** + * @brief Switch page tables + * + * This switches the page tables to the incoming ones (@a ptables). + * Since data TLBs to L2 page tables are auto-filled, @a dtlb_inv + * can be used to invalidate these data TLBs. @a cache_inv can be + * set to true to invalidate cache to the page tables. + * + * @param[in] ptables Page tables to be switched to. + * @param[in] dtlb_inv True if to invalidate auto-fill data TLBs. + * @param[in] cache_inv True if to invalidate cache to page tables. + */ +static ALWAYS_INLINE void switch_page_tables(uint32_t *ptables, bool dtlb_inv, bool cache_inv) +{ + if (cache_inv) { + sys_cache_data_invd_range((void *)ptables, XTENSA_L1_PAGE_TABLE_SIZE); + sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + } + + /* Invalidate data TLB to L1 page table */ + xtensa_dtlb_vaddr_invalidate((void *)Z_XTENSA_PAGE_TABLE_VADDR); + + /* Now map the pagetable itself with KERNEL asid to avoid user thread + * from tampering with it. + */ + xtensa_dtlb_entry_write_sync( + Z_XTENSA_PTE((uint32_t)ptables, Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), + Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); + + if (dtlb_inv) { + /* Since L2 page tables are auto-refilled, + * invalidate all of them to flush the old entries out. + */ + xtensa_tlb_autorefill_invalidate(); + } +} + static void map_memory_range(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, bool shared) { uint32_t page, *table; for (page = start; page < end; page += CONFIG_MMU_PAGE_SIZE) { - uint32_t pte = Z_XTENSA_PTE(page, Z_XTENSA_KERNEL_RING, attrs); + uint32_t pte = Z_XTENSA_PTE(page, + shared ? Z_XTENSA_SHARED_RING : Z_XTENSA_KERNEL_RING, + attrs); uint32_t l2_pos = Z_XTENSA_L2_POS(page); - uint32_t l1_pos = page >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { + if (is_pte_illegal(z_xtensa_kernel_ptables[l1_pos])) { table = alloc_l2_table(); __ASSERT(table != NULL, "There is no l2 page table available to " "map 0x%08x\n", page); - l1_page_table[l1_pos] = + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + z_xtensa_kernel_ptables[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); + Z_XTENSA_PAGE_TABLE_ATTR); } - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table = (uint32_t *)(z_xtensa_kernel_ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); table[l2_pos] = pte; } } static void map_memory(const uint32_t start, const uint32_t end, - const uint32_t attrs) + const uint32_t attrs, bool shared) { - map_memory_range(start, end, attrs); + map_memory_range(start, end, attrs, shared); #ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP if (arch_xtensa_is_ptr_uncached((void *)start)) { map_memory_range(POINTER_TO_UINT(z_soc_cached_ptr((void *)start)), POINTER_TO_UINT(z_soc_cached_ptr((void *)end)), - attrs | Z_XTENSA_MMU_CACHED_WB); + attrs | Z_XTENSA_MMU_CACHED_WB, shared); } else if (arch_xtensa_is_ptr_cached((void *)start)) { map_memory_range(POINTER_TO_UINT(z_soc_uncached_ptr((void *)start)), - POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs); + POINTER_TO_UINT(z_soc_uncached_ptr((void *)end)), attrs, shared); } #endif } @@ -171,16 +297,19 @@ static void map_memory(const uint32_t start, const uint32_t end, static void xtensa_init_page_tables(void) { volatile uint8_t entry; - uint32_t page; - for (page = 0; page < XTENSA_L1_PAGE_TABLE_ENTRIES; page++) { - l1_page_table[page] = Z_XTENSA_MMU_ILLEGAL; - } + init_page_table(z_xtensa_kernel_ptables, XTENSA_L1_PAGE_TABLE_ENTRIES); + atomic_set_bit(l1_page_table_track, 0); for (entry = 0; entry < ARRAY_SIZE(mmu_zephyr_ranges); entry++) { const struct xtensa_mmu_range *range = &mmu_zephyr_ranges[entry]; + bool shared; + uint32_t attrs; - map_memory(range->start, range->end, range->attrs); + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; + + map_memory(range->start, range->end, attrs, shared); } /** @@ -198,8 +327,13 @@ static void xtensa_init_page_tables(void) #endif for (entry = 0; entry < xtensa_soc_mmu_ranges_num; entry++) { const struct xtensa_mmu_range *range = &xtensa_soc_mmu_ranges[entry]; + bool shared; + uint32_t attrs; + + shared = !!(range->attrs & Z_XTENSA_MMU_MAP_SHARED); + attrs = range->attrs & ~Z_XTENSA_MMU_MAP_SHARED; - map_memory(range->start, range->end, range->attrs); + map_memory(range->start, range->end, attrs, shared); } #if defined(__GNUC__) #pragma GCC diagnostic pop @@ -231,6 +365,9 @@ void z_xtensa_mmu_init(void) /* Set the page table location in the virtual address */ xtensa_ptevaddr_set((void *)Z_XTENSA_PTEVADDR); + /* Set rasid */ + xtensa_rasid_asid_set(Z_XTENSA_MMU_SHARED_ASID, Z_XTENSA_SHARED_RING); + /* Next step is to invalidate the tlb entry that contains the top level * page table. This way we don't cause a multi hit exception. */ @@ -243,9 +380,9 @@ void z_xtensa_mmu_init(void) * Lets use one of the wired entry, so we never have tlb miss for * the top level table. */ - xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)l1_page_table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, MMU_PTE_WAY)); + xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)z_xtensa_kernel_ptables, + Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), + Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); /* Before invalidate the text region in the TLB entry 6, we need to * map the exception vector into one of the wired entries to avoid @@ -297,7 +434,7 @@ void z_xtensa_mmu_init(void) xtensa_dtlb_entry_write( Z_XTENSA_PTE((uint32_t)vecbase, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_CACHED_WB), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, MMU_VECBASE_WAY)); + Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, Z_XTENSA_MMU_VECBASE_WAY)); /* * Pre-load TLB for vecbase so exception handling won't result @@ -325,6 +462,12 @@ void z_xtensa_mmu_init(void) xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); + /* + * Clear out THREADPTR as we use it to indicate + * whether we are in user mode or not. + */ + XTENSA_WUR("THREADPTR", 0); + arch_xtensa_mmu_post_init(_current_cpu->id == 0); } @@ -351,32 +494,121 @@ __weak void arch_reserved_pages_update(void) } #endif /* CONFIG_ARCH_HAS_RESERVED_PAGE_FRAMES */ -static bool l2_page_table_map(void *vaddr, uintptr_t phys, uint32_t flags) +static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, + uint32_t flags, bool is_user) { uint32_t l1_pos = (uint32_t)vaddr >> 22; - uint32_t pte = Z_XTENSA_PTE(phys, Z_XTENSA_KERNEL_RING, flags); uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); uint32_t *table; - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { table = alloc_l2_table(); if (table == NULL) { return false; } - l1_page_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_CACHED_WT); + init_page_table(table, XTENSA_L2_PAGE_TABLE_ENTRIES); + + l1_table[l1_pos] = Z_XTENSA_PTE((uint32_t)table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + } + + table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + table[l2_pos] = Z_XTENSA_PTE(phys, is_user ? Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING, + flags); + + sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); + + return true; +} + +static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, bool is_user) +{ + bool ret; + void *vaddr, *vaddr_uc; + uintptr_t paddr, paddr_uc; + uint32_t flags, flags_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + + if (arch_xtensa_is_ptr_cached((void *)pa)) { + paddr = pa; + paddr_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)pa); + } else { + paddr = (uintptr_t)arch_xtensa_cached_ptr((void *)pa); + paddr_uc = pa; + } + + flags_uc = (xtensa_flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + flags = flags_uc | Z_XTENSA_MMU_CACHED_WB; + } else { + vaddr = va; + paddr = pa; + flags = xtensa_flags; + } + + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", va); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(z_xtensa_kernel_ptables, (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped", vaddr_uc); } - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = pte; +#ifndef CONFIG_USERSPACE + ARG_UNUSED(ret); +#else + if (ret) { + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + ret = l2_page_table_map(domain->ptables, (void *)vaddr, paddr, + flags, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr, domain); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP) && ret) { + ret = l2_page_table_map(domain->ptables, + (void *)vaddr_uc, paddr_uc, + flags_uc, is_user); + __ASSERT(ret, "Virtual address (%p) already mapped for domain %p", + vaddr_uc, domain); + } + } + k_spin_unlock(&z_mem_domain_lock, key); + } +#endif /* CONFIG_USERSPACE */ - if ((flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { + if ((xtensa_flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { xtensa_itlb_vaddr_invalidate(vaddr); } xtensa_dtlb_vaddr_invalidate(vaddr); - return true; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (xtensa_flags & Z_XTENSA_MMU_X) { + xtensa_itlb_vaddr_invalidate(vaddr_uc); + } + xtensa_dtlb_vaddr_invalidate(vaddr_uc); + } } void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) @@ -385,7 +617,8 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) uint32_t pa = (uint32_t)phys; uint32_t rem_size = (uint32_t)size; uint32_t xtensa_flags = 0; - int key; + k_spinlock_key_t key; + bool is_user; if (size == 0) { LOG_ERR("Cannot map physical memory at 0x%08X: invalid " @@ -414,63 +647,130 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) xtensa_flags |= Z_XTENSA_MMU_X; } - key = arch_irq_lock(); + is_user = (flags & K_MEM_PERM_USER) == K_MEM_PERM_USER; + + key = k_spin_lock(&xtensa_mmu_lock); while (rem_size > 0) { - bool ret = l2_page_table_map((void *)va, pa, xtensa_flags); + __arch_mem_map((void *)va, pa, xtensa_flags, is_user); - ARG_UNUSED(ret); - __ASSERT(ret, "Virtual address (%u) already mapped", (uint32_t)virt); rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); pa += KB(4); } - arch_irq_unlock(key); + k_spin_unlock(&xtensa_mmu_lock, key); } -static void l2_page_table_unmap(void *vaddr) +/** + * @return True if page is executable (thus need to invalidate ITLB), + * false if not. + */ +static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) { uint32_t l1_pos = (uint32_t)vaddr >> 22; uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); - uint32_t *table; + uint32_t *l2_table; uint32_t table_pos; bool exec; - if (l1_page_table[l1_pos] == Z_XTENSA_MMU_ILLEGAL) { - return; + sys_cache_data_invd_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + if (is_pte_illegal(l1_table[l1_pos])) { + /* We shouldn't be unmapping an illegal entry. + * Return true so that we can invalidate ITLB too. + */ + return true; } - exec = l1_page_table[l1_pos] & Z_XTENSA_MMU_X; + exec = l1_table[l1_pos] & Z_XTENSA_MMU_X; + + l2_table = (uint32_t *)(l1_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); - table = (uint32_t *)(l1_page_table[l1_pos] & Z_XTENSA_PTE_PPN_MASK); - table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; + l2_table[l2_pos] = Z_XTENSA_MMU_ILLEGAL; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); for (l2_pos = 0; l2_pos < XTENSA_L2_PAGE_TABLE_ENTRIES; l2_pos++) { - if (table[l2_pos] != Z_XTENSA_MMU_ILLEGAL) { + if (!is_pte_illegal(l2_table[l2_pos])) { goto end; } } - l1_page_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; - table_pos = (table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); + l1_table[l1_pos] = Z_XTENSA_MMU_ILLEGAL; + sys_cache_data_flush_range((void *)&l1_table[l1_pos], sizeof(l1_table[0])); + + table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); atomic_clear_bit(l2_page_tables_track, table_pos); /* Need to invalidate L2 page table as it is no longer valid. */ - xtensa_dtlb_vaddr_invalidate((void *)table); + xtensa_dtlb_vaddr_invalidate((void *)l2_table); end: - if (exec) { + return exec; +} + +static inline void __arch_mem_unmap(void *va) +{ + bool is_exec; + void *vaddr, *vaddr_uc; + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (arch_xtensa_is_ptr_cached(va)) { + vaddr = va; + vaddr_uc = arch_xtensa_uncached_ptr(va); + } else { + vaddr = arch_xtensa_cached_ptr(va); + vaddr_uc = va; + } + } else { + vaddr = va; + } + + is_exec = l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(z_xtensa_kernel_ptables, (void *)vaddr_uc); + } + +#ifdef CONFIG_USERSPACE + sys_snode_t *node; + struct arch_mem_domain *domain; + k_spinlock_key_t key; + + key = k_spin_lock(&z_mem_domain_lock); + SYS_SLIST_FOR_EACH_NODE(&xtensa_domain_list, node) { + domain = CONTAINER_OF(node, struct arch_mem_domain, node); + + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + (void)l2_page_table_unmap(domain->ptables, (void *)vaddr_uc); + } + } + k_spin_unlock(&z_mem_domain_lock, key); +#endif /* CONFIG_USERSPACE */ + + if (is_exec) { xtensa_itlb_vaddr_invalidate(vaddr); } xtensa_dtlb_vaddr_invalidate(vaddr); + + if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { + if (is_exec) { + xtensa_itlb_vaddr_invalidate(vaddr_uc); + } + xtensa_dtlb_vaddr_invalidate(vaddr_uc); + } } void arch_mem_unmap(void *addr, size_t size) { uint32_t va = (uint32_t)addr; uint32_t rem_size = (uint32_t)size; - int key; + k_spinlock_key_t key; if (addr == NULL) { LOG_ERR("Cannot unmap NULL pointer"); @@ -482,13 +782,363 @@ void arch_mem_unmap(void *addr, size_t size) return; } - key = arch_irq_lock(); + key = k_spin_lock(&xtensa_mmu_lock); while (rem_size > 0) { - l2_page_table_unmap((void *)va); + __arch_mem_unmap((void *)va); + rem_size -= (rem_size >= KB(4)) ? KB(4) : rem_size; va += KB(4); } - arch_irq_unlock(key); + k_spin_unlock(&xtensa_mmu_lock, key); } + +#ifdef CONFIG_USERSPACE + +static inline uint32_t *alloc_l1_table(void) +{ + uint16_t idx; + + for (idx = 0; idx < CONFIG_XTENSA_MMU_NUM_L1_TABLES; idx++) { + if (!atomic_test_and_set_bit(l1_page_table_track, idx)) { + return (uint32_t *)&l1_page_table[idx]; + } + } + + return NULL; +} + +static uint32_t *dup_table(uint32_t *source_table) +{ + uint16_t i, j; + uint32_t *dst_table = alloc_l1_table(); + + if (!dst_table) { + return NULL; + } + + for (i = 0; i < XTENSA_L1_PAGE_TABLE_ENTRIES; i++) { + uint32_t *l2_table, *src_l2_table; + + if (is_pte_illegal(source_table[i])) { + dst_table[i] = Z_XTENSA_MMU_ILLEGAL; + continue; + } + + src_l2_table = (uint32_t *)(source_table[i] & Z_XTENSA_PTE_PPN_MASK); + l2_table = alloc_l2_table(); + if (l2_table == NULL) { + goto err; + } + + for (j = 0; j < XTENSA_L2_PAGE_TABLE_ENTRIES; j++) { + l2_table[j] = src_l2_table[j]; + } + + /* The page table is using kernel ASID because we don't + * user thread manipulate it. + */ + dst_table[i] = Z_XTENSA_PTE((uint32_t)l2_table, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PAGE_TABLE_ATTR); + + sys_cache_data_flush_range((void *)l2_table, XTENSA_L2_PAGE_TABLE_SIZE); + } + + sys_cache_data_flush_range((void *)dst_table, XTENSA_L1_PAGE_TABLE_SIZE); + + return dst_table; + +err: + /* TODO: Cleanup failed allocation*/ + return NULL; +} + +int arch_mem_domain_init(struct k_mem_domain *domain) +{ + uint32_t *ptables; + k_spinlock_key_t key; + int ret; + + /* + * For now, lets just assert if we have reached the maximum number + * of asid we assert. + */ + __ASSERT(asid_count < (Z_XTENSA_MMU_SHARED_ASID), "Reached maximum of ASID available"); + + key = k_spin_lock(&xtensa_mmu_lock); + ptables = dup_table(z_xtensa_kernel_ptables); + + if (ptables == NULL) { + ret = -ENOMEM; + goto err; + } + + domain->arch.ptables = ptables; + domain->arch.asid = ++asid_count; + + sys_slist_append(&xtensa_domain_list, &domain->arch.node); + + ret = 0; + +err: + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static int region_map_update(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags) +{ + int ret = 0; + + for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { + uint32_t *l2_table, pte; + uint32_t page = start + offset; + uint32_t l1_pos = page >> 22; + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + + /* Make sure we grab a fresh copy of L1 page table */ + sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0])); + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + + sys_cache_data_invd_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + pte = Z_XTENSA_PTE_RING_SET(l2_table[l2_pos], ring); + pte = Z_XTENSA_PTE_ATTR_SET(pte, flags); + + l2_table[l2_pos] = pte; + + sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); + + xtensa_dtlb_vaddr_invalidate( + (void *)(pte & Z_XTENSA_PTE_PPN_MASK)); + } + + return ret; +} + +static inline int update_region(uint32_t *ptables, uintptr_t start, + size_t size, uint32_t ring, uint32_t flags) +{ + int ret; + k_spinlock_key_t key; + + key = k_spin_lock(&xtensa_mmu_lock); + +#ifdef CONFIG_XTENSA_MMU_DOUBLE_MAP + uintptr_t va, va_uc; + uint32_t new_flags, new_flags_uc; + + if (arch_xtensa_is_ptr_cached((void *)start)) { + va = start; + va_uc = (uintptr_t)arch_xtensa_uncached_ptr((void *)start); + } else { + va = (uintptr_t)arch_xtensa_cached_ptr((void *)start); + va_uc = start; + } + + new_flags_uc = (flags & ~Z_XTENSA_PTE_ATTR_CACHED_MASK); + new_flags = new_flags_uc | Z_XTENSA_MMU_CACHED_WB; + + ret = region_map_update(ptables, va, size, ring, new_flags); + + if (ret == 0) { + ret = region_map_update(ptables, va_uc, size, ring, new_flags_uc); + } +#else + ret = region_map_update(ptables, start, size, ring, flags); +#endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ + + k_spin_unlock(&xtensa_mmu_lock, key); + + return ret; +} + +static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size) +{ + return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W); +} + +void xtensa_set_stack_perms(struct k_thread *thread) +{ + if ((thread->base.user_options & K_USER) == 0) { + return; + } + + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); +} + +void xtensa_user_stack_perms(struct k_thread *thread) +{ + (void)memset((void *)thread->stack_info.start, 0xAA, + thread->stack_info.size - thread->stack_info.delta); + + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); +} + +int arch_mem_domain_max_partitions_get(void) +{ + return CONFIG_MAX_DOMAIN_PARTITIONS; +} + +int arch_mem_domain_partition_remove(struct k_mem_domain *domain, + uint32_t partition_id) +{ + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + /* Reset the partition's region back to defaults */ + return reset_region(domain->arch.ptables, partition->start, + partition->size); +} + +int arch_mem_domain_partition_add(struct k_mem_domain *domain, + uint32_t partition_id) +{ + uint32_t ring = domain->arch.asid == 0 ? Z_XTENSA_KERNEL_RING : Z_XTENSA_USER_RING; + struct k_mem_partition *partition = &domain->partitions[partition_id]; + + return update_region(domain->arch.ptables, partition->start, + partition->size, ring, partition->attr); +} + +/* These APIs don't need to do anything */ +int arch_mem_domain_thread_add(struct k_thread *thread) +{ + int ret = 0; + bool is_user, is_migration; + uint32_t *old_ptables; + struct k_mem_domain *domain; + + old_ptables = thread->arch.ptables; + domain = thread->mem_domain_info.mem_domain; + thread->arch.ptables = domain->arch.ptables; + + is_user = (thread->base.user_options & K_USER) != 0; + is_migration = (old_ptables != NULL) && is_user; + + /* Give access to the thread's stack in its new + * memory domain if it is migrating. + */ + if (is_migration) { + xtensa_set_stack_perms(thread); + } + + if (is_migration) { + ret = reset_region(old_ptables, + thread->stack_info.start, + thread->stack_info.size); + } + + return ret; +} + +int arch_mem_domain_thread_remove(struct k_thread *thread) +{ + struct k_mem_domain *domain = thread->mem_domain_info.mem_domain; + + if ((thread->base.user_options & K_USER) == 0) { + return 0; + } + + if ((thread->base.thread_state & _THREAD_DEAD) == 0) { + /* Thread is migrating to another memory domain and not + * exiting for good; we weren't called from + * z_thread_abort(). Resetting the stack region will + * take place in the forthcoming thread_add() call. + */ + return 0; + } + + /* Restore permissions on the thread's stack area since it is no + * longer a member of the domain. + */ + return reset_region(domain->arch.ptables, + thread->stack_info.start, + thread->stack_info.size); +} + +static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write) +{ + uint8_t asid_ring; + uint32_t rasid, pte, *l2_table; + uint32_t l1_pos = page >> 22; + uint32_t l2_pos = Z_XTENSA_L2_POS(page); + + if (is_pte_illegal(ptables[l1_pos])) { + return false; + } + + l2_table = (uint32_t *)(ptables[l1_pos] & Z_XTENSA_PTE_PPN_MASK); + pte = l2_table[l2_pos]; + + if (is_pte_illegal(pte)) { + return false; + } + + asid_ring = 0; + rasid = xtensa_rasid_get(); + for (uint32_t i = 0; i < 4; i++) { + if (Z_XTENSA_PTE_ASID_GET(pte, rasid) == + Z_XTENSA_RASID_ASID_GET(rasid, i)) { + asid_ring = i; + break; + } + } + + if (ring > asid_ring) { + return false; + } + + if (write) { + return (Z_XTENSA_PTE_ATTR_GET((pte)) & Z_XTENSA_MMU_W) != 0; + } + + return true; +} + +int arch_buffer_validate(void *addr, size_t size, int write) +{ + int ret = 0; + uint8_t *virt; + size_t aligned_size; + const struct k_thread *thread = _current; + uint32_t *ptables = thread_page_tables_get(thread); + uint8_t ring = ((thread->base.user_options & K_USER) != 0) ? + Z_XTENSA_USER_RING : Z_XTENSA_KERNEL_RING; + + /* addr/size arbitrary, fix this up into an aligned region */ + k_mem_region_align((uintptr_t *)&virt, &aligned_size, + (uintptr_t)addr, size, CONFIG_MMU_PAGE_SIZE); + + for (size_t offset = 0; offset < aligned_size; + offset += CONFIG_MMU_PAGE_SIZE) { + if (!page_validate(ptables, (uint32_t)(virt + offset), ring, write)) { + ret = -1; + break; + } + } + + return ret; +} + +void z_xtensa_swap_update_page_tables(struct k_thread *incoming) +{ + uint32_t *ptables = incoming->arch.ptables; + struct arch_mem_domain *domain = + &(incoming->mem_domain_info.mem_domain->arch); + + /* Lets set the asid for the incoming thread */ + if ((incoming->base.user_options & K_USER) != 0) { + xtensa_rasid_asid_set(domain->asid, Z_XTENSA_USER_RING); + } + + switch_page_tables(ptables, true, false); +} + +#endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index 2256f72f64544d..8ce5cc52a5b51a 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -174,6 +174,13 @@ static inline bool arch_is_in_isr(void) return arch_curr_cpu()->nested != 0U; } +#ifdef CONFIG_USERSPACE +extern void z_xtensa_userspace_enter(k_thread_entry_t user_entry, + void *p1, void *p2, void *p3, + uintptr_t stack_end, + uintptr_t stack_start); +#endif /* CONFIG_USERSPACE */ + #ifdef __cplusplus } #endif diff --git a/arch/xtensa/include/offsets_short_arch.h b/arch/xtensa/include/offsets_short_arch.h index 34a4a5842cf753..f19750dc0ac8b9 100644 --- a/arch/xtensa/include/offsets_short_arch.h +++ b/arch/xtensa/include/offsets_short_arch.h @@ -2,4 +2,18 @@ * Copyright (c) 2021 Intel Corporation * SPDX-License-Identifier: Apache-2.0 */ -/* Empty File */ +#ifndef ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ +#define ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ + +#define _thread_offset_to_flags \ + (___thread_t_arch_OFFSET + ___thread_arch_t_flags_OFFSET) + +#ifdef CONFIG_USERSPACE +#define _thread_offset_to_psp \ + (___thread_t_arch_OFFSET + ___thread_arch_t_psp_OFFSET) + +#define _thread_offset_to_ptables \ + (___thread_t_arch_OFFSET + ___thread_arch_t_ptables_OFFSET) +#endif /* CONFIG_USERSPACE */ + +#endif /* ZEPHYR_ARCH_XTENSA_INCLUDE_OFFSETS_SHORT_ARCH_H_ */ diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index f691dbc6cad9b1..a98c61db253a99 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -176,7 +176,8 @@ rsr.SCOMPARE1 a0 s32i a0, a1, ___xtensa_irq_bsa_t_scompare1_OFFSET #endif -#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE) +#if XCHAL_HAVE_THREADPTR && \ + (defined(CONFIG_USERSPACE) || defined(CONFIG_THREAD_LOCAL_STORAGE)) rur.THREADPTR a0 s32i a0, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET #endif @@ -409,6 +410,16 @@ _xstack_returned_\@: l32i a2, a1, 0 l32i a2, a2, ___xtensa_irq_bsa_t_scratch_OFFSET +#if XCHAL_HAVE_THREADPTR && defined(CONFIG_USERSPACE) + /* Clear up the threadptr because it is used + * to check if a thread is runnig on user mode. Since + * we are in a interruption we don't want the system + * thinking it is possbly running in user mode. + */ + movi.n a0, 0 + wur.THREADPTR a0 +#endif /* XCHAL_HAVE_THREADPTR && CONFIG_USERSPACE */ + /* There's a gotcha with level 1 handlers: the INTLEVEL field * gets left at zero and not set like high priority interrupts * do. That works fine for exceptions, but for L1 interrupts, diff --git a/include/zephyr/arch/syscall.h b/include/zephyr/arch/syscall.h index b657717e3d4bc6..5b41561b681907 100644 --- a/include/zephyr/arch/syscall.h +++ b/include/zephyr/arch/syscall.h @@ -23,6 +23,8 @@ #include #elif defined(CONFIG_RISCV) #include +#elif defined(CONFIG_XTENSA) +#include #endif #endif /* ZEPHYR_INCLUDE_ARCH_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 0838e68ee5a64d..397e5c7fa7894a 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -47,6 +48,15 @@ extern "C" { #endif +struct arch_mem_domain { +#ifdef CONFIG_XTENSA_MMU + uint32_t *ptables __aligned(CONFIG_MMU_PAGE_SIZE); + uint8_t asid; + bool dirty; +#endif + sys_snode_t node; +}; + extern void xtensa_arch_except(int reason_p); #define ARCH_EXCEPT(reason_p) do { \ diff --git a/include/zephyr/arch/xtensa/syscall.h b/include/zephyr/arch/xtensa/syscall.h new file mode 100644 index 00000000000000..3d78827b77c7cc --- /dev/null +++ b/include/zephyr/arch/xtensa/syscall.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2022 Intel Corporation. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief Xtensa specific syscall header + * + * This header contains the Xtensa specific syscall interface. It is + * included by the syscall interface architecture-abstraction header + * (include/arch/syscall.h) + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ +#define ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ + +#ifdef CONFIG_USERSPACE +#ifndef _ASMLANGUAGE + +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER +uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id); + +uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id); + +uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id); + +uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id); +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ + +/** + * We are following Linux Xtensa syscall ABI: + * + * syscall number arg1, arg2, arg3, arg4, arg5, arg6 + * -------------- ---------------------------------- + * a2 a6, a3, a4, a5, a8, a9 + * + **/ + +static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke6_helper(arg1, arg2, arg3, + arg4, arg5, arg6, + call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + register uintptr_t a9 __asm__("%a9") = arg6; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8), "r" (a9) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke5_helper(arg1, arg2, arg3, + arg4, arg5, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + register uintptr_t a8 __asm__("%a8") = arg5; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5), "r" (a8) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke4_helper(arg1, arg2, arg3, arg4, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + register uintptr_t a5 __asm__("%a5") = arg4; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4), + "r" (a5) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke3_helper(arg1, arg2, arg3, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + register uintptr_t a4 __asm__("%a4") = arg3; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3), "r" (a4) + : "memory"); + + return a2; +#endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ +} + +static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke2_helper(arg1, arg2, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + register uintptr_t a3 __asm__("%a3") = arg2; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6), "r" (a3) + : "memory"); + + return a2; +#endif +} + +static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1, + uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke1_helper(arg1, call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + register uintptr_t a6 __asm__("%a6") = arg1; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2), "r" (a6) + : "memory"); + + return a2; +#endif +} + +static inline uintptr_t arch_syscall_invoke0(uintptr_t call_id) +{ +#ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER + return arch_syscall_invoke0_helper(call_id); +#else + register uintptr_t a2 __asm__("%a2") = call_id; + + __asm__ volatile("syscall\n\t" + : "=r" (a2) + : "r" (a2) + : "memory"); + + return a2; +#endif +} + +/* + * There is no easy (or generic) way to figure out if a thread is runnining + * in un-privileged mode. Reading the currrent ring (PS.CRING) is a privileged + * instruction and not thread local storage is not available in xcc. + */ +static inline bool arch_is_user_context(void) +{ + uint32_t thread; + + __asm__ volatile( + "rur.THREADPTR %0\n\t" + : "=a" (thread) + ); + + return !!thread; +} + +#ifdef __cplusplus +} +#endif + +#endif /* _ASMLANGUAGE */ +#endif /* CONFIG_USERSPACE */ +#endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_SYSCALL_H_ */ diff --git a/include/zephyr/arch/xtensa/thread.h b/include/zephyr/arch/xtensa/thread.h index 4ec5da1ea2c048..2bebe2722bcb2d 100644 --- a/include/zephyr/arch/xtensa/thread.h +++ b/include/zephyr/arch/xtensa/thread.h @@ -7,6 +7,7 @@ #ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ #define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_H_ +#include #ifndef _ASMLANGUAGE /* Xtensa doesn't use these structs, but Zephyr core requires they be @@ -22,6 +23,14 @@ typedef struct _callee_saved _callee_saved_t; struct _thread_arch { uint32_t last_cpu; +#ifdef CONFIG_USERSPACE + uint32_t *ptables; + + /* Initial privilege mode stack pointer when doing a system call. + * Un-set for surpervisor threads. + */ + uint8_t *psp; +#endif }; typedef struct _thread_arch _thread_arch_t; diff --git a/include/zephyr/arch/xtensa/xtensa_mmu.h b/include/zephyr/arch/xtensa/xtensa_mmu.h index 2ee3dc6c9ab8ee..40fd0ff8f5c114 100644 --- a/include/zephyr/arch/xtensa/xtensa_mmu.h +++ b/include/zephyr/arch/xtensa/xtensa_mmu.h @@ -9,8 +9,40 @@ #define Z_XTENSA_MMU_X BIT(0) #define Z_XTENSA_MMU_W BIT(1) +#define Z_XTENSA_MMU_XW (BIT(1) | BIT(0)) + #define Z_XTENSA_MMU_CACHED_WB BIT(2) #define Z_XTENSA_MMU_CACHED_WT BIT(3) + +#define K_MEM_PARTITION_IS_EXECUTABLE(attr) (((attr) & Z_XTENSA_MMU_X) != 0) +#define K_MEM_PARTITION_IS_WRITABLE(attr) (((attr) & Z_XENSA_MMU_W) != 0) + +/* Read-Write access permission attributes */ +#define K_MEM_PARTITION_P_RW_U_RW ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_W}) +#define K_MEM_PARTITION_P_RW_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_RO ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_RO_U_NA ((k_mem_partition_attr_t) \ + {0}) +#define K_MEM_PARTITION_P_NA_U_NA ((k_mem_partition_attr_t) \ + {0}) + +/* Execution-allowed attributes */ +#define K_MEM_PARTITION_P_RX_U_RX ((k_mem_partition_attr_t) \ + {Z_XTENSA_MMU_X}) + +/* + * This BIT tells the mapping code whether the uncached pointer should + * be shared between all threads. That is not used in the HW, it is + * just for the implementation. + * + * The pte mapping this memory will use an ASID that is set in the + * ring 4 spot in RASID. + */ +#define Z_XTENSA_MMU_MAP_SHARED BIT(30) + #define Z_XTENSA_MMU_ILLEGAL (BIT(3) | BIT(2)) /* Struct used to map a memory region */ @@ -21,6 +53,8 @@ struct xtensa_mmu_range { const uint32_t attrs; }; +typedef uint32_t k_mem_partition_attr_t; + extern const struct xtensa_mmu_range xtensa_soc_mmu_ranges[]; extern int xtensa_soc_mmu_ranges_num; From 06b57cbbe113d19be7b9a54bbf5c558c00ce66ca Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Wed, 16 Nov 2022 22:48:42 -0800 Subject: [PATCH 07/33] xtensa: userspace: Stack object header Add a header with architecture specific macros and definitions that re used on userspace for stack objects. Signed-off-by: Flavio Ceolin --- include/zephyr/arch/xtensa/arch.h | 7 +-- include/zephyr/arch/xtensa/thread_stack.h | 67 +++++++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 include/zephyr/arch/xtensa/thread_stack.h diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 397e5c7fa7894a..9f2fb757a116c5 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -29,16 +29,11 @@ #include #include #include +#include #include #include -#ifdef CONFIG_KERNEL_COHERENCE -#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE -#else -#define ARCH_STACK_PTR_ALIGN 16 -#endif - /* Xtensa GPRs are often designated by two different names */ #define sys_define_gpr_with_alias(name1, name2) union { uint32_t name1, name2; } diff --git a/include/zephyr/arch/xtensa/thread_stack.h b/include/zephyr/arch/xtensa/thread_stack.h new file mode 100644 index 00000000000000..eaa160ccf1fdd4 --- /dev/null +++ b/include/zephyr/arch/xtensa/thread_stack.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2022 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_STACK_H_ +#define ZEPHYR_INCLUDE_ARCH_XTENSA_THREAD_STACK_H_ + +#include +#include + +#ifdef CONFIG_KERNEL_COHERENCE +#define ARCH_STACK_PTR_ALIGN XCHAL_DCACHE_LINESIZE +#else +#define ARCH_STACK_PTR_ALIGN 16 +#endif + + +#if CONFIG_USERSPACE +#define Z_XTENSA_STACK_BASE_ALIGN CONFIG_MMU_PAGE_SIZE +#define Z_XTENSA_STACK_SIZE_ALIGN CONFIG_MMU_PAGE_SIZE +#else +#define Z_XTENSA_STACK_BASE_ALIGN ARCH_STACK_PTR_ALIGN +#define Z_XTENSA_STACK_SIZE_ALIGN ARCH_STACK_PTR_ALIGN +#endif + +/* + * + * High memory addresses + * + * +-------------------+ <- thread.stack_info.start + thread.stack_info.size + * | TLS | + * +-------------------+ <- initial sp (computable with thread.stack_info.delta) + * | | + * | Thread stack | + * | | + * +-------------------+ <- thread.stack_info.start + * | Privileged stack | } CONFIG_MMU_PAGE_SIZE + * +-------------------+ <- thread.stack_obj + * + * Low Memory addresses + */ + +#ifndef _ASMLANGUAGE + +/* thread stack */ +#ifdef CONFIG_XTENSA_MMU +struct z_xtensa_thread_stack_header { + char privilege_stack[CONFIG_MMU_PAGE_SIZE]; +} __packed __aligned(Z_XTENSA_STACK_BASE_ALIGN); + +#define ARCH_THREAD_STACK_RESERVED \ + sizeof(struct z_xtensa_thread_stack_header) +#endif /* CONFIG_XTENSA_MMU */ + +#define ARCH_THREAD_STACK_OBJ_ALIGN(size) Z_XTENSA_STACK_BASE_ALIGN +#define ARCH_THREAD_STACK_SIZE_ADJUST(size) \ + ROUND_UP((size), Z_XTENSA_STACK_SIZE_ALIGN) + +/* kernel stack */ +#define ARCH_KERNEL_STACK_RESERVED 0 +#define ARCH_KERNEL_STACK_OBJ_ALIGN ARCH_STACK_PTR_ALIGN + +#endif /* _ASMLANGUAGE */ + +#endif From 725bd969dbc55be11d61cf1f2ced80c48c24d2cc Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 31 Jan 2023 13:07:48 -0800 Subject: [PATCH 08/33] xtensa: mmu: handle page faults in double exception handler This changes the TLB misses handling back to the assembly in user exception, and any page faults during TLB misses to be handled in double exception handler. This should speed up simple TLB miss handling as we don't have to go all the way to the C handler. Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- arch/xtensa/core/xtensa-asm2-util.S | 6 +- arch/xtensa/core/xtensa-asm2.c | 95 +++++++++++------------------ 2 files changed, 41 insertions(+), 60 deletions(-) diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index c108a09dee4814..1ef9e5e1b4fd4b 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -447,7 +447,10 @@ _DoubleExceptionVector: beqz a0, _handle_tlb_miss_dblexc rsr a0, ZSR_A0SAVE -#endif + + j _Level1Vector +#else + #if defined(CONFIG_SIMULATOR_XTENSA) || defined(XT_SIMULATOR) 1: /* Tell simulator to stop executing here, instead of trying to do @@ -465,6 +468,7 @@ _DoubleExceptionVector: 1: #endif j 1b +#endif /* CONFIG_XTENSA_MMU */ #ifdef CONFIG_XTENSA_MMU _handle_tlb_miss_dblexc: diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index 1e8c92759e42fb..3fb519cd80de8f 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -293,34 +293,31 @@ void *xtensa_excint1_c(int *interrupted_stack) uint32_t ps; void *pc; +#ifdef CONFIG_XTENSA_MMU + bool is_dblexc; + uint32_t depc; +#else + const bool is_dblexc = false; +#endif /* CONFIG_XTENSA_MMU */ + __asm__ volatile("rsr.exccause %0" : "=r"(cause)); #ifdef CONFIG_XTENSA_MMU - /* TLB miss exception comes through level 1 interrupt also. - * We need to preserve execution context after we have handled - * the TLB miss, so we cannot unconditionally unmask interrupts. - * For other cause, we can unmask interrupts so this would act - * the same as if there is no MMU. - */ - switch (cause) { - case EXCCAUSE_ITLB_MISS: - /* Instruction TLB miss */ - __fallthrough; - case EXCCAUSE_DTLB_MISS: - /* Data TLB miss */ + __asm__ volatile("rsr.depc %0" : "=r"(depc)); - /* Do not unmask interrupt while handling TLB misses. */ - break; - default: - /* For others, we can unmask interrupts. */ - bsa->ps &= ~PS_INTLEVEL_MASK; - break; - } + is_dblexc = (depc != 0U); #endif /* CONFIG_XTENSA_MMU */ switch (cause) { case EXCCAUSE_LEVEL1_INTERRUPT: - return xtensa_int1_c(interrupted_stack); + if (!is_dblexc) { + return xtensa_int1_c(interrupted_stack); + } + break; +#ifndef CONFIG_USERSPACE + /* Syscalls are handled earlier in assembly if MMU is enabled. + * So we don't need this here. + */ case EXCCAUSE_SYSCALL: /* Just report it to the console for now */ LOG_ERR(" ** SYSCALL PS %p PC %p", @@ -333,38 +330,7 @@ void *xtensa_excint1_c(int *interrupted_stack) */ bsa->pc += 3; break; -#ifdef CONFIG_XTENSA_MMU - case EXCCAUSE_ITLB_MISS: - /* Instruction TLB miss */ - __fallthrough; - case EXCCAUSE_DTLB_MISS: - /* Data TLB miss */ - - /** - * The way it works is, when we try to access an address - * that is not mapped, we will have a miss. The HW then - * will try to get the correspondent memory in the page - * table. As the page table is not mapped in memory we will - * have a second miss, which will trigger an exception. - * In the exception (here) what we do is to exploit this - * hardware capability just trying to load the page table - * (not mapped address), which will cause a miss, but then - * the hardware will automatically map it again from - * the page table. This time it will work since the page - * necessary to map the page table itself are wired map. - */ - __asm__ volatile("wsr a0, " ZSR_EXTRA0_STR "\n\t" - "rsr.ptevaddr a0\n\t" - "l32i a0, a0, 0\n\t" - "rsr a0, " ZSR_EXTRA0_STR "\n\t" - "rsync" - : : : "a0", "memory"); - - /* Since we are dealing with TLB misses, we will probably not - * want to switch to another thread. - */ - return interrupted_stack; -#endif /* CONFIG_XTENSA_MMU */ +#endif /* !CONFIG_USERSPACE */ default: ps = bsa->ps; pc = (void *)bsa->pc; @@ -373,6 +339,7 @@ void *xtensa_excint1_c(int *interrupted_stack) /* Default for exception */ int reason = K_ERR_CPU_EXCEPTION; + is_fatal_error = true; /* We need to distinguish between an ill in xtensa_arch_except, * e.g for k_panic, and any other ill. For exceptions caused by @@ -389,13 +356,19 @@ void *xtensa_excint1_c(int *interrupted_stack) reason = bsa->a2; } - LOG_ERR(" ** FATAL EXCEPTION"); + LOG_ERR(" ** FATAL EXCEPTION%s", (is_dblexc ? " (DOUBLE)" : "")); LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)", arch_curr_cpu()->id, cause, z_xtensa_exccause(cause)); LOG_ERR(" ** PC %p VADDR %p", pc, (void *)vaddr); LOG_ERR(" ** PS %p", (void *)bsa->ps); + if (is_dblexc) { + LOG_ERR(" ** DEPC %p", (void *)depc); + } +#ifdef CONFIG_USERSPACE + LOG_ERR(" ** THREADPTR %p", (void *)bsa->threadptr); +#endif /* CONFIG_USERSPACE */ LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", get_bits(0, 4, ps), get_bits(4, 1, ps), get_bits(5, 1, ps), get_bits(6, 2, ps), @@ -412,13 +385,12 @@ void *xtensa_excint1_c(int *interrupted_stack) break; } - +#ifdef CONFIG_XTENSA_MMU switch (cause) { - case EXCCAUSE_SYSCALL: case EXCCAUSE_LEVEL1_INTERRUPT: - case EXCCAUSE_ALLOCA: - case EXCCAUSE_ITLB_MISS: - case EXCCAUSE_DTLB_MISS: +#ifndef CONFIG_USERSPACE + case EXCCAUSE_SYSCALL: +#endif /* !CONFIG_USERSPACE */ is_fatal_error = false; break; default: @@ -426,7 +398,12 @@ void *xtensa_excint1_c(int *interrupted_stack) break; } - if (is_fatal_error) { + if (is_dblexc) { + __asm__ volatile("wsr.depc %0" : : "r"(0)); + } +#endif /* CONFIG_XTENSA_MMU */ + + if (is_dblexc || is_fatal_error) { uint32_t ignore; /* We are going to manipulate _current_cpu->nested manually. From 5396ef6e13f2d86f6dce6d464b013a76f18623d5 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 16 Feb 2023 11:25:48 -0800 Subject: [PATCH 09/33] xtensa: mmu: allocate scratch registers for MMU When MMU is enabled, we need some scratch registers to preload page table entries. So update gen_zsr.py to that. Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- arch/xtensa/core/CMakeLists.txt | 1 + arch/xtensa/core/gen_zsr.py | 24 ++++++++++++++++++++---- arch/xtensa/core/xtensa-asm2-util.S | 6 +++--- arch/xtensa/include/xtensa-asm2-s.h | 8 ++++---- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index 8a23b65b9a9b16..331bcd9bfc52d3 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -59,6 +59,7 @@ add_custom_command(OUTPUT ${CORE_ISA_DM} set(ZSR_H ${CMAKE_BINARY_DIR}/zephyr/include/generated/zsr.h) add_custom_command(OUTPUT ${ZSR_H} DEPENDS ${CORE_ISA_DM} COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/gen_zsr.py + $<$:--mmu> ${CORE_ISA_DM} ${ZSR_H}) add_custom_target(zsr_h DEPENDS ${ZSR_H}) add_dependencies(zephyr_interface zsr_h) diff --git a/arch/xtensa/core/gen_zsr.py b/arch/xtensa/core/gen_zsr.py index 574542a4578ed1..0e3069a4c450d4 100755 --- a/arch/xtensa/core/gen_zsr.py +++ b/arch/xtensa/core/gen_zsr.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Copyright (c) 2022 Intel corporation # SPDX-License-Identifier: Apache-2.0 -import sys +import argparse import re # Scratch register allocator. Zephyr uses multiple Xtensa SRs as @@ -11,10 +11,26 @@ # -dM") core-isa.h file for the current architecture and assigns # registers to usages. -NEEDED = ("A0SAVE", "CPU", "FLUSH") +def parse_args(): + parser = argparse.ArgumentParser(allow_abbrev=False) -coreisa = sys.argv[1] -outfile = sys.argv[2] + parser.add_argument("--mmu", action="store_true", + help="Enable scratch registers for MMU usage") + parser.add_argument("coreisa", + help="Path to preprocessed core-isa.h") + parser.add_argument("outfile", + help="Output file") + + return parser.parse_args() + +args = parse_args() + +NEEDED = ["A0SAVE", "CPU", "FLUSH"] +if args.mmu: + NEEDED += ["MMU_0", "MMU_1", "DBLEXC"] + +coreisa = args.coreisa +outfile = args.outfile syms = {} diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index 1ef9e5e1b4fd4b..7dfd1d0bc92b31 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -439,14 +439,14 @@ _handle_tlb_miss_kernel: .global _DoubleExceptionVector _DoubleExceptionVector: #ifdef CONFIG_XTENSA_MMU - wsr a0, ZSR_A0SAVE + wsr a0, ZSR_DBLEXC rsync rsr.exccause a0 addi a0, a0, -EXCCAUSE_DTLB_MISS beqz a0, _handle_tlb_miss_dblexc - rsr a0, ZSR_A0SAVE + rsr a0, ZSR_DBLEXC j _Level1Vector #else @@ -482,7 +482,7 @@ _handle_tlb_miss_dblexc: rsr.ptevaddr a0 l32i a0, a0, 0 - rsr a0, ZSR_A0SAVE + rsr a0, ZSR_DBLEXC rfde #endif .popsection diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index a98c61db253a99..8c08916c8c6521 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -558,8 +558,8 @@ _Level\LVL\()VectorHelper : _Level\LVL\()Vector: #endif #ifdef CONFIG_XTENSA_MMU - wsr.ZSR_EXTRA0 a2 - wsr.ZSR_EXTRA1 a3 + wsr.ZSR_MMU_0 a2 + wsr.ZSR_MMU_1 a3 rsync /* Calculations below will clobber registers used. @@ -579,8 +579,8 @@ _Level\LVL\()Vector: rsr.ZSR_CPU a3 PRELOAD_PTEVADDR a3, a2 - rsr.ZSR_EXTRA1 a3 - rsr.ZSR_EXTRA0 a2 + rsr.ZSR_MMU_1 a3 + rsr.ZSR_MMU_0 a2 #endif /* CONFIG_XTENSA_MMU */ addi a1, a1, -___xtensa_irq_bsa_t_SIZEOF s32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET From 4f0199e39532bae05b1b1e16da52778dfbdfd5ec Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 16 Mar 2023 14:34:28 -0700 Subject: [PATCH 10/33] xtensa: mmu: do not fault for known exceptions There are known exceptions which are not fatal, and we need to handle them properly by returning to the fixup addresses as indicated. This adds the code necessary in the exception handler for this situation. Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- arch/xtensa/core/fatal.c | 8 ------- arch/xtensa/core/xtensa-asm2.c | 38 ++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index 3117ebc4a56d70..b262a8aad4dac7 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -121,14 +121,6 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf) z_fatal_error(reason, esf); } -#ifdef CONFIG_USERSPACE -Z_EXC_DECLARE(z_xtensa_user_string_nlen); - -static const struct z_exc_handle exceptions[] = { - Z_EXC_HANDLE(z_xtensa_user_string_nlen) -}; -#endif /* CONFIG_USERSPACE */ - #ifdef XT_SIMULATOR void exit(int return_code) { diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index 3fb519cd80de8f..79c784f5f7d99a 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -15,11 +15,20 @@ #include #include #include +#include LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); extern char xtensa_arch_except_epc[]; +#ifdef CONFIG_USERSPACE +Z_EXC_DECLARE(z_xtensa_user_string_nlen); + +static const struct z_exc_handle exceptions[] = { + Z_EXC_HANDLE(z_xtensa_user_string_nlen) +}; +#endif /* CONFIG_USERSPACE */ + void *xtensa_init_stack(struct k_thread *thread, int *stack_top, void (*entry)(void *, void *, void *), void *arg1, void *arg2, void *arg3) @@ -335,6 +344,21 @@ void *xtensa_excint1_c(int *interrupted_stack) ps = bsa->ps; pc = (void *)bsa->pc; +#ifdef CONFIG_USERSPACE + /* If the faulting address is from one of the known + * exceptions that should not be fatal, return to + * the fixup address. + */ + for (int i = 0; i < ARRAY_SIZE(exceptions); i++) { + if ((pc >= exceptions[i].start) && + (pc < exceptions[i].end)) { + bsa->pc = (uintptr_t)exceptions[i].fixup; + + goto fixup_out; + } + } +#endif /* CONFIG_USERSPACE */ + __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); /* Default for exception */ @@ -397,10 +421,6 @@ void *xtensa_excint1_c(int *interrupted_stack) is_fatal_error = true; break; } - - if (is_dblexc) { - __asm__ volatile("wsr.depc %0" : : "r"(0)); - } #endif /* CONFIG_XTENSA_MMU */ if (is_dblexc || is_fatal_error) { @@ -432,6 +452,16 @@ void *xtensa_excint1_c(int *interrupted_stack) _current_cpu->nested = 1; } +#ifdef CONFIG_XTENSA_MMU +#ifdef CONFIG_USERSPACE +fixup_out: +#endif + if (is_dblexc) { + __asm__ volatile("wsr.depc %0" : : "r"(0)); + } +#endif /* CONFIG_XTENSA_MMU */ + + return return_to(interrupted_stack); } From e62efe6249e76a24e7b877b62ea5633b4ca215a1 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 12 Apr 2023 16:04:12 -0700 Subject: [PATCH 11/33] xtensa: extract printing of fatal exception into its own func This extracts the printing of fatal exception information into its own function to declutter xtensa_excint1_c(). Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- arch/xtensa/core/xtensa-asm2.c | 65 +++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index 79c784f5f7d99a..92b410c46c8258 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -202,6 +202,39 @@ static inline unsigned int get_bits(int offset, int num_bits, unsigned int val) return val & mask; } +static void print_fatal_exception(_xtensa_irq_bsa_t *bsa, int cause, + bool is_dblexc, uint32_t depc) +{ + void *pc; + uint32_t ps, vaddr; + + ps = bsa->ps; + pc = (void *)bsa->pc; + + __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); + + LOG_ERR(" ** FATAL EXCEPTION%s", (is_dblexc ? " (DOUBLE)" : "")); + LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)", + arch_curr_cpu()->id, cause, + z_xtensa_exccause(cause)); + LOG_ERR(" ** PC %p VADDR %p", pc, (void *)vaddr); + + if (is_dblexc) { + LOG_ERR(" ** DEPC %p", (void *)depc); + } + +#ifdef CONFIG_USERSPACE + LOG_ERR(" ** THREADPTR %p", (void *)bsa->threadptr); +#endif /* CONFIG_USERSPACE */ + + LOG_ERR(" ** PS %p", (void *)bsa->ps); + LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", + get_bits(0, 4, ps), get_bits(4, 1, ps), + get_bits(5, 1, ps), get_bits(6, 2, ps), + get_bits(18, 1, ps), + get_bits(8, 4, ps), get_bits(16, 2, ps)); +} + static ALWAYS_INLINE void usage_stop(void) { #ifdef CONFIG_SCHED_THREAD_USAGE @@ -296,18 +329,13 @@ static inline DEF_INT_C_HANDLER(1) */ void *xtensa_excint1_c(int *interrupted_stack) { - int cause, vaddr; + int cause; _xtensa_irq_bsa_t *bsa = (void *)*(int **)interrupted_stack; bool is_fatal_error = false; + bool is_dblexc = false; uint32_t ps; void *pc; - -#ifdef CONFIG_XTENSA_MMU - bool is_dblexc; - uint32_t depc; -#else - const bool is_dblexc = false; -#endif /* CONFIG_XTENSA_MMU */ + uint32_t depc = 0; __asm__ volatile("rsr.exccause %0" : "=r"(cause)); @@ -359,8 +387,6 @@ void *xtensa_excint1_c(int *interrupted_stack) } #endif /* CONFIG_USERSPACE */ - __asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr)); - /* Default for exception */ int reason = K_ERR_CPU_EXCEPTION; is_fatal_error = true; @@ -380,24 +406,7 @@ void *xtensa_excint1_c(int *interrupted_stack) reason = bsa->a2; } - LOG_ERR(" ** FATAL EXCEPTION%s", (is_dblexc ? " (DOUBLE)" : "")); - LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)", - arch_curr_cpu()->id, cause, - z_xtensa_exccause(cause)); - LOG_ERR(" ** PC %p VADDR %p", - pc, (void *)vaddr); - LOG_ERR(" ** PS %p", (void *)bsa->ps); - if (is_dblexc) { - LOG_ERR(" ** DEPC %p", (void *)depc); - } -#ifdef CONFIG_USERSPACE - LOG_ERR(" ** THREADPTR %p", (void *)bsa->threadptr); -#endif /* CONFIG_USERSPACE */ - LOG_ERR(" ** (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)", - get_bits(0, 4, ps), get_bits(4, 1, ps), - get_bits(5, 1, ps), get_bits(6, 2, ps), - get_bits(18, 1, ps), - get_bits(8, 4, ps), get_bits(16, 2, ps)); + print_fatal_exception(bsa, cause, is_dblexc, depc); /* FIXME: legacy xtensa port reported "HW" exception * for all unhandled exceptions, which seems incorrect From 6d0119d6af6b76359998a63f051dc37e5f3d06cb Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Thu, 15 Dec 2022 15:05:14 -0800 Subject: [PATCH 12/33] xtensa: userspace: Implement arch_syscall_oops This function is needed by userspace. Signed-off-by: Flavio Ceolin --- arch/xtensa/core/fatal.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index b262a8aad4dac7..fbf19d9df91039 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -141,3 +141,11 @@ FUNC_NORETURN void z_system_halt(unsigned int reason) CODE_UNREACHABLE; } #endif + +FUNC_NORETURN void arch_syscall_oops(void *ssf) +{ + z_arch_esf_t *esf = ssf; + + z_xtensa_fatal_error(K_ERR_KERNEL_OOPS, esf); + CODE_UNREACHABLE; +} From 04718a1c93fb1ae8ec7bc27c277c60ddfad27c81 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Wed, 8 Feb 2023 13:35:18 -0800 Subject: [PATCH 13/33] xtensa: userspace: Add syscall for user exception Trigger exception on Xtensa requires kernel privileges. Add a new syscall that is used when ARCH_EXCEPT is invoked from userspace. Signed-off-by: Flavio Ceolin --- arch/xtensa/CMakeLists.txt | 1 + arch/xtensa/core/fatal.c | 21 +++++++++++++++++++++ include/zephyr/arch/xtensa/arch.h | 21 +++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/arch/xtensa/CMakeLists.txt b/arch/xtensa/CMakeLists.txt index 133d74331d8e66..4626421f11a4ae 100644 --- a/arch/xtensa/CMakeLists.txt +++ b/arch/xtensa/CMakeLists.txt @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 +zephyr_syscall_header(${ZEPHYR_BASE}/include/zephyr/arch/xtensa/arch.h) set_property(GLOBAL PROPERTY PROPERTY_OUTPUT_FORMAT elf32-xtensa-le) add_subdirectory(core) diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index fbf19d9df91039..307f4b6ff1de0a 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -149,3 +149,24 @@ FUNC_NORETURN void arch_syscall_oops(void *ssf) z_xtensa_fatal_error(K_ERR_KERNEL_OOPS, esf); CODE_UNREACHABLE; } + +#ifdef CONFIG_USERSPACE +void z_impl_xtensa_user_fault(unsigned int reason) +{ + if ((_current->base.user_options & K_USER) != 0) { + if ((reason != K_ERR_KERNEL_OOPS) && + (reason != K_ERR_STACK_CHK_FAIL)) { + reason = K_ERR_KERNEL_OOPS; + } + } + xtensa_arch_except(reason); +} + +static void z_vrfy_xtensa_user_fault(unsigned int reason) +{ + z_impl_xtensa_user_fault(reason); +} + +#include + +#endif /* CONFIG_USERSPACE */ diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 9f2fb757a116c5..16bfd08091a475 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -54,11 +55,31 @@ struct arch_mem_domain { extern void xtensa_arch_except(int reason_p); +#ifdef CONFIG_USERSPACE + +#define ARCH_EXCEPT(reason_p) do { \ + if (k_is_user_context()) { \ + arch_syscall_invoke1(reason_p, \ + K_SYSCALL_XTENSA_USER_FAULT); \ + } else { \ + xtensa_arch_except(reason_p); \ + } \ + CODE_UNREACHABLE; \ +} while (false) + +#else + #define ARCH_EXCEPT(reason_p) do { \ xtensa_arch_except(reason_p); \ CODE_UNREACHABLE; \ } while (false) +#endif + +__syscall void xtensa_user_fault(unsigned int reason); + +#include + /* internal routine documented in C file, needed by IRQ_CONNECT() macro */ extern void z_irq_priority_set(uint32_t irq, uint32_t prio, uint32_t flags); From 9fe55ba7c9538fb86a21f2fd542a6a83ac7f47a3 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 12 Apr 2023 15:42:38 -0700 Subject: [PATCH 14/33] xtensa: rework kernel oops exception path When kernel OOPS is raised, we need to actually go through the process of terminating the offending thread, instead of simply printing the stack and continue running. This change employs similar mechanism to xtensa_arch_except() to use illegal instruction to raise hardware exception, and going through the fatal exception path. Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- arch/xtensa/core/fatal.c | 8 ++++--- arch/xtensa/core/xtensa-asm2-util.S | 14 +++++++++++ arch/xtensa/core/xtensa-asm2.c | 36 ++++++++++++++++++++++------- include/zephyr/arch/xtensa/arch.h | 1 + 4 files changed, 48 insertions(+), 11 deletions(-) diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index 307f4b6ff1de0a..79a24baa105624 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -87,6 +87,8 @@ char *z_xtensa_exccause(unsigned int cause_code) case 63: /* i.e. z_except_reason */ return "zephyr exception"; + case 64: + return "kernel oops"; default: return "unknown/reserved"; } @@ -114,7 +116,6 @@ void z_xtensa_fatal_error(unsigned int reason, const z_arch_esf_t *esf) z_xtensa_backtrace_print(100, (int *)esf); #endif #endif - arch_irq_unlock(key); } @@ -144,9 +145,10 @@ FUNC_NORETURN void z_system_halt(unsigned int reason) FUNC_NORETURN void arch_syscall_oops(void *ssf) { - z_arch_esf_t *esf = ssf; + ARG_UNUSED(ssf); + + xtensa_arch_kernel_oops(K_ERR_KERNEL_OOPS, ssf); - z_xtensa_fatal_error(K_ERR_KERNEL_OOPS, esf); CODE_UNREACHABLE; } diff --git a/arch/xtensa/core/xtensa-asm2-util.S b/arch/xtensa/core/xtensa-asm2-util.S index 7dfd1d0bc92b31..4f8af894ccb572 100644 --- a/arch/xtensa/core/xtensa-asm2-util.S +++ b/arch/xtensa/core/xtensa-asm2-util.S @@ -204,6 +204,20 @@ xtensa_arch_except_epc: ill retw +/* + * void xtensa_arch_kernel_oops(int reason_p, void *ssf); + * + * Simply to raise hardware exception for Kernel OOPS. + */ +.global xtensa_arch_kernel_oops +.global xtensa_arch_kernel_oops_epc +.align 4 +xtensa_arch_kernel_oops: + entry a1, 16 +xtensa_arch_kernel_oops_epc: + ill + retw + /* * void xtensa_switch(void *new, void **old_return); * diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index 92b410c46c8258..ed8f340f1bcb45 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -20,6 +20,7 @@ LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL); extern char xtensa_arch_except_epc[]; +extern char xtensa_arch_kernel_oops_epc[]; #ifdef CONFIG_USERSPACE Z_EXC_DECLARE(z_xtensa_user_string_nlen); @@ -202,11 +203,12 @@ static inline unsigned int get_bits(int offset, int num_bits, unsigned int val) return val & mask; } -static void print_fatal_exception(_xtensa_irq_bsa_t *bsa, int cause, +static void print_fatal_exception(void *print_stack, int cause, bool is_dblexc, uint32_t depc) { void *pc; uint32_t ps, vaddr; + _xtensa_irq_bsa_t *bsa = (void *)*(int **)print_stack; ps = bsa->ps; pc = (void *)bsa->pc; @@ -334,7 +336,7 @@ void *xtensa_excint1_c(int *interrupted_stack) bool is_fatal_error = false; bool is_dblexc = false; uint32_t ps; - void *pc; + void *pc, *print_stack = (void *)interrupted_stack; uint32_t depc = 0; __asm__ volatile("rsr.exccause %0" : "=r"(cause)); @@ -399,14 +401,32 @@ void *xtensa_excint1_c(int *interrupted_stack) * We assign EXCCAUSE the unused, reserved code 63; this may be * problematic if the app or new boards also decide to repurpose * this code. + * + * Another intentionally ill is from xtensa_arch_kernel_oops. + * Kernel OOPS has to be explicity raised so we can simply + * set the reason and continue. */ - if ((pc == (void *) &xtensa_arch_except_epc) && (cause == 0)) { - cause = 63; - __asm__ volatile("wsr.exccause %0" : : "r"(cause)); - reason = bsa->a2; + if (cause == EXCCAUSE_ILLEGAL) { + if (pc == (void *)&xtensa_arch_except_epc) { + cause = 63; + __asm__ volatile("wsr.exccause %0" : : "r"(cause)); + reason = bsa->a2; + } else if (pc == (void *)&xtensa_arch_kernel_oops_epc) { + cause = 64; /* kernel oops */ + reason = K_ERR_KERNEL_OOPS; + + /* A3 contains the second argument to + * xtensa_arch_kernel_oops(reason, ssf) + * where ssf is the stack frame causing + * the kernel oops. + */ + print_stack = (void *)bsa->a3; + } } - print_fatal_exception(bsa, cause, is_dblexc, depc); + if (reason != K_ERR_KERNEL_OOPS) { + print_fatal_exception(print_stack, cause, is_dblexc, depc); + } /* FIXME: legacy xtensa port reported "HW" exception * for all unhandled exceptions, which seems incorrect @@ -414,7 +434,7 @@ void *xtensa_excint1_c(int *interrupted_stack) * up. */ z_xtensa_fatal_error(reason, - (void *)interrupted_stack); + (void *)print_stack); break; } diff --git a/include/zephyr/arch/xtensa/arch.h b/include/zephyr/arch/xtensa/arch.h index 16bfd08091a475..d486bb4d7b6212 100644 --- a/include/zephyr/arch/xtensa/arch.h +++ b/include/zephyr/arch/xtensa/arch.h @@ -54,6 +54,7 @@ struct arch_mem_domain { }; extern void xtensa_arch_except(int reason_p); +extern void xtensa_arch_kernel_oops(int reason_p, void *ssf); #ifdef CONFIG_USERSPACE From 5051923ffaeb8351cfe23489da0ff9934b2d33c5 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Thu, 16 Mar 2023 20:27:47 -0700 Subject: [PATCH 15/33] xtensa: mmu: send IPI to invalidate TLBs on other CPUs After changing content of page table(s), it is needed to notify the other CPUs that the page table(s) have been changed so they can do the necessary steps to use the updated version. Note that the actual way to send IPI is SoC specific as Xtensa does not have a common way to do this at the moment. Signed-off-by: Daniel Leung --- arch/xtensa/core/xtensa_mmu.c | 79 +++++++++++++++++++++++++ include/zephyr/arch/xtensa/xtensa_mmu.h | 20 +++++++ 2 files changed, 99 insertions(+) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 915a26357eb894..8a2dcab7ee8bda 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -659,6 +659,10 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) pa += KB(4); } +#if CONFIG_MP_MAX_NUM_CPUS > 1 + z_xtensa_mmu_tlb_ipi(); +#endif + k_spin_unlock(&xtensa_mmu_lock, key); } @@ -791,9 +795,80 @@ void arch_mem_unmap(void *addr, size_t size) va += KB(4); } +#if CONFIG_MP_MAX_NUM_CPUS > 1 + z_xtensa_mmu_tlb_ipi(); +#endif + k_spin_unlock(&xtensa_mmu_lock, key); } +/* This should be implemented in the SoC layer. + * This weak version is here to avoid build errors. + */ +void __weak z_xtensa_mmu_tlb_ipi(void) +{ +} + +void z_xtensa_mmu_tlb_shootdown(void) +{ + unsigned int key; + + /* Need to lock interrupts to prevent any context + * switching until all the page tables are updated. + * Or else we would be switching to another thread + * and running that with incorrect page tables + * which would result in permission issues. + */ + key = arch_irq_lock(); + + /* We don't have information on which page tables have changed, + * so we just invalidate the cache for all L1 page tables. + */ + sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table)); + sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + +#ifdef CONFIG_USERSPACE + struct k_thread *thread = _current_cpu->current; + + /* If current thread is a user thread, we need to see if it has + * been migrated to another memory domain as the L1 page table + * is different from the currently used one. + */ + if ((thread->base.user_options & K_USER) == K_USER) { + uint32_t ptevaddr_entry, ptevaddr, thread_ptables; + + /* Need to read the currently used L1 page table. + * We know that L1 page table is always mapped at way + * MMU_PTE_WAY, so we can skip the probing step by + * generating the query entry directly. + */ + ptevaddr_entry = Z_XTENSA_PAGE_TABLE_VADDR | MMU_PTE_WAY; + ptevaddr = xtensa_dtlb_paddr_read(ptevaddr_entry); + + thread_ptables = (uint32_t)thread->arch.ptables; + + if (thread_ptables != ptevaddr) { + /* Need to remap the thread page tables if the ones + * indicated by the current thread are different + * than the current mapped page table. + */ + switch_page_tables((uint32_t *)thread_ptables, false, false); + } + + } +#endif /* CONFIG_USERSPACE */ + + /* L2 are done via autofill, so invalidate autofill TLBs + * would refresh the L2 page tables. + * + * L1 will be refreshed during context switch so no need + * to do anything here. + */ + xtensa_tlb_autorefill_invalidate(); + + arch_irq_unlock(key); +} + #ifdef CONFIG_USERSPACE static inline uint32_t *alloc_l1_table(void) @@ -951,6 +1026,10 @@ static inline int update_region(uint32_t *ptables, uintptr_t start, ret = region_map_update(ptables, start, size, ring, flags); #endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ +#if CONFIG_MP_MAX_NUM_CPUS > 1 + z_xtensa_mmu_tlb_ipi(); +#endif + k_spin_unlock(&xtensa_mmu_lock, key); return ret; diff --git a/include/zephyr/arch/xtensa/xtensa_mmu.h b/include/zephyr/arch/xtensa/xtensa_mmu.h index 40fd0ff8f5c114..aa00f935a8a4f0 100644 --- a/include/zephyr/arch/xtensa/xtensa_mmu.h +++ b/include/zephyr/arch/xtensa/xtensa_mmu.h @@ -60,4 +60,24 @@ extern int xtensa_soc_mmu_ranges_num; void z_xtensa_mmu_init(void); +/** + * @brief Tell other processors to flush TLBs. + * + * This sends IPI to other processors to telling them to + * invalidate cache to page tables and flush TLBs. This is + * needed when one processor is updating page tables that + * may affect threads running on other processors. + * + * @note This needs to be implemented in the SoC layer. + */ +void z_xtensa_mmu_tlb_ipi(void); + +/** + * @brief Invalidate cache to page tables and flush TLBs. + * + * This invalidates cache to all page tables and flush TLBs + * as they may have been modified by other processors. + */ +void z_xtensa_mmu_tlb_shootdown(void); + #endif /* ZEPHYR_INCLUDE_ARCH_XTENSA_XTENSA_MMU_H */ From c24a09bbdb84f893be8b3bdc55f01bfeaee79a82 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 27 Mar 2023 16:27:34 -0700 Subject: [PATCH 16/33] xtensa: mmu: cache and TLB actions when adding thread to domain When adding a thread to a memory domain, we need to also update the mapped page table if it is the current running thread on the same CPU. If it's not on the same CPU, we need to notify the other CPUs in case the thread is running in one of them. Signed-off-by: Daniel Leung Signed-off-by: Anas Nashif Signed-off-by: Flavio Ceolin --- arch/xtensa/core/xtensa_mmu.c | 80 +++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 27 deletions(-) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 8a2dcab7ee8bda..ef0020a76c657d 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -16,6 +16,12 @@ #include #include +/* Skip TLB IPI when updating page tables. + * This allows us to send IPI only after the last + * changes of a series. + */ +#define OPTION_NO_TLB_IPI BIT(0) + /* Level 1 contains page table entries * necessary to map the page table itself. */ @@ -995,7 +1001,8 @@ static int region_map_update(uint32_t *ptables, uintptr_t start, } static inline int update_region(uint32_t *ptables, uintptr_t start, - size_t size, uint32_t ring, uint32_t flags) + size_t size, uint32_t ring, uint32_t flags, + uint32_t option) { int ret; k_spinlock_key_t key; @@ -1027,7 +1034,9 @@ static inline int update_region(uint32_t *ptables, uintptr_t start, #endif /* CONFIG_XTENSA_MMU_DOUBLE_MAP */ #if CONFIG_MP_MAX_NUM_CPUS > 1 - z_xtensa_mmu_tlb_ipi(); + if ((option & OPTION_NO_TLB_IPI) != OPTION_NO_TLB_IPI) { + z_xtensa_mmu_tlb_ipi(); + } #endif k_spin_unlock(&xtensa_mmu_lock, key); @@ -1035,20 +1044,9 @@ static inline int update_region(uint32_t *ptables, uintptr_t start, return ret; } -static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size) +static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size, uint32_t option) { - return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W); -} - -void xtensa_set_stack_perms(struct k_thread *thread) -{ - if ((thread->base.user_options & K_USER) == 0) { - return; - } - - update_region(thread_page_tables_get(thread), - thread->stack_info.start, thread->stack_info.size, - Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); + return update_region(ptables, start, size, Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_W, option); } void xtensa_user_stack_perms(struct k_thread *thread) @@ -1058,7 +1056,7 @@ void xtensa_user_stack_perms(struct k_thread *thread) update_region(thread_page_tables_get(thread), thread->stack_info.start, thread->stack_info.size, - Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB); + Z_XTENSA_USER_RING, Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, 0); } int arch_mem_domain_max_partitions_get(void) @@ -1073,7 +1071,7 @@ int arch_mem_domain_partition_remove(struct k_mem_domain *domain, /* Reset the partition's region back to defaults */ return reset_region(domain->arch.ptables, partition->start, - partition->size); + partition->size, 0); } int arch_mem_domain_partition_add(struct k_mem_domain *domain, @@ -1083,7 +1081,7 @@ int arch_mem_domain_partition_add(struct k_mem_domain *domain, struct k_mem_partition *partition = &domain->partitions[partition_id]; return update_region(domain->arch.ptables, partition->start, - partition->size, ring, partition->attr); + partition->size, ring, partition->attr, 0); } /* These APIs don't need to do anything */ @@ -1101,19 +1099,42 @@ int arch_mem_domain_thread_add(struct k_thread *thread) is_user = (thread->base.user_options & K_USER) != 0; is_migration = (old_ptables != NULL) && is_user; - /* Give access to the thread's stack in its new - * memory domain if it is migrating. - */ - if (is_migration) { - xtensa_set_stack_perms(thread); - } - if (is_migration) { + /* Give access to the thread's stack in its new + * memory domain if it is migrating. + */ + update_region(thread_page_tables_get(thread), + thread->stack_info.start, thread->stack_info.size, + Z_XTENSA_USER_RING, + Z_XTENSA_MMU_W | Z_XTENSA_MMU_CACHED_WB, + OPTION_NO_TLB_IPI); + /* and reset thread's stack permission in + * the old page tables. + */ ret = reset_region(old_ptables, thread->stack_info.start, - thread->stack_info.size); + thread->stack_info.size, 0); + } + + /* Need to switch to new page tables if this is + * the current thread running. + */ + if (thread == _current_cpu->current) { + switch_page_tables(thread->arch.ptables, true, true); } +#if CONFIG_MP_MAX_NUM_CPUS > 1 + /* Need to tell other CPUs to switch to the new page table + * in case the thread is running on one of them. + * + * Note that there is no need to send TLB IPI if this is + * migration as it was sent above during reset_region(). + */ + if ((thread != _current_cpu->current) && !is_migration) { + z_xtensa_mmu_tlb_ipi(); + } +#endif + return ret; } @@ -1136,10 +1157,15 @@ int arch_mem_domain_thread_remove(struct k_thread *thread) /* Restore permissions on the thread's stack area since it is no * longer a member of the domain. + * + * Note that, since every thread must have an associated memory + * domain, removing a thread from domain will be followed by + * adding it back to another. So there is no need to send TLB IPI + * at this point. */ return reset_region(domain->arch.ptables, thread->stack_info.start, - thread->stack_info.size); + thread->stack_info.size, OPTION_NO_TLB_IPI); } static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool write) From fc117a5b2bd0309418700d3d3e69fe46891a52de Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 6 Sep 2023 11:26:41 -0700 Subject: [PATCH 17/33] xtensa: userspace: swap page tables at context restore Swap page tables at exit of exception handler if we are going to be restored to another thread context. Or else we would be using the outgoing thread's page tables which is not going to work correctly due to mapping and permissions. Signed-off-by: Daniel Leung --- arch/xtensa/include/xtensa-asm2-s.h | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index 8c08916c8c6521..3f3ffd90b7ae04 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -507,6 +507,8 @@ _do_call_\@: * spills to the right place. */ beq a6, a1, _restore_\@ + +#ifndef CONFIG_USERSPACE l32i a1, a1, 0 l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF @@ -516,7 +518,37 @@ _do_call_\@: */ SPILL_ALL_WINDOWS #endif + + /* Restore A1 stack pointer from "next" handle. */ mov a1, a6 +#else + /* With userspace, we cannot simply restore A1 stack pointer + * at this pointer because we need to swap page tables to + * the incoming thread, and we do not want to call that + * function with thread's stack. So we stash the new stack + * pointer into A2 first, then move it to A1 after we have + * swapped the page table. + */ + mov a2, a6 + + /* Need to switch page tables because the "next" handle + * returned above is not the same handle as we started + * with. This means we are being restored to another + * thread. + */ + rsr a6, ZSR_CPU + l32i a6, a6, ___cpu_t_current_OFFSET + + call4 z_xtensa_swap_update_page_tables + l32i a1, a1, 0 + l32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET + addi a1, a1, ___xtensa_irq_bsa_t_SIZEOF + + SPILL_ALL_WINDOWS + + /* Moved stashed stack pointer to A1 to restore stack. */ + mov a1, a2 +#endif _restore_\@: j _restore_context From 9645a2c6939ac32cad3134d7d1788ac085f724b1 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Thu, 19 Jan 2023 10:05:43 -0800 Subject: [PATCH 18/33] tests: userspace: Add xtensa support This test requires architecture specific code to work. Signed-off-by: Flavio Ceolin Signed-off-by: Daniel Leung --- tests/kernel/mem_protect/userspace/src/main.c | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/kernel/mem_protect/userspace/src/main.c b/tests/kernel/mem_protect/userspace/src/main.c index 1c0e7d3ef21010..bcf504102db87c 100644 --- a/tests/kernel/mem_protect/userspace/src/main.c +++ b/tests/kernel/mem_protect/userspace/src/main.c @@ -20,6 +20,11 @@ #include "test_syscall.h" #include /* for z_libc_partition */ +#if defined(CONFIG_XTENSA) +#include +#include +#endif + #if defined(CONFIG_ARC) #include #endif @@ -178,6 +183,12 @@ ZTEST_USER(userspace, test_write_control) set_fault(K_ERR_CPU_EXCEPTION); __asm__ volatile("csrr %0, mstatus" : "=r" (status)); +#elif defined(CONFIG_XTENSA) + unsigned int ps; + + set_fault(K_ERR_CPU_EXCEPTION); + + __asm__ volatile("rsr.ps %0" : "=r" (ps)); #else #error "Not implemented for this architecture" zassert_unreachable("Write to control register did not fault"); @@ -245,6 +256,24 @@ ZTEST_USER(userspace, test_disable_mmu_mpu) */ csr_write(pmpaddr3, LLONG_MAX); csr_write(pmpcfg0, (PMP_R|PMP_W|PMP_X|PMP_NAPOT) << 24); +#elif defined(CONFIG_XTENSA) + set_fault(K_ERR_CPU_EXCEPTION); + + /* Reset way 6 to do identity mapping. + * Complier would complain addr going out of range if we + * simply do addr = i * 0x20000000 inside the loop. So + * we do increment instead. + */ + uint32_t addr = 0U; + + for (int i = 0; i < 8; i++) { + uint32_t attr = addr | Z_XTENSA_MMU_XW; + + __asm__ volatile("wdtlb %0, %1; witlb %0, %1" + :: "r"(attr), "r"(addr)); + + addr += 0x20000000; + } #else #error "Not implemented for this architecture" #endif @@ -381,7 +410,8 @@ ZTEST_USER(userspace, test_read_priv_stack) s[0] = 0; priv_stack_ptr = (char *)&s[0] - size; -#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || defined(CONFIG_ARM64) +#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || \ + defined(CONFIG_ARM64) || defined(CONFIG_XTENSA) /* priv_stack_ptr set by test_main() */ #else #error "Not implemented for this architecture" @@ -405,7 +435,8 @@ ZTEST_USER(userspace, test_write_priv_stack) s[0] = 0; priv_stack_ptr = (char *)&s[0] - size; -#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || defined(CONFIG_ARM64) +#elif defined(CONFIG_ARM) || defined(CONFIG_X86) || defined(CONFIG_RISCV) || \ + defined(CONFIG_ARM64) || defined(CONFIG_XTENSA) /* priv_stack_ptr set by test_main() */ #else #error "Not implemented for this architecture" From a22cb1b14cf949519a201e9d449a0d38ad2c79d7 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 31 Jan 2023 15:35:15 -0800 Subject: [PATCH 19/33] tests: mem_protect: enable for Xtensa This enables tests/kernel/mem_protect/mem_protect to be tested on Xtensa. Signed-off-by: Daniel Leung Signed-off-by: Flavio Ceolin --- tests/kernel/mem_protect/mem_protect/src/mem_protect.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/kernel/mem_protect/mem_protect/src/mem_protect.h b/tests/kernel/mem_protect/mem_protect/src/mem_protect.h index ef0ba48a519048..4d5400bb1d6bbf 100644 --- a/tests/kernel/mem_protect/mem_protect/src/mem_protect.h +++ b/tests/kernel/mem_protect/mem_protect/src/mem_protect.h @@ -67,6 +67,8 @@ static inline void set_fault_valid(bool valid) #else #define MEM_REGION_ALLOC (4) #endif +#elif defined(CONFIG_XTENSA) +#define MEM_REGION_ALLOC (4096) #else #error "Test suite not compatible for the given architecture" #endif From 9b0c1e6e023ed730d8f8d3c40ed234ba47e9b096 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Fri, 2 Jun 2023 22:36:14 -0700 Subject: [PATCH 20/33] xtensa: mmu: Flush cache when altering pages When the target has a cache way size (cache size / cache wasy) bigger than the page size we have cache aliasing, since the number of bits required by the cache index is bigger than the number of bits in the page offset. To avoid this problem we flush the whole cache on context switch or when the current page table is changed. Signed-off-by: Flavio Ceolin --- arch/xtensa/core/xtensa_mmu.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index ef0020a76c657d..74ced4242109b6 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -231,8 +231,7 @@ static inline uint32_t *alloc_l2_table(void) static ALWAYS_INLINE void switch_page_tables(uint32_t *ptables, bool dtlb_inv, bool cache_inv) { if (cache_inv) { - sys_cache_data_invd_range((void *)ptables, XTENSA_L1_PAGE_TABLE_SIZE); - sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + sys_cache_data_flush_and_invd_all(); } /* Invalidate data TLB to L1 page table */ @@ -669,6 +668,7 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) z_xtensa_mmu_tlb_ipi(); #endif + sys_cache_data_flush_and_invd_all(); k_spin_unlock(&xtensa_mmu_lock, key); } @@ -805,6 +805,7 @@ void arch_mem_unmap(void *addr, size_t size) z_xtensa_mmu_tlb_ipi(); #endif + sys_cache_data_flush_and_invd_all(); k_spin_unlock(&xtensa_mmu_lock, key); } @@ -858,7 +859,7 @@ void z_xtensa_mmu_tlb_shootdown(void) * indicated by the current thread are different * than the current mapped page table. */ - switch_page_tables((uint32_t *)thread_ptables, false, false); + switch_page_tables((uint32_t *)thread_ptables, true, true); } } @@ -1039,6 +1040,7 @@ static inline int update_region(uint32_t *ptables, uintptr_t start, } #endif + sys_cache_data_flush_and_invd_all(); k_spin_unlock(&xtensa_mmu_lock, key); return ret; From 281f7173250158842400ec0c00fcc56b67df2ec8 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Tue, 29 Aug 2023 10:08:27 -0700 Subject: [PATCH 21/33] xtensa: mmu: Fix possible race condition on tlb shootdown We need to use the mmu spin lock when invalidating the cache during tlb shootdown, otherwise it is possible that this happens when another thread is updating the page tables. Signed-off-by: Flavio Ceolin Signed-off-by: Anas Nashif --- arch/xtensa/core/xtensa_mmu.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 74ced4242109b6..1c120d5eb31a41 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -828,11 +828,13 @@ void z_xtensa_mmu_tlb_shootdown(void) */ key = arch_irq_lock(); - /* We don't have information on which page tables have changed, - * so we just invalidate the cache for all L1 page tables. - */ - sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table)); - sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + K_SPINLOCK(&xtensa_mmu_lock) { + /* We don't have information on which page tables have changed, + * so we just invalidate the cache for all L1 page tables. + */ + sys_cache_data_invd_range((void *)l1_page_table, sizeof(l1_page_table)); + sys_cache_data_invd_range((void *)l2_page_tables, sizeof(l2_page_tables)); + } #ifdef CONFIG_USERSPACE struct k_thread *thread = _current_cpu->current; From c6a8e99fd573f39e27bb5aa2df37eb6a8c44b7c8 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 29 Aug 2023 10:49:56 -0700 Subject: [PATCH 22/33] xtensa: mmu: do not map heap if not using heap Do not map the heap area by default if we are not using heap at all. Signed-off-by: Daniel Leung --- arch/xtensa/core/xtensa_mmu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index 1c120d5eb31a41..eba89c970ad81c 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -133,6 +133,7 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { #endif .name = "data", }, +#if CONFIG_HEAP_MEM_POOL_SIZE > 0 /* System heap memory */ { .start = (uint32_t)_heap_start, @@ -144,6 +145,7 @@ static const struct xtensa_mmu_range mmu_zephyr_ranges[] = { #endif .name = "heap", }, +#endif /* Mark text segment cacheable, read only and executable */ { .start = (uint32_t)__text_region_start, From e5c9e7d0e426ea685ff0c0f2215d23870a411215 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Tue, 29 Aug 2023 12:07:19 -0700 Subject: [PATCH 23/33] xtensa: userspace: simplify syscall helper Consolidate all syscall helpers into one functions. Signed-off-by: Daniel Leung --- arch/xtensa/core/syscall_helper.c | 103 ++------------------------- include/zephyr/arch/xtensa/syscall.h | 62 ++++++---------- 2 files changed, 27 insertions(+), 138 deletions(-) diff --git a/arch/xtensa/core/syscall_helper.c b/arch/xtensa/core/syscall_helper.c index fbbdf1041910d5..f8fb7ec903ec3e 100644 --- a/arch/xtensa/core/syscall_helper.c +++ b/arch/xtensa/core/syscall_helper.c @@ -6,10 +6,10 @@ #include -uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t arg5, uintptr_t arg6, - uintptr_t call_id) +uintptr_t xtensa_syscall_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id) { register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -27,98 +27,3 @@ uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, return a2; } - -uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t arg5, uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - register uintptr_t a6 __asm__("%a6") = arg1; - register uintptr_t a3 __asm__("%a3") = arg2; - register uintptr_t a4 __asm__("%a4") = arg3; - register uintptr_t a5 __asm__("%a5") = arg4; - register uintptr_t a8 __asm__("%a8") = arg5; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2), "r" (a6), "r" (a3), "r" (a4), - "r" (a5), "r" (a8) - : "memory"); - - return a2; -} - -uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - register uintptr_t a6 __asm__("%a6") = arg1; - register uintptr_t a3 __asm__("%a3") = arg2; - register uintptr_t a4 __asm__("%a4") = arg3; - register uintptr_t a5 __asm__("%a5") = arg4; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2), "r" (a6), "r" (a3), "r" (a4), - "r" (a5) - : "memory"); - - return a2; -} - -uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - register uintptr_t a6 __asm__("%a6") = arg1; - register uintptr_t a3 __asm__("%a3") = arg2; - register uintptr_t a4 __asm__("%a4") = arg3; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2), "r" (a6), "r" (a3), "r" (a4) - : "memory"); - - return a2; -} - -uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - register uintptr_t a6 __asm__("%a6") = arg1; - register uintptr_t a3 __asm__("%a3") = arg2; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2), "r" (a6), "r" (a3) - : "memory"); - - return a2; -} - -uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - register uintptr_t a6 __asm__("%a6") = arg1; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2), "r" (a6) - : "memory"); - - return a2; -} - -uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id) -{ - register uintptr_t a2 __asm__("%a2") = call_id; - - __asm__ volatile("syscall\n\t" - : "=r" (a2) - : "r" (a2) - : "memory"); - - return a2; -} diff --git a/include/zephyr/arch/xtensa/syscall.h b/include/zephyr/arch/xtensa/syscall.h index 3d78827b77c7cc..70a075db454328 100644 --- a/include/zephyr/arch/xtensa/syscall.h +++ b/include/zephyr/arch/xtensa/syscall.h @@ -29,29 +29,14 @@ extern "C" { #endif #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER -uintptr_t arch_syscall_invoke6_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t arg5, uintptr_t arg6, - uintptr_t call_id); +uintptr_t xtensa_syscall_helper(uintptr_t arg1, uintptr_t arg2, + uintptr_t arg3, uintptr_t arg4, + uintptr_t arg5, uintptr_t arg6, + uintptr_t call_id); -uintptr_t arch_syscall_invoke5_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t arg5, - uintptr_t call_id); - -uintptr_t arch_syscall_invoke4_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t arg4, - uintptr_t call_id); - -uintptr_t arch_syscall_invoke3_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t arg3, uintptr_t call_id); - -uintptr_t arch_syscall_invoke2_helper(uintptr_t arg1, uintptr_t arg2, - uintptr_t call_id); - -uintptr_t arch_syscall_invoke1_helper(uintptr_t arg1, uintptr_t call_id); - -uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id); +#define SYSINL ALWAYS_INLINE +#else +#define SYSINL inline #endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ /** @@ -63,15 +48,13 @@ uintptr_t arch_syscall_invoke0_helper(uintptr_t call_id); * **/ -static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, +static SYSINL uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, uintptr_t arg6, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke6_helper(arg1, arg2, arg3, - arg4, arg5, arg6, - call_id); + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, arg5, arg6, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -91,13 +74,12 @@ static inline uintptr_t arch_syscall_invoke6(uintptr_t arg1, uintptr_t arg2, #endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ } -static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, +static SYSINL uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke5_helper(arg1, arg2, arg3, - arg4, arg5, call_id); + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, arg5, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -116,12 +98,12 @@ static inline uintptr_t arch_syscall_invoke5(uintptr_t arg1, uintptr_t arg2, #endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ } -static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, +static SYSINL uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke4_helper(arg1, arg2, arg3, arg4, call_id); + return xtensa_syscall_helper(arg1, arg2, arg3, arg4, 0, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -139,11 +121,11 @@ static inline uintptr_t arch_syscall_invoke4(uintptr_t arg1, uintptr_t arg2, #endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ } -static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, +static SYSINL uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke3_helper(arg1, arg2, arg3, call_id); + return xtensa_syscall_helper(arg1, arg2, arg3, 0, 0, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -159,11 +141,11 @@ static inline uintptr_t arch_syscall_invoke3(uintptr_t arg1, uintptr_t arg2, #endif /* CONFIG_XTENSA_SYSCALL_USE_HELPER */ } -static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, +static SYSINL uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke2_helper(arg1, arg2, call_id); + return xtensa_syscall_helper(arg1, arg2, 0, 0, 0, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -178,11 +160,11 @@ static inline uintptr_t arch_syscall_invoke2(uintptr_t arg1, uintptr_t arg2, #endif } -static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1, +static SYSINL uintptr_t arch_syscall_invoke1(uintptr_t arg1, uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke1_helper(arg1, call_id); + return xtensa_syscall_helper(arg1, 0, 0, 0, 0, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; register uintptr_t a6 __asm__("%a6") = arg1; @@ -196,10 +178,10 @@ static inline uintptr_t arch_syscall_invoke1(uintptr_t arg1, #endif } -static inline uintptr_t arch_syscall_invoke0(uintptr_t call_id) +static SYSINL uintptr_t arch_syscall_invoke0(uintptr_t call_id) { #ifdef CONFIG_XTENSA_SYSCALL_USE_HELPER - return arch_syscall_invoke0_helper(call_id); + return xtensa_syscall_helper(0, 0, 0, 0, 0, 0, call_id); #else register uintptr_t a2 __asm__("%a2") = call_id; @@ -229,6 +211,8 @@ static inline bool arch_is_user_context(void) return !!thread; } +#undef SYSINL + #ifdef __cplusplus } #endif From 21c0c5f9e3f81a01983d0e548478fedc7296a4cf Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 11 Sep 2023 16:25:22 -0700 Subject: [PATCH 24/33] xtensa: selectively init interrupt stack at boot During arch_kernel_init(), the interrupt stack is being initialized. However, if the current in-use stack is the interrupt stack, it would wipe all the data up to that point in stack, and might result in crash. So skip initializing the interrupt stack if the current stack pointer is within the boundary of interrupt stack. Signed-off-by: Daniel Leung --- arch/xtensa/include/kernel_arch_func.h | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/include/kernel_arch_func.h b/arch/xtensa/include/kernel_arch_func.h index 8ce5cc52a5b51a..080b81d72bf952 100644 --- a/arch/xtensa/include/kernel_arch_func.h +++ b/arch/xtensa/include/kernel_arch_func.h @@ -53,8 +53,22 @@ static ALWAYS_INLINE void arch_kernel_init(void) XTENSA_WSR(ZSR_CPU_STR, cpu0); #ifdef CONFIG_INIT_STACKS - memset(Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[0]), 0xAA, - K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[0])); + char *stack_start = Z_KERNEL_STACK_BUFFER(z_interrupt_stacks[0]); + size_t stack_sz = K_KERNEL_STACK_SIZEOF(z_interrupt_stacks[0]); + char *stack_end = stack_start + stack_sz; + + uint32_t sp; + + __asm__ volatile("mov %0, sp" : "=a"(sp)); + + /* Only clear the interrupt stack if the current stack pointer + * is not within the interrupt stack. Or else we would be + * wiping the in-use stack. + */ + if (((uintptr_t)sp < (uintptr_t)stack_start) || + ((uintptr_t)sp >= (uintptr_t)stack_end)) { + memset(stack_start, 0xAA, stack_sz); + } #endif #ifdef CONFIG_XTENSA_MMU From 3771eaa15f36fdee82154aa193a0a2ef4a8cef09 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Wed, 30 Aug 2023 15:36:21 -0700 Subject: [PATCH 25/33] xtensa: userspace: only write 0xAA to stack if INIT_STACKS Only clear the user stack to 0xAA if CONFIG_INIT_STACKS is enabled. Otherwise, write 0x00 as if the stack is in BSS. Signed-off-by: Daniel Leung --- arch/xtensa/core/xtensa_mmu.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/xtensa_mmu.c index eba89c970ad81c..2130a870e7757d 100644 --- a/arch/xtensa/core/xtensa_mmu.c +++ b/arch/xtensa/core/xtensa_mmu.c @@ -1057,8 +1057,9 @@ static inline int reset_region(uint32_t *ptables, uintptr_t start, size_t size, void xtensa_user_stack_perms(struct k_thread *thread) { - (void)memset((void *)thread->stack_info.start, 0xAA, - thread->stack_info.size - thread->stack_info.delta); + (void)memset((void *)thread->stack_info.start, + (IS_ENABLED(CONFIG_INIT_STACKS)) ? 0xAA : 0x00, + thread->stack_info.size - thread->stack_info.delta); update_region(thread_page_tables_get(thread), thread->stack_info.start, thread->stack_info.size, From 0fe2d7261fc9420667cb7094a43a03eac20ffc23 Mon Sep 17 00:00:00 2001 From: Daniel Leung Date: Mon, 28 Aug 2023 17:01:16 -0700 Subject: [PATCH 26/33] xtensa: dc233c: enable userspace support This massages kconfig and linker script to enable userspace support on dc233c core. Signed-off-by: Daniel Leung --- soc/xtensa/dc233c/Kconfig.soc | 1 + soc/xtensa/dc233c/include/xtensa-dc233c.ld | 94 +++++++++++++--------- soc/xtensa/dc233c/mmu.c | 7 +- 3 files changed, 61 insertions(+), 41 deletions(-) diff --git a/soc/xtensa/dc233c/Kconfig.soc b/soc/xtensa/dc233c/Kconfig.soc index 89b814da4d2bd9..617eaa13849b16 100644 --- a/soc/xtensa/dc233c/Kconfig.soc +++ b/soc/xtensa/dc233c/Kconfig.soc @@ -9,3 +9,4 @@ config SOC_XTENSA_DC233C select ARCH_HAS_THREAD_LOCAL_STORAGE select CPU_HAS_MMU select ARCH_HAS_RESERVED_PAGE_FRAMES if XTENSA_MMU + select ARCH_HAS_USERSPACE if XTENSA_MMU diff --git a/soc/xtensa/dc233c/include/xtensa-dc233c.ld b/soc/xtensa/dc233c/include/xtensa-dc233c.ld index b165016fa6a1da..2153c4a420cef8 100644 --- a/soc/xtensa/dc233c/include/xtensa-dc233c.ld +++ b/soc/xtensa/dc233c/include/xtensa-dc233c.ld @@ -20,7 +20,7 @@ #include #define RAMABLE_REGION RAM :sram0_phdr -#define ROMABLE_REGION rom0_seg :rom0_phdr +#define ROMABLE_REGION RAM :sram0_phdr #ifdef CONFIG_MMU #define MMU_PAGE_ALIGN . = ALIGN(CONFIG_MMU_PAGE_SIZE); @@ -287,6 +287,9 @@ SECTIONS _DoubleExceptionVector_text_end = ABSOLUTE(.); } >sram0_18_seg :sram0_18_phdr +#define LIB_OBJ_FUNC_IN_SECT(library, obj_file, func) \ + *##library##:##obj_file##(.literal.##func .text.##func) \ + #ifdef CONFIG_XTENSA_MMU .vec_helpers : { @@ -301,48 +304,45 @@ SECTIONS * TLB multi-hit exception. */ - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.literal) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.text) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram.text) - *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram0.text) + *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.literal .text) + *libarch__xtensa__core.a:xtensa-asm2-util.S.obj(.iram.text .iram0.text) *libarch__xtensa__core.a:window_vectors.S.obj(.iram.text) - *libarch__xtensa__core.a:xtensa-asm2.c.obj(.literal.*) - *libarch__xtensa__core.a:xtensa-asm2.c.obj(.text.*) - - *libarch__xtensa__core.a:fatal.c.obj(.literal.*) - *libarch__xtensa__core.a:fatal.c.obj(.text.*) - - *libarch__xtensa__core.a:crt1.S.obj(.literal) - *libarch__xtensa__core.a:crt1.S.obj(.text) + *libarch__xtensa__core.a:crt1.S.obj(.literal .text) - *libarch__xtensa__core.a:cpu_idle.c.obj(.literal.*) - *libarch__xtensa__core.a:cpu_idle.c.obj(.text.*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa-asm2.c.obj,*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,fatal.c.obj,*) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,cpu_idle.c.obj,*) *(.text.arch_is_in_isr) /* To support backtracing */ - *libarch__xtensa__core.a:xtensa_backtrace.c.obj(.literal.*) - *libarch__xtensa__core.a:xtensa_backtrace.c.obj(.text.*) - *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1.literal) - *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa_backtrace.c.obj,*) - *libkernel.a:fatal.c.obj(.literal.*) - *libkernel.a:fatal.c.obj(.text.*) + *libarch__xtensa__core.a:debug_helpers_asm.S.obj(.iram1.literal .iram1) + + /* Userspace related stuff */ + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,userspace.S.obj,z_xtensa_do_syscall) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa_mmu.c.obj,z_xtensa_swap_update_page_tables) /* Below are to speed up execution by avoiding TLB misses * on frequently used functions. + * + * There is almost 1MB space (due to TLB pinning) so we can + * be generous. */ - *libkernel.a:sched.c.obj(.literal.*) - *libkernel.a:sched.c.obj(.text.*) - *libkernel.a:timeout.c.obj(.literal.*) - *libkernel.a:timeout.c.obj(.text.*) - - *libdrivers__console.a:(.literal.*) - *libdrivers__console.a:(.text.*) - *libdrivers__timer.a:(.literal.*) - *libdrivers__timer.a:(.text.*) + LIB_OBJ_FUNC_IN_SECT(libkernel.a,,*) + + LIB_OBJ_FUNC_IN_SECT(libdrivers__console.a,,*) + LIB_OBJ_FUNC_IN_SECT(libdrivers__timer.a,,*) + + *(.literal.z_vrfy_* .text.z_vrfy_*) + *(.literal.z_mrsh_* .text.z_mrsh_*) + *(.literal.z_impl_* .text.z_impl_*) + *(.literal.z_obj_* .text.z_obj_*) + + *(.literal.k_sys_fatal_error_handler .text.k_sys_fatal_error_handler) } >vec_helpers :vec_helpers_phdr #endif /* CONFIG_XTENSA_MMU */ @@ -380,7 +380,7 @@ SECTIONS _text_end = ABSOLUTE(.); _etext = .; - } >RAM :sram0_phdr + } >RAMABLE_REGION __text_region_end = .; .rodata : HDR_MMU_PAGE_ALIGN @@ -394,7 +394,16 @@ SECTIONS . = ALIGN(4); #include #include + } >RAMABLE_REGION + +#include + +#include +#include + + .rodata_end : ALIGN(4) + { . = ALIGN(4); /* this table MUST be 4-byte aligned */ _bss_table_start = ABSOLUTE(.); LONG(_bss_start) @@ -404,19 +413,26 @@ SECTIONS MMU_PAGE_ALIGN __rodata_region_end = ABSOLUTE(.); - } >RAM :sram0_phdr + } >RAMABLE_REGION -#include +#ifdef CONFIG_USERSPACE +#define SMEM_PARTITION_ALIGN(size) MMU_PAGE_ALIGN +#define APP_SHARED_ALIGN MMU_PAGE_ALIGN -#include +#include -#include - -#include + _image_ram_start = _app_smem_start; + _app_smem_size = _app_smem_end - _app_smem_start; + _app_smem_num_words = _app_smem_size >> 2; + _app_smem_rom_start = LOADADDR(_APP_SMEM_SECTION_NAME); + _app_smem_num_words = _app_smem_size >> 2; +#endif /* CONFIG_USERSPACE */ .data : HDR_MMU_PAGE_ALIGN { +#ifndef CONFIG_USERSPACE _image_ram_start = ABSOLUTE(.); +#endif __data_start = ABSOLUTE(.); *(.data) *(.data.*) @@ -438,7 +454,9 @@ SECTIONS MMU_PAGE_ALIGN __data_end = ABSOLUTE(.); - } >RAM :sram0_phdr + } >RAMABLE_REGION + +#include #include diff --git a/soc/xtensa/dc233c/mmu.c b/soc/xtensa/dc233c/mmu.c index 4b8d6b154b84c6..718f79779ebd5c 100644 --- a/soc/xtensa/dc233c/mmu.c +++ b/soc/xtensa/dc233c/mmu.c @@ -18,7 +18,7 @@ const struct xtensa_mmu_range xtensa_soc_mmu_ranges[] = { { .start = (uint32_t)XCHAL_VECBASE_RESET_VADDR, .end = (uint32_t)CONFIG_SRAM_OFFSET, - .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB, + .attrs = Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WB | Z_XTENSA_MMU_MAP_SHARED, .name = "vecbase", }, { @@ -52,7 +52,8 @@ void arch_xtensa_mmu_post_init(bool is_core0) /* Map VECBASE permanently in instr TLB way 4 so we will always have * access to exception handlers. Each way 4 TLB covers 1MB (unless * ITLBCFG has been changed before this, which should not have - * happened). + * happened). Also this needs to be mapped as SHARED so both kernel + * and userspace can execute code here => same as .text. * * Note that we don't want to map the first 1MB in data TLB as * we want to keep page 0 (0x00000000) unmapped to catch null pointer @@ -60,7 +61,7 @@ void arch_xtensa_mmu_post_init(bool is_core0) */ vecbase = ROUND_DOWN(vecbase, MB(1)); xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, + Z_XTENSA_PTE(vecbase, Z_XTENSA_SHARED_RING, Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, 4)); } From 067abcab81e2800d410c039a59e4957764d4adba Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Thu, 7 Sep 2023 19:48:35 +0000 Subject: [PATCH 27/33] xtensa: userspace: Supports tls on userspace Use thread local storage to check whether or not a thread is running in user mode. This allows to use threadptr to properly support tls. Signed-off-by: Flavio Ceolin --- arch/xtensa/core/userspace.S | 31 +++++++++++++++++++++++++--- arch/xtensa/core/xtensa-asm2.c | 8 +++++++ include/zephyr/arch/xtensa/syscall.h | 9 ++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/arch/xtensa/core/userspace.S b/arch/xtensa/core/userspace.S index b649014fd52bec..1a000d75d57071 100644 --- a/arch/xtensa/core/userspace.S +++ b/arch/xtensa/core/userspace.S @@ -100,8 +100,17 @@ _id_ok: * we are in a interruption we don't want the system * thinking it is possibly running in user mode. */ +#ifdef CONFIG_THREAD_LOCAL_STORAGE + movi a0, is_user_mode@tpoff + rur.THREADPTR a3 + add a0, a3, a0 + + movi a3, 0 + s32i a3, a0, 0 +#else movi a0, 0 wur.THREADPTR a0 +#endif /* Set syscall parameters. We have an initial call4 to set up the * the stack and then a new call4 for the syscall function itself. @@ -157,9 +166,17 @@ _syscall_returned: wsr a3, SCOMPARE1 #endif +#ifdef CONFIG_THREAD_LOCAL_STORAGE + l32i a3, a1, ___xtensa_irq_bsa_t_threadptr_OFFSET + movi a0, is_user_mode@tpoff + add a0, a3, a0 + movi a3, 1 + s32i a3, a0, 0 +#else rsr a3, ZSR_CPU l32i a3, a3, ___cpu_t_current_OFFSET wur.THREADPTR a3 +#endif l32i a3, a1, ___xtensa_irq_bsa_t_ps_OFFSET wsr.ps a3 @@ -225,9 +242,17 @@ z_xtensa_userspace_enter: l32i a6, a1, 24 call4 z_xtensa_swap_update_page_tables - /* Set threadptr with the thread address, we are going to user mode. */ - l32i a0, a1, 24 - wur.THREADPTR a0 +#ifdef CONFIG_THREAD_LOCAL_STORAGE + rur.threadptr a3 + movi a0, is_user_mode@tpoff + add a0, a3, a0 + movi a3, 1 + s32i a3, a0, 0 +#else + rsr a3, ZSR_CPU + l32i a3, a3, ___cpu_t_current_OFFSET + wur.THREADPTR a3 +#endif /* Set now z_thread_entry parameters, we are simulating a call4 * call, so parameters start at a6, a7, ... diff --git a/arch/xtensa/core/xtensa-asm2.c b/arch/xtensa/core/xtensa-asm2.c index ed8f340f1bcb45..88783c178c6426 100644 --- a/arch/xtensa/core/xtensa-asm2.c +++ b/arch/xtensa/core/xtensa-asm2.c @@ -28,6 +28,14 @@ Z_EXC_DECLARE(z_xtensa_user_string_nlen); static const struct z_exc_handle exceptions[] = { Z_EXC_HANDLE(z_xtensa_user_string_nlen) }; + +#ifdef CONFIG_THREAD_LOCAL_STORAGE +/* + * Per-thread (TLS) variable indicating whether execution is in user mode. + */ +__thread uint32_t is_user_mode; +#endif + #endif /* CONFIG_USERSPACE */ void *xtensa_init_stack(struct k_thread *thread, int *stack_top, diff --git a/include/zephyr/arch/xtensa/syscall.h b/include/zephyr/arch/xtensa/syscall.h index 70a075db454328..b8b0bea8cdbce9 100644 --- a/include/zephyr/arch/xtensa/syscall.h +++ b/include/zephyr/arch/xtensa/syscall.h @@ -207,8 +207,17 @@ static inline bool arch_is_user_context(void) "rur.THREADPTR %0\n\t" : "=a" (thread) ); +#ifdef CONFIG_THREAD_LOCAL_STORAGE + extern __thread uint32_t is_user_mode; + if (!thread) { + return false; + } + + return is_user_mode != 0; +#else return !!thread; +#endif } #undef SYSINL From 9d929ec4df6e2ef35e125663f78226fcbd0f904f Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Mon, 18 Sep 2023 12:44:03 -0700 Subject: [PATCH 28/33] kernel: Option to not use tls to get current thread Add a Kconfig option to tell whether or not using thread local storage to store current thread. The function using it can be called from ISR and using TLS variables in this context may (should ???) not be allowed Signed-off-by: Flavio Ceolin --- include/zephyr/kernel.h | 3 ++- kernel/Kconfig | 14 ++++++++++++++ lib/os/thread_entry.c | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/include/zephyr/kernel.h b/include/zephyr/kernel.h index fac7df01904fb6..d2b912f6d46e13 100644 --- a/include/zephyr/kernel.h +++ b/include/zephyr/kernel.h @@ -596,7 +596,8 @@ __syscall k_tid_t k_sched_current_thread_query(void); __attribute_const__ static inline k_tid_t k_current_get(void) { -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS + /* Thread-local cache of current thread ID, set in z_thread_entry() */ extern __thread k_tid_t z_tls_current; diff --git a/kernel/Kconfig b/kernel/Kconfig index 6280e80a400b4b..4a72d76da67054 100644 --- a/kernel/Kconfig +++ b/kernel/Kconfig @@ -288,6 +288,20 @@ config ERRNO_IN_TLS Use thread local storage to store errno instead of storing it in the kernel thread struct. This avoids a syscall if userspace is enabled. +config CURRENT_THREAD_USE_NO_TLS + bool + help + Hidden symbol to not use thread local storage to store current + thread. + +config CURRENT_THREAD_USE_TLS + bool "Store current thread in thread local storage (TLS)" + depends on THREAD_LOCAL_STORAGE && !CURRENT_THREAD_USE_NO_TLS + default y + help + Use thread local storage to store the current thread. This avoids a + syscall if userspace is enabled. + choice SCHED_ALGORITHM prompt "Scheduler priority queue algorithm" default SCHED_DUMB diff --git a/lib/os/thread_entry.c b/lib/os/thread_entry.c index 89eb2fe7b4b32f..ed6ca142d99f7c 100644 --- a/lib/os/thread_entry.c +++ b/lib/os/thread_entry.c @@ -12,7 +12,7 @@ */ #include -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS #include __thread k_tid_t z_tls_current; @@ -35,7 +35,7 @@ extern __thread volatile uintptr_t __stack_chk_guard; FUNC_NORETURN void z_thread_entry(k_thread_entry_t entry, void *p1, void *p2, void *p3) { -#ifdef CONFIG_THREAD_LOCAL_STORAGE +#ifdef CONFIG_CURRENT_THREAD_USE_TLS z_tls_current = k_sched_current_thread_query(); #endif #ifdef CONFIG_STACK_CANARIES_TLS From c3ead7efecdd24dcf4fa4f321f1e0c784942a4c0 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Fri, 29 Sep 2023 21:33:45 -0700 Subject: [PATCH 29/33] arch: xtensa: Not use TLS to store current thread Xtensa clears out threadptr durint ISR when userspace is enabled. Signed-off-by: Flavio Ceolin --- arch/xtensa/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index c875aa3097230b..fefc77e2521173 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -116,6 +116,7 @@ config XTENSA_MMU select ARCH_MEM_DOMAIN_SYNCHRONOUS_API if USERSPACE select XTENSA_SMALL_VECTOR_TABLE_ENTRY select KERNEL_VM_USE_CUSTOM_MEM_RANGE_CHECK if XTENSA_RPO_CACHE + select CURRENT_THREAD_USE_NO_TLS if USERSPACE help Enable support for Xtensa Memory Management Unit. From 7d4ba0c2b2ce9aedd0e030e10cb5a3c018424e56 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Tue, 17 Oct 2023 15:50:31 -0700 Subject: [PATCH 30/33] xtensa: userspace: Warning about impl security Add a Kconfig option (and build warning) alerting about the problem of the kernel spilling register in behave of the userspace. Signed-off-by: Flavio Ceolin --- arch/xtensa/CMakeLists.txt | 6 ++++++ arch/xtensa/Kconfig | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/arch/xtensa/CMakeLists.txt b/arch/xtensa/CMakeLists.txt index 4626421f11a4ae..21de223d4ec201 100644 --- a/arch/xtensa/CMakeLists.txt +++ b/arch/xtensa/CMakeLists.txt @@ -3,3 +3,9 @@ zephyr_syscall_header(${ZEPHYR_BASE}/include/zephyr/arch/xtensa/arch.h) set_property(GLOBAL PROPERTY PROPERTY_OUTPUT_FORMAT elf32-xtensa-le) add_subdirectory(core) + +if (CONFIG_XTENSA_INSECURE_USERSPACE) + message(WARNING " + This userspace implementation uses the window ABI this means that the kernel + will spill registers in behave of the userpsace. Use it carefully.") +endif() diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig index fefc77e2521173..326c7749e6b20f 100644 --- a/arch/xtensa/Kconfig +++ b/arch/xtensa/Kconfig @@ -180,6 +180,11 @@ config XTENSA_SYSCALL_USE_HELPER This is a workaround for toolchains where they have issue modeling register usage. +config XTENSA_INSECURE_USERSPACE + bool + default y if USERSPACE + depends on XTENSA_MMU + endif # CPU_HAS_MMU endmenu From 9982d5ce335d05766b662a4bba10d335843e7b22 Mon Sep 17 00:00:00 2001 From: Andy Ross Date: Sun, 27 Aug 2023 12:32:57 -0700 Subject: [PATCH 31/33] arch/xtensa: #include cleanup This file doesn't need the asm2 header, and the preprocessor logic around whether to include the backtrace header is needless (all it does is declare functions). Signed-off-by: Andy Ross --- arch/xtensa/core/fatal.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/arch/xtensa/core/fatal.c b/arch/xtensa/core/fatal.c index 79a24baa105624..6e939e243cd3f6 100644 --- a/arch/xtensa/core/fatal.c +++ b/arch/xtensa/core/fatal.c @@ -8,12 +8,7 @@ #include #include #include -#include -#if defined(CONFIG_XTENSA_ENABLE_BACKTRACE) -#if XCHAL_HAVE_WINDOWED #include -#endif -#endif #include #include #include From b73b570edf5cce4062b8c317bd038412c6ac0bf1 Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Wed, 1 Nov 2023 16:07:02 -0700 Subject: [PATCH 32/33] arch: xtensa: Rename xtensa_mmu.c to ptables.c Initial work to split page table manipulation from mmu hardware interaction. Signed-off-by: Flavio Ceolin --- arch/xtensa/core/CMakeLists.txt | 2 +- arch/xtensa/core/{xtensa_mmu.c => ptables.c} | 0 soc/xtensa/dc233c/include/xtensa-dc233c.ld | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename arch/xtensa/core/{xtensa_mmu.c => ptables.c} (100%) diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index 331bcd9bfc52d3..1e4b045085ec5b 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -21,7 +21,7 @@ zephyr_library_sources_ifdef(CONFIG_XTENSA_ENABLE_BACKTRACE debug_helpers_asm.S) zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c) zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c) zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c) -zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU xtensa_mmu.c) +zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU ptables.c) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S) zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c) diff --git a/arch/xtensa/core/xtensa_mmu.c b/arch/xtensa/core/ptables.c similarity index 100% rename from arch/xtensa/core/xtensa_mmu.c rename to arch/xtensa/core/ptables.c diff --git a/soc/xtensa/dc233c/include/xtensa-dc233c.ld b/soc/xtensa/dc233c/include/xtensa-dc233c.ld index 2153c4a420cef8..9c120e1b9d8622 100644 --- a/soc/xtensa/dc233c/include/xtensa-dc233c.ld +++ b/soc/xtensa/dc233c/include/xtensa-dc233c.ld @@ -324,7 +324,7 @@ SECTIONS /* Userspace related stuff */ LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,userspace.S.obj,z_xtensa_do_syscall) - LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,xtensa_mmu.c.obj,z_xtensa_swap_update_page_tables) + LIB_OBJ_FUNC_IN_SECT(libarch__xtensa__core.a,ptables.c.obj,z_xtensa_swap_update_page_tables) /* Below are to speed up execution by avoiding TLB misses * on frequently used functions. From 6a93f230dde8996e44900caa72ea47b14cc73d0c Mon Sep 17 00:00:00 2001 From: Flavio Ceolin Date: Sun, 19 Nov 2023 15:44:56 -0800 Subject: [PATCH 33/33] arch/xtensa: Add new MMU layer Andy Ross re-implementation of MMU layer with some subtle changes, like re-using existent macros, fix page table cache property when direct mapping it in TLB. From Andy's original commit message: This is a reworked MMU layer, sitting cleanly below the page table handling in the OS. Notable differences from the original work: + Significantly smaller code and simpler API (just three functions to be called from the OS/userspace/ptable layer). + Big README-MMU document containing my learnings over the process, so hopefully fewer people need to go through this in the future. + No TLB flushing needed. Clean separation of ASIDs, just requires that the upper levels match the ASID to the L1 page table page consistently. + Vector mapping is done with a 4k page and not a 4M page, leading to much more flexibility with hardware memory layout. The original scheme required that the 4M region containing vecbase be mapped virtually to a location other than the hardware address, which makes confusing linkage with call0 and difficult initialization constraints where the exception vectors run at different addresses before and after MMU setup (effectively forcing them to be PIC code). + More provably correct initialization, all MMU changes happen in a single asm block with no memory accesses which would generate a refill. Signed-off-by: Andy Ross Signed-off-by: Flavio Ceolin --- arch/xtensa/core/CMakeLists.txt | 2 +- arch/xtensa/core/README-MMU.txt | 268 +++++++++++++++++++++ arch/xtensa/core/include/xtensa_mmu_priv.h | 15 +- arch/xtensa/core/mmu.c | 178 ++++++++++++++ arch/xtensa/core/ptables.c | 213 ++-------------- arch/xtensa/include/xtensa-asm2-s.h | 25 -- soc/xtensa/dc233c/mmu.c | 30 --- 7 files changed, 479 insertions(+), 252 deletions(-) create mode 100644 arch/xtensa/core/README-MMU.txt create mode 100644 arch/xtensa/core/mmu.c diff --git a/arch/xtensa/core/CMakeLists.txt b/arch/xtensa/core/CMakeLists.txt index 1e4b045085ec5b..f3122c1a5504da 100644 --- a/arch/xtensa/core/CMakeLists.txt +++ b/arch/xtensa/core/CMakeLists.txt @@ -21,7 +21,7 @@ zephyr_library_sources_ifdef(CONFIG_XTENSA_ENABLE_BACKTRACE debug_helpers_asm.S) zephyr_library_sources_ifdef(CONFIG_DEBUG_COREDUMP coredump.c) zephyr_library_sources_ifdef(CONFIG_TIMING_FUNCTIONS timing.c) zephyr_library_sources_ifdef(CONFIG_GDBSTUB gdbstub.c) -zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU ptables.c) +zephyr_library_sources_ifdef(CONFIG_XTENSA_MMU ptables.c mmu.c) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace.S) zephyr_library_sources_ifdef(CONFIG_XTENSA_SYSCALL_USE_HELPER syscall_helper.c) diff --git a/arch/xtensa/core/README-MMU.txt b/arch/xtensa/core/README-MMU.txt new file mode 100644 index 00000000000000..499a251cdf2f2f --- /dev/null +++ b/arch/xtensa/core/README-MMU.txt @@ -0,0 +1,268 @@ +# Xtensa MMU Operation + +As with other elements of the architecture, paged virtual memory +management on Xtensa is somewhat unique. And there is similarly a +lack of introductory material available. This document is an attempt +to introduce the architecture at an overview/tutorial level, and to +describe Zephyr's specific implementation choices. + +## General TLB Operation + +The Xtensa MMU operates on top of a fairly conventional TLB cache. +The TLB stores virtual to physical translation for individual pages of +memory. It is partitioned into an automatically managed +4-way-set-associative bank of entries mapping 4k pages, and 3-6 +"special" ways storing mappings under OS control. Some of these are +for mapping pages larger than 4k, which Zephyr does not directly +support. A few are for bootstrap and initialization, and will be +discussed below. + +Like the L1 cache, the TLB is split into separate instruction and data +entries. Zephyr manages both as needed, but symmetrically. The +architecture technically supports separately-virtualized instruction +and data spaces, but the hardware page table refill mechanism (see +below) does not, and Zephyr's memory spaces are unified regardless. + +The TLB may be loaded with permissions and attributes controlling +cacheability, access control based on ring (i.e. the contents of the +RING field of the PS register) and togglable write and execute access. +Memory access, even with a matching TLB entry, may therefore create +Kernel/User exceptions as desired to enforce permissions choices on +userspace code. + +Live TLB entries are tagged with an 8-bit "ASID" value derived from +their ring field of the PTE that loaded them, via a simple translation +specified in the RASID special register. The intent is that each +non-kernel address space will get a separate ring 3 ASID set in RASID, +such that you can switch between them without a TLB flush. The ASID +value of ring zero is fixed at 1, it may not be changed. (An ASID +value of zero is used to tag an invalid/unmapped TLB entry at +initialization, but this mechanism isn't accessible to OS code except +in special circumstances, and in any case there is already an invalid +attribute value that can be used in a PTE). + +## Virtually-mapped Page Tables + +Xtensa has a unique (and, to someone exposed for the first time, +extremely confusing) "page table" format. The simplest was to begin +to explain this is just to describe the (quite simple) hardware +behavior: + +On a TLB miss, the hardware immediately does a single fetch (at ring 0 +privilege) from RAM by adding the "desired address right shifted by +10 bits with the bottom two bits set to zero" (i.e. the page frame +number in units of 4 bytes) to the value in the PTEVADDR special +register. If this load succeeds, then the word is treated as a PTE +with which to fill the TLB and use for a (restarted) memory access. +This is extremely simple (just one extra hardware state that does just +one thing the hardware can already do), and quite fast (only one +memory fetch vs. e.g. the 2-5 fetches required to walk a page table on +x86). + +This special "refill" fetch is otherwise identical to any other memory +access, meaning it too uses the TLB to translate from a virtual to +physical address. Which means that the page tables occupy a 4M region +of virtual, not physical, address space, in the same memory space +occupied by the running code. The 1024 pages in that range (not all +of which might be mapped in physical memory) are a linear array of +1048576 4-byte PTE entries, each describing a mapping for 4k of +virtual memory. Note especially that exactly one of those pages +contains the 1024 PTE entries for the 4M page table itself, pointed to +by PTEVADDR. + +Obviously, the page table memory being virtual means that the fetch +can fail: there are 1024 possible pages in a complete page table +covering all of memory, and the ~16 entry TLB clearly won't contain +entries mapping all of them. If we are missing a TLB entry for the +page translation we want (NOT for the original requested address, we +already know we're missing that TLB entry), the hardware has exactly +one more special trick: it throws a TLB Miss exception (there are two, +one each for instruction/data TLBs, but in Zephyr they operate +identically). + +The job of that exception handler is simply to ensure that the TLB has +an entry for the page table page we want. And the simplest way to do +that is to just load the faulting PTE as an address, which will then +go through the same refill process above. This second TLB fetch in +the exception handler may result in an invalid/inapplicable mapping +within the 4M page table region. This is an typical/expected runtime +fault, and simply indicates unmapped memory. The result is TLB miss +exception from within the TLB miss exception handler (i.e. while the +EXCM bit is set). This will produce a Double Exception fault, which +is handled by the OS identically to a general Kernel/User data access +prohibited exception. + +After the TLB refill exception, the original faulting instruction is +restarted, which retries the refill process, which succeeds in +fetching a new TLB entry, which is then used to service the original +memory access. (And may then result in yet another exception if it +turns out that the TLB entry doesn't permit the access requested, of +course.) + +## Special Cases + +The page-tables-specified-in-virtual-memory trick works very well in +practice. But it does have a chicken/egg problem with the initial +state. Because everything depends on state in the TLB, something +needs to tell the hardware how to find a physical address using the +TLB to begin the process. Here we exploit the separate +non-automatically-refilled TLB ways to store bootstrap records. + +First, note that the refill process to load a PTE requires that the 4M +space of PTE entries be resolvable by the TLB directly, without +requiring another refill. This 4M mapping is provided by a single +page of PTE entries (which itself lives in the 4M page table region!). +This page must always be in the TLB. + +Thankfully, for the data TLB Xtensa provides 3 special/non-refillable +ways (ways 7-9) with at least one 4k page mapping each. We can use +one of these to "pin" the top-level page table entry in place, +ensuring that a refill access will be able to find a PTE address. + +But now note that the load from that PTE address for the refill is +done in an exception handler. And running an exception handler +requires doing a fetch via the instruction TLB. And that obviously +means that the page(s) containing the exception handler must never +require a refill exception of its own. + +Ideally we would just pin the vector/handler page in the ITLB in the +same way we do for data, but somewhat inexplicably, Xtensa does not +provide 4k "pinnable" ways in the instruction TLB (frankly this seems +like a design flaw). + +Instead, we load ITLB entries for vector handlers via the refill +mechanism using the data TLB, and so need the refill mechanism for the +vector page to succeed always. The way to do this is to similarly pin +the page table page containing the (single) PTE for the vector page in +the data TLB, such that instruction fetches always find their TLB +mapping via refill, without requiring an exception. + +## Initialization + +Unlike most other architectures, Xtensa does not have a "disable" mode +for the MMU. Virtual address translation through the TLB is active at +all times. There therefore needs to be a mechanism for the CPU to +execute code before the OS is able to initialize a refillable page +table. + +The way Xtensa resolves this (on the hardware Zephyr supports, see the +note below) is to have an 8-entry set ("way 6") of 512M pages able to +cover all of memory. These 8 entries are initialized as valid, with +attributes specifying that they are accessible only to an ASID of 1 +(i.e. the fixed ring zero / kernel ASID), writable, executable, and +uncached. So at boot the CPU relies on these TLB entries to provide a +clean view of hardware memory. + +But that means that enabling page-level translation requires some +care, as the CPU will throw an exception ("multi hit") if a memory +access matches more than one live entry in the TLB. The +initialization algorithm is therefore: + +0. Start with a fully-initialized page table layout, including the + top-level "L1" page containing the mappings for the page table + itself. + +1. Ensure that the initialization routine does not cross a page + boundary (to prevent stray TLB refill exceptions), that it occupies + a separate 4k page than the exception vectors (which we must + temporarily double-map), and that it operates entirely in registers + (to avoid doing memory access at inopportune moments). + +2. Pin the L1 page table PTE into the data TLB. This creates a double + mapping condition, but it is safe as nothing will use it until we + start refilling. + +3. Pin the page table page containing the PTE for the TLB miss + exception handler into the data TLB. This will likewise not be + accessed until the double map condition is resolved. + +4. Set PTEVADDR appropriately. The CPU state to handle refill + exceptions is now complete, but cannot be used until we resolve the + double mappings. + +5. Disable the initial/way6 data TLB entries first, by setting them to + an ASID of zero. This is safe as the code being executed is not + doing data accesses yet (including refills), and will resolve the + double mapping conditions we created above. + +6. Disable the initial/way6 instruction TLBs second. The very next + instruction following the invalidation of the currently-executing + code page will then cause a TLB refill exception, which will work + normally because we just resolved the final double-map condition. + (Pedantic note: if the vector page and the currently-executing page + are in different 512M way6 pages, disable the mapping for the + exception handlers first so the trap from our current code can be + handled. Currently Zephyr doesn't handle this condition as in all + reasonable hardware these regions will be near each other) + +Note: there is a different variant of the Xtensa MMU architecture +where the way 5/6 pages are immutable, and specify a set of +unchangable mappings from the final 384M of memory to the bottom and +top of physical memory. The intent here would (presumably) be that +these would be used by the kernel for all physical memory and that the +remaining memory space would be used for virtual mappings. This +doesn't match Zephyr's architecture well, as we tend to assume +page-level control over physical memory (e.g. .text/.rodata is cached +but .data is not on SMP, etc...). And in any case we don't have any +such hardware to experiment with. But with a little address +translation we could support this. + +## ASID vs. Virtual Mapping + +The ASID mechanism in Xtensa works like other architectures, and is +intended to be used similarly. The intent of the design is that at +context switch time, you can simply change RADID and the page table +data, and leave any existing mappings in place in the TLB using the +old ASID value(s). So in the common case where you switch back, +nothing needs to be flushed. + +Unfortunately this runs afoul of the virtual mapping of the page +refill: data TLB entries storing the 4M page table mapping space are +stored at ASID 1 (ring 0), they can't change when the page tables +change! So this region naively would have to be flushed, which is +tantamount to flushing the entire TLB regardless (the TLB is much +smaller than the 1024-page PTE array). + +The resolution in Zephyr is to give each ASID its own PTEVADDR mapping +in virtual space, such that the page tables don't overlap. This is +expensive in virtual address space: assigning 4M of space to each of +the 256 ASIDs (actually 254 as 0 and 1 are never used by user access) +would take a full gigabyte of address space. Zephyr optimizes this a +bit by deriving a unique sequential ASID from the hardware address of +the statically allocated array of L1 page table pages. + +Note, obviously, that any change of the mappings within an ASID +(e.g. to re-use it for another memory domain, or just for any runtime +mapping change other than mapping previously-unmapped pages) still +requires a TLB flush, and always will. + +## SMP/Cache Interaction + +A final important note is that the hardware PTE refill fetch works +like any other CPU memory access, and in particular it is governed by +the cacheability attributes of the TLB entry through which it was +loaded. This means that if the page table entries are marked +cacheable, then the hardware TLB refill process will be downstream of +the L1 data cache on the CPU. If the physical memory storing page +tables has been accessed recently by the CPU (for a refill of another +page mapped within the same cache line, or to change the tables) then +the refill will be served from the data cache and not main memory. + +This may or may not be desirable depending on access patterns. It +lets the L1 data cache act as a "L2 TLB" for applications with a lot +of access variability. But it also means that the TLB entries end up +being stored twice in the same CPU, wasting transistors that could +presumably store other useful data. + +But it it also important to note that the L1 data cache on Xtensa is +incoherent! The cache being used for refill reflects the last access +on the current CPU only, and not of the underlying memory being +mapped. Page table changes in the data cache of one CPU will be +invisible to the data cache of another. There is no simple way of +notifying another CPU of changes to page mappings beyond doing +system-wide flushes on all cpus every time a memory domain is +modified. + +The result is that, when SMP is enabled, Zephyr must ensure that all +page table mappings in the system are set uncached. The OS makes no +attempt to bolt on a software coherence layer. diff --git a/arch/xtensa/core/include/xtensa_mmu_priv.h b/arch/xtensa/core/include/xtensa_mmu_priv.h index cf72c92138373c..7b1030786f424d 100644 --- a/arch/xtensa/core/include/xtensa_mmu_priv.h +++ b/arch/xtensa/core/include/xtensa_mmu_priv.h @@ -132,15 +132,8 @@ * * PTE_ENTRY_ADDRESS = PTEVADDR + ((VADDR / 4096) * 4) */ -#define Z_XTENSA_PTE_ENTRY_VADDR(vaddr) \ - (Z_XTENSA_PTEVADDR + (((vaddr) / KB(4)) * 4)) - -/* - * The address of the top level page where the page - * is located in the virtual address. - */ -#define Z_XTENSA_PAGE_TABLE_VADDR \ - Z_XTENSA_PTE_ENTRY_VADDR(Z_XTENSA_PTEVADDR) +#define Z_XTENSA_PTE_ENTRY_VADDR(base, vaddr) \ + ((base) + (((vaddr) / KB(4)) * 4)) /* * Get asid for a given ring from rasid register. @@ -349,4 +342,8 @@ static inline void xtensa_dtlb_vaddr_invalidate(void *vaddr) } } +void xtensa_init_paging(uint32_t *l1_page); + +void xtensa_set_paging(uint32_t asid, uint32_t *l1_page); + #endif /* ZEPHYR_ARCH_XTENSA_XTENSA_MMU_PRIV_H_ */ diff --git a/arch/xtensa/core/mmu.c b/arch/xtensa/core/mmu.c new file mode 100644 index 00000000000000..24e47a42b9ded3 --- /dev/null +++ b/arch/xtensa/core/mmu.c @@ -0,0 +1,178 @@ +/* + * Copyright 2023 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ +#include +#include +#include +#include +#include + +#define ASID_INVALID 0 + +struct tlb_regs { + uint32_t rasid; + uint32_t ptevaddr; + uint32_t ptepin_as; + uint32_t ptepin_at; + uint32_t vecpin_as; + uint32_t vecpin_at; +}; + +static void compute_regs(uint32_t user_asid, uint32_t *l1_page, struct tlb_regs *regs) +{ + uint32_t vecbase = XTENSA_RSR("VECBASE"); + + __ASSERT_NO_MSG((((uint32_t)l1_page) & 0xfff) == 0); + __ASSERT_NO_MSG((user_asid == 0) || ((user_asid > 2) && + (user_asid < Z_XTENSA_MMU_SHARED_ASID))); + + /* We don't use ring 1, ring 0 ASID must be 1 */ + regs->rasid = (Z_XTENSA_MMU_SHARED_ASID << 24) | + (user_asid << 16) | 0x000201; + + /* Derive PTEVADDR from ASID so each domain gets its own PTE area */ + regs->ptevaddr = CONFIG_XTENSA_MMU_PTEVADDR + user_asid * 0x400000; + + /* The ptables code doesn't add the mapping for the l1 page itself */ + l1_page[Z_XTENSA_L1_POS(regs->ptevaddr)] = + (uint32_t)l1_page | Z_XTENSA_PAGE_TABLE_ATTR; + + regs->ptepin_at = (uint32_t)l1_page; + regs->ptepin_as = Z_XTENSA_PTE_ENTRY_VADDR(regs->ptevaddr, regs->ptevaddr) + | Z_XTENSA_MMU_PTE_WAY; + + /* Pin mapping for refilling the vector address into the ITLB + * (for handling TLB miss exceptions). Note: this is NOT an + * instruction TLB entry for the vector code itself, it's a + * DATA TLB entry for the page containing the vector mapping + * so the refill on instruction fetch can find it. The + * hardware doesn't have a 4k pinnable instruction TLB way, + * frustratingly. + */ + uint32_t vb_pte = l1_page[Z_XTENSA_L1_POS(vecbase)]; + + regs->vecpin_at = vb_pte; + regs->vecpin_as = Z_XTENSA_PTE_ENTRY_VADDR(regs->ptevaddr, vecbase) + | Z_XTENSA_MMU_VECBASE_WAY; +} + +/* Switch to a new page table. There are four items we have to set in + * the hardware: the PTE virtual address, the ring/ASID mapping + * register, and two pinned entries in the data TLB handling refills + * for the page tables and the vector handlers. + * + * These can be done in any order, provided that we ensure that no + * memory access which cause a TLB miss can happen during the process. + * This means that we must work entirely within registers in a single + * asm block. Also note that instruction fetches are memory accesses + * too, which means we cannot cross a page boundary which might reach + * a new page not in the TLB (a single jump to an aligned address that + * holds our five instructions is sufficient to guarantee that: I + * couldn't think of a way to do the alignment statically that also + * interoperated well with inline assembly). + */ +void xtensa_set_paging(uint32_t user_asid, uint32_t *l1_page) +{ + /* Optimization note: the registers computed here are pure + * functions of the two arguments. With a minor API tweak, + * they could be cached in e.g. a thread struct instead of + * being recomputed. This is called on context switch paths + * and is performance-sensitive. + */ + struct tlb_regs regs; + + compute_regs(user_asid, l1_page, ®s); + + __asm__ volatile("j 1f\n" + ".align 16\n" /* enough for 5 insns */ + "1:\n" + "wsr %0, PTEVADDR\n" + "wsr %1, RASID\n" + "wdtlb %2, %3\n" + "wdtlb %4, %5\n" + "isync" + :: "r"(regs.ptevaddr), "r"(regs.rasid), + "r"(regs.ptepin_at), "r"(regs.ptepin_as), + "r"(regs.vecpin_at), "r"(regs.vecpin_as)); +} + +/* This is effectively the same algorithm from xtensa_set_paging(), + * but it also disables the hardware-initialized 512M TLB entries in + * way 6 (because the hardware disallows duplicate TLB mappings). For + * instruction fetches this produces a critical ordering constraint: + * the instruction following the invalidation of ITLB entry mapping + * the current PC will by definition create a refill condition, which + * will (because the data TLB was invalidated) cause a refill + * exception. Therefore this step must be the very last one, once + * everything else is setup up and working, which includes the + * invalidation of the virtual PTEVADDR area so that the resulting + * refill can complete. + * + * Note that we can't guarantee that the compiler won't insert a data + * fetch from our stack memory after exit from the asm block (while it + * might be double-mapped), so we invalidate that data TLB inside the + * asm for correctness. The other 13 entries get invalidated in a C + * loop at the end. + */ +void xtensa_init_paging(uint32_t *l1_page) +{ + extern char z_xt_init_pc; /* defined in asm below */ + struct tlb_regs regs; + +#if CONFIG_MP_MAX_NUM_CPUS > 1 + /* The incoherent cache can get into terrible trouble if it's + * allowed to cache PTEs differently across CPUs. We require + * that all page tables supplied by the OS have exclusively + * uncached mappings for page data, but can't do anything + * about earlier code/firmware. Dump the cache to be safe. + */ + sys_cache_data_flush_and_invd_all(); +#endif + + compute_regs(ASID_INVALID, l1_page, ®s); + + uint32_t idtlb_pte = (regs.ptevaddr & 0xe0000000) | XCHAL_SPANNING_WAY; + uint32_t idtlb_stk = (((uint32_t)®s) & ~0xfff) | XCHAL_SPANNING_WAY; + uint32_t iitlb_pc = (((uint32_t)&z_xt_init_pc) & ~0xfff) | XCHAL_SPANNING_WAY; + + /* Note: the jump is mostly pedantry, as it's almost + * inconceivable that a hardware memory region at boot is + * going to cross a 512M page boundary. But we need the entry + * symbol to get the address above, so the jump is here for + * symmetry with the set_paging() code. + */ + __asm__ volatile("j z_xt_init_pc\n" + ".align 32\n" /* room for 10 insns */ + ".globl z_xt_init_pc\n" + "z_xt_init_pc:\n" + "wsr %0, PTEVADDR\n" + "wsr %1, RASID\n" + "wdtlb %2, %3\n" + "wdtlb %4, %5\n" + "idtlb %6\n" /* invalidate pte */ + "idtlb %7\n" /* invalidate stk */ + "isync\n" + "iitlb %8\n" /* invalidate pc */ + "isync\n" /* <--- traps a ITLB miss */ + :: "r"(regs.ptevaddr), "r"(regs.rasid), + "r"(regs.ptepin_at), "r"(regs.ptepin_as), + "r"(regs.vecpin_at), "r"(regs.vecpin_as), + "r"(idtlb_pte), "r"(idtlb_stk), "r"(iitlb_pc)); + + /* Invalidate the remaining (unused by this function) + * initialization entries. Now we're flying free with our own + * page table. + */ + for (int i = 0; i < 8; i++) { + uint32_t ixtlb = (i * 0x2000000000) | XCHAL_SPANNING_WAY; + + if (ixtlb != iitlb_pc) { + __asm__ volatile("iitlb %0" :: "r"(ixtlb)); + } + if (ixtlb != idtlb_stk && ixtlb != idtlb_pte) { + __asm__ volatile("idtlb %0" :: "r"(ixtlb)); + } + } + __asm__ volatile("isync"); +} diff --git a/arch/xtensa/core/ptables.c b/arch/xtensa/core/ptables.c index 2130a870e7757d..17950114544f4b 100644 --- a/arch/xtensa/core/ptables.c +++ b/arch/xtensa/core/ptables.c @@ -218,42 +218,6 @@ static inline uint32_t *alloc_l2_table(void) return NULL; } -/** - * @brief Switch page tables - * - * This switches the page tables to the incoming ones (@a ptables). - * Since data TLBs to L2 page tables are auto-filled, @a dtlb_inv - * can be used to invalidate these data TLBs. @a cache_inv can be - * set to true to invalidate cache to the page tables. - * - * @param[in] ptables Page tables to be switched to. - * @param[in] dtlb_inv True if to invalidate auto-fill data TLBs. - * @param[in] cache_inv True if to invalidate cache to page tables. - */ -static ALWAYS_INLINE void switch_page_tables(uint32_t *ptables, bool dtlb_inv, bool cache_inv) -{ - if (cache_inv) { - sys_cache_data_flush_and_invd_all(); - } - - /* Invalidate data TLB to L1 page table */ - xtensa_dtlb_vaddr_invalidate((void *)Z_XTENSA_PAGE_TABLE_VADDR); - - /* Now map the pagetable itself with KERNEL asid to avoid user thread - * from tampering with it. - */ - xtensa_dtlb_entry_write_sync( - Z_XTENSA_PTE((uint32_t)ptables, Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), - Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); - - if (dtlb_inv) { - /* Since L2 page tables are auto-refilled, - * invalidate all of them to flush the old entries out. - */ - xtensa_tlb_autorefill_invalidate(); - } -} - static void map_memory_range(const uint32_t start, const uint32_t end, const uint32_t attrs, bool shared) { @@ -345,6 +309,17 @@ static void xtensa_init_page_tables(void) #if defined(__GNUC__) #pragma GCC diagnostic pop #endif + /* Finally, the direct-mapped pages used in the page tables + * must be fixed up to use the same cache attribute (but these + * must be writable, obviously). They shouldn't be left at + * the default. + */ + map_memory_range((uint32_t) &l1_page_table[0], + (uint32_t) &l1_page_table[CONFIG_XTENSA_MMU_NUM_L1_TABLES], + Z_XTENSA_PAGE_TABLE_ATTR | Z_XTENSA_MMU_W, false); + map_memory_range((uint32_t) &l2_page_tables[0], + (uint32_t) &l2_page_tables[CONFIG_XTENSA_MMU_NUM_L2_TABLES], + Z_XTENSA_PAGE_TABLE_ATTR | Z_XTENSA_MMU_W, false); sys_cache_data_flush_all(); } @@ -356,9 +331,6 @@ __weak void arch_xtensa_mmu_post_init(bool is_core0) void z_xtensa_mmu_init(void) { - volatile uint8_t entry; - uint32_t ps, vecbase; - if (_current_cpu->id == 0) { /* This is normally done via arch_kernel_init() inside z_cstart(). * However, before that is called, we go through the sys_init of @@ -369,111 +341,7 @@ void z_xtensa_mmu_init(void) xtensa_init_page_tables(); } - /* Set the page table location in the virtual address */ - xtensa_ptevaddr_set((void *)Z_XTENSA_PTEVADDR); - - /* Set rasid */ - xtensa_rasid_asid_set(Z_XTENSA_MMU_SHARED_ASID, Z_XTENSA_SHARED_RING); - - /* Next step is to invalidate the tlb entry that contains the top level - * page table. This way we don't cause a multi hit exception. - */ - xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, 6)); - xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, 6)); - - /* We are not using a flat table page, so we need to map - * only the top level page table (which maps the page table itself). - * - * Lets use one of the wired entry, so we never have tlb miss for - * the top level table. - */ - xtensa_dtlb_entry_write(Z_XTENSA_PTE((uint32_t)z_xtensa_kernel_ptables, - Z_XTENSA_KERNEL_RING, Z_XTENSA_PAGE_TABLE_ATTR), - Z_XTENSA_TLB_ENTRY(Z_XTENSA_PAGE_TABLE_VADDR, Z_XTENSA_MMU_PTE_WAY)); - - /* Before invalidate the text region in the TLB entry 6, we need to - * map the exception vector into one of the wired entries to avoid - * a page miss for the exception. - */ - __asm__ volatile("rsr.vecbase %0" : "=r"(vecbase)); - - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY( - Z_XTENSA_PTEVADDR + MB(4), 3)); - - xtensa_dtlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY( - Z_XTENSA_PTEVADDR + MB(4), 3)); - - /* Temporarily uses KernelExceptionVector for level 1 interrupts - * handling. This is due to UserExceptionVector needing to jump to - * _Level1Vector. The jump ('j') instruction offset is incorrect - * when we move VECBASE below. - */ - __asm__ volatile("rsr.ps %0" : "=r"(ps)); - ps &= ~PS_UM; - __asm__ volatile("wsr.ps %0; rsync" :: "a"(ps)); - - __asm__ volatile("wsr.vecbase %0; rsync\n\t" - :: "a"(Z_XTENSA_PTEVADDR + MB(4))); - - - /* Finally, lets invalidate all entries in way 6 as the page tables - * should have already mapped the regions we care about for boot. - */ - for (entry = 0; entry < BIT(XCHAL_ITLB_ARF_ENTRIES_LOG2); entry++) { - __asm__ volatile("iitlb %[idx]\n\t" - "isync" - :: [idx] "a"((entry << 29) | 6)); - } - - for (entry = 0; entry < BIT(XCHAL_DTLB_ARF_ENTRIES_LOG2); entry++) { - __asm__ volatile("idtlb %[idx]\n\t" - "dsync" - :: [idx] "a"((entry << 29) | 6)); - } - - /* Map VECBASE to a fixed data TLB */ - xtensa_dtlb_entry_write( - Z_XTENSA_PTE((uint32_t)vecbase, - Z_XTENSA_KERNEL_RING, Z_XTENSA_MMU_CACHED_WB), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, Z_XTENSA_MMU_VECBASE_WAY)); - - /* - * Pre-load TLB for vecbase so exception handling won't result - * in TLB miss during boot, and that we can handle single - * TLB misses. - */ - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_KERNEL_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_AUTOFILL_TLB_ENTRY(vecbase)); - - /* To finish, just restore vecbase and invalidate TLB entries - * used to map the relocated vecbase. - */ - __asm__ volatile("wsr.vecbase %0; rsync\n\t" - :: "a"(vecbase)); - - /* Restore PS_UM so that level 1 interrupt handling will go to - * UserExceptionVector. - */ - __asm__ volatile("rsr.ps %0" : "=r"(ps)); - ps |= PS_UM; - __asm__ volatile("wsr.ps %0; rsync" :: "a"(ps)); - - xtensa_dtlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); - xtensa_itlb_entry_invalidate_sync(Z_XTENSA_TLB_ENTRY(Z_XTENSA_PTEVADDR + MB(4), 3)); - - /* - * Clear out THREADPTR as we use it to indicate - * whether we are in user mode or not. - */ - XTENSA_WUR("THREADPTR", 0); + xtensa_init_paging(z_xtensa_kernel_ptables); arch_xtensa_mmu_post_init(_current_cpu->id == 0); } @@ -504,7 +372,7 @@ __weak void arch_reserved_pages_update(void) static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, uint32_t flags, bool is_user) { - uint32_t l1_pos = (uint32_t)vaddr >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS((uint32_t)vaddr); uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); uint32_t *table; @@ -530,6 +398,7 @@ static bool l2_page_table_map(uint32_t *l1_table, void *vaddr, uintptr_t phys, flags); sys_cache_data_flush_range((void *)&table[l2_pos], sizeof(table[0])); + xtensa_tlb_autorefill_invalidate(); return true; } @@ -604,18 +473,6 @@ static inline void __arch_mem_map(void *va, uintptr_t pa, uint32_t xtensa_flags, k_spin_unlock(&z_mem_domain_lock, key); } #endif /* CONFIG_USERSPACE */ - - if ((xtensa_flags & Z_XTENSA_MMU_X) == Z_XTENSA_MMU_X) { - xtensa_itlb_vaddr_invalidate(vaddr); - } - xtensa_dtlb_vaddr_invalidate(vaddr); - - if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { - if (xtensa_flags & Z_XTENSA_MMU_X) { - xtensa_itlb_vaddr_invalidate(vaddr_uc); - } - xtensa_dtlb_vaddr_invalidate(vaddr_uc); - } } void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) @@ -680,7 +537,7 @@ void arch_mem_map(void *virt, uintptr_t phys, size_t size, uint32_t flags) */ static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) { - uint32_t l1_pos = (uint32_t)vaddr >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS((uint32_t)vaddr); uint32_t l2_pos = Z_XTENSA_L2_POS((uint32_t)vaddr); uint32_t *l2_table; uint32_t table_pos; @@ -717,10 +574,9 @@ static bool l2_page_table_unmap(uint32_t *l1_table, void *vaddr) table_pos = (l2_table - (uint32_t *)l2_page_tables) / (XTENSA_L2_PAGE_TABLE_ENTRIES); atomic_clear_bit(l2_page_tables_track, table_pos); - /* Need to invalidate L2 page table as it is no longer valid. */ - xtensa_dtlb_vaddr_invalidate((void *)l2_table); - end: + /* Need to invalidate L2 page table as it is no longer valid. */ + xtensa_tlb_autorefill_invalidate(); return exec; } @@ -764,18 +620,6 @@ static inline void __arch_mem_unmap(void *va) } k_spin_unlock(&z_mem_domain_lock, key); #endif /* CONFIG_USERSPACE */ - - if (is_exec) { - xtensa_itlb_vaddr_invalidate(vaddr); - } - xtensa_dtlb_vaddr_invalidate(vaddr); - - if (IS_ENABLED(CONFIG_XTENSA_MMU_DOUBLE_MAP)) { - if (is_exec) { - xtensa_itlb_vaddr_invalidate(vaddr_uc); - } - xtensa_dtlb_vaddr_invalidate(vaddr_uc); - } } void arch_mem_unmap(void *addr, size_t size) @@ -853,7 +697,7 @@ void z_xtensa_mmu_tlb_shootdown(void) * MMU_PTE_WAY, so we can skip the probing step by * generating the query entry directly. */ - ptevaddr_entry = Z_XTENSA_PAGE_TABLE_VADDR | MMU_PTE_WAY; + ptevaddr_entry = (uint32_t)xtensa_ptevaddr_get() | Z_XTENSA_MMU_PTE_WAY; ptevaddr = xtensa_dtlb_paddr_read(ptevaddr_entry); thread_ptables = (uint32_t)thread->arch.ptables; @@ -863,7 +707,9 @@ void z_xtensa_mmu_tlb_shootdown(void) * indicated by the current thread are different * than the current mapped page table. */ - switch_page_tables((uint32_t *)thread_ptables, true, true); + struct arch_mem_domain *domain = + &(thread->mem_domain_info.mem_domain->arch); + xtensa_set_paging(domain->asid, (uint32_t *)thread_ptables); } } @@ -981,9 +827,8 @@ static int region_map_update(uint32_t *ptables, uintptr_t start, for (size_t offset = 0; offset < size; offset += CONFIG_MMU_PAGE_SIZE) { uint32_t *l2_table, pte; uint32_t page = start + offset; - uint32_t l1_pos = page >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); uint32_t l2_pos = Z_XTENSA_L2_POS(page); - /* Make sure we grab a fresh copy of L1 page table */ sys_cache_data_invd_range((void *)&ptables[l1_pos], sizeof(ptables[0])); @@ -998,8 +843,7 @@ static int region_map_update(uint32_t *ptables, uintptr_t start, sys_cache_data_flush_range((void *)&l2_table[l2_pos], sizeof(l2_table[0])); - xtensa_dtlb_vaddr_invalidate( - (void *)(pte & Z_XTENSA_PTE_PPN_MASK)); + xtensa_dtlb_vaddr_invalidate((void *)page); } return ret; @@ -1127,7 +971,7 @@ int arch_mem_domain_thread_add(struct k_thread *thread) * the current thread running. */ if (thread == _current_cpu->current) { - switch_page_tables(thread->arch.ptables, true, true); + xtensa_set_paging(domain->arch.asid, thread->arch.ptables); } #if CONFIG_MP_MAX_NUM_CPUS > 1 @@ -1179,7 +1023,7 @@ static bool page_validate(uint32_t *ptables, uint32_t page, uint8_t ring, bool w { uint8_t asid_ring; uint32_t rasid, pte, *l2_table; - uint32_t l1_pos = page >> 22; + uint32_t l1_pos = Z_XTENSA_L1_POS(page); uint32_t l2_pos = Z_XTENSA_L2_POS(page); if (is_pte_illegal(ptables[l1_pos])) { @@ -1245,12 +1089,7 @@ void z_xtensa_swap_update_page_tables(struct k_thread *incoming) struct arch_mem_domain *domain = &(incoming->mem_domain_info.mem_domain->arch); - /* Lets set the asid for the incoming thread */ - if ((incoming->base.user_options & K_USER) != 0) { - xtensa_rasid_asid_set(domain->asid, Z_XTENSA_USER_RING); - } - - switch_page_tables(ptables, true, false); + xtensa_set_paging(domain->asid, ptables); } #endif /* CONFIG_USERSPACE */ diff --git a/arch/xtensa/include/xtensa-asm2-s.h b/arch/xtensa/include/xtensa-asm2-s.h index 3f3ffd90b7ae04..416a83453a2d0e 100644 --- a/arch/xtensa/include/xtensa-asm2-s.h +++ b/arch/xtensa/include/xtensa-asm2-s.h @@ -589,31 +589,6 @@ _Level\LVL\()VectorHelper : .global _Level\LVL\()Vector _Level\LVL\()Vector: #endif -#ifdef CONFIG_XTENSA_MMU - wsr.ZSR_MMU_0 a2 - wsr.ZSR_MMU_1 a3 - rsync - - /* Calculations below will clobber registers used. - * So we make a copy of the stack pointer to avoid - * changing it. - */ - mov a3, a1 - - CALC_PTEVADDR_BASE a2 - - /* Preload PTE entry page of current stack. */ - PRELOAD_PTEVADDR a3, a2 - - /* Preload PTE entry page of new stack, where - * it will be used later (in EXCINT_HANDLER above). - */ - rsr.ZSR_CPU a3 - PRELOAD_PTEVADDR a3, a2 - - rsr.ZSR_MMU_1 a3 - rsr.ZSR_MMU_0 a2 -#endif /* CONFIG_XTENSA_MMU */ addi a1, a1, -___xtensa_irq_bsa_t_SIZEOF s32i a0, a1, ___xtensa_irq_bsa_t_a0_OFFSET s32i a2, a1, ___xtensa_irq_bsa_t_a2_OFFSET diff --git a/soc/xtensa/dc233c/mmu.c b/soc/xtensa/dc233c/mmu.c index 718f79779ebd5c..8decc952bf2ac6 100644 --- a/soc/xtensa/dc233c/mmu.c +++ b/soc/xtensa/dc233c/mmu.c @@ -35,33 +35,3 @@ const struct xtensa_mmu_range xtensa_soc_mmu_ranges[] = { }; int xtensa_soc_mmu_ranges_num = ARRAY_SIZE(xtensa_soc_mmu_ranges); - -void arch_xtensa_mmu_post_init(bool is_core0) -{ - uint32_t vecbase; - - ARG_UNUSED(is_core0); - - __asm__ volatile("rsr.vecbase %0" : "=r"(vecbase)); - - /* Invalidate any autorefill instr TLBs of VECBASE so we can map it - * permanently below. - */ - xtensa_itlb_vaddr_invalidate((void *)vecbase); - - /* Map VECBASE permanently in instr TLB way 4 so we will always have - * access to exception handlers. Each way 4 TLB covers 1MB (unless - * ITLBCFG has been changed before this, which should not have - * happened). Also this needs to be mapped as SHARED so both kernel - * and userspace can execute code here => same as .text. - * - * Note that we don't want to map the first 1MB in data TLB as - * we want to keep page 0 (0x00000000) unmapped to catch null pointer - * de-references. - */ - vecbase = ROUND_DOWN(vecbase, MB(1)); - xtensa_itlb_entry_write_sync( - Z_XTENSA_PTE(vecbase, Z_XTENSA_SHARED_RING, - Z_XTENSA_MMU_X | Z_XTENSA_MMU_CACHED_WT), - Z_XTENSA_TLB_ENTRY((uint32_t)vecbase, 4)); -}