diff --git a/arch/arm/core/cortex_a_r/thread.c b/arch/arm/core/cortex_a_r/thread.c index 14f6d3f7de901..1fca44055eb83 100644 --- a/arch/arm/core/cortex_a_r/thread.c +++ b/arch/arm/core/cortex_a_r/thread.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2013-2014 Wind River Systems, Inc. * Copyright (c) 2021 Lexmark International, Inc. + * Copyright 2025 Arm Limited and/or its affiliates * * SPDX-License-Identifier: Apache-2.0 */ @@ -37,6 +38,34 @@ */ #define DEFAULT_EXC_RETURN 0xFD; +#ifdef CONFIG_USERSPACE +static void setup_priv_stack(struct k_thread *thread) +{ + /* Set up privileged stack before entering user mode */ + thread->arch.priv_stack_start = (uint32_t)z_priv_stack_find(thread->stack_obj); + + /* CONFIG_PRIVILEGED_STACK_SIZE does not account for MPU_GUARD_ALIGN_AND_SIZE or + * MPU_GUARD_ALIGN_AND_SIZE_FLOAT. Therefore, we must compute priv_stack_end here before + * adjusting priv_stack_start for the mpu guard alignment + */ + thread->arch.priv_stack_end = thread->arch.priv_stack_start + CONFIG_PRIVILEGED_STACK_SIZE; + +#if defined(CONFIG_MPU_STACK_GUARD) + /* Stack guard area reserved at the bottom of the thread's + * privileged stack. Adjust the available (writable) stack + * buffer area accordingly. + */ +#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING) + thread->arch.priv_stack_start += + ((thread->arch.mode & Z_ARM_MODE_MPU_GUARD_FLOAT_Msk) != 0) ? + MPU_GUARD_ALIGN_AND_SIZE_FLOAT : MPU_GUARD_ALIGN_AND_SIZE; +#else + thread->arch.priv_stack_start += MPU_GUARD_ALIGN_AND_SIZE; +#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */ +#endif /* CONFIG_MPU_STACK_GUARD */ +} +#endif + /* An initial context, to be "restored" by z_arm_pendsv(), is put at the other * end of the stack, and thus reusable by the stack when not needed anymore. * @@ -80,7 +109,10 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, iframe = Z_STACK_PTR_TO_FRAME(struct __basic_sf, stack_ptr); #if defined(CONFIG_USERSPACE) + thread->arch.priv_stack_start = 0; if ((thread->base.user_options & K_USER) != 0) { + setup_priv_stack(thread); + iframe = Z_STACK_PTR_TO_FRAME(struct __basic_sf, thread->arch.priv_stack_end); iframe->pc = (uint32_t)arch_user_mode_enter; } else { iframe->pc = (uint32_t)z_thread_entry; @@ -122,9 +154,6 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, thread->arch.mode |= Z_ARM_MODE_MPU_GUARD_FLOAT_Msk; } #endif -#if defined(CONFIG_USERSPACE) - thread->arch.priv_stack_start = 0; -#endif #endif /* * initial values in all other registers/thread entries are @@ -196,10 +225,8 @@ static inline void z_arm_thread_stack_info_adjust(struct k_thread *thread, FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, void *p1, void *p2, void *p3) { + uint32_t sp_is_priv = 1; - /* Set up privileged stack before entering user mode */ - _current->arch.priv_stack_start = - (uint32_t)z_priv_stack_find(_current->stack_obj); #if defined(CONFIG_MPU_STACK_GUARD) #if defined(CONFIG_THREAD_STACK_INFO) /* We're dropping to user mode which means the guard area is no @@ -216,29 +243,29 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, _current->stack_info.start -= MPU_GUARD_ALIGN_AND_SIZE; _current->stack_info.size += MPU_GUARD_ALIGN_AND_SIZE; #endif /* CONFIG_THREAD_STACK_INFO */ - - /* Stack guard area reserved at the bottom of the thread's - * privileged stack. Adjust the available (writable) stack - * buffer area accordingly. - */ -#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING) - _current->arch.priv_stack_start += - ((_current->arch.mode & Z_ARM_MODE_MPU_GUARD_FLOAT_Msk) != 0) ? - MPU_GUARD_ALIGN_AND_SIZE_FLOAT : MPU_GUARD_ALIGN_AND_SIZE; -#else - _current->arch.priv_stack_start += MPU_GUARD_ALIGN_AND_SIZE; -#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */ #endif /* CONFIG_MPU_STACK_GUARD */ -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) - _current->arch.priv_stack_end = - _current->arch.priv_stack_start + CONFIG_PRIVILEGED_STACK_SIZE; -#endif + /* 2 ways how arch_user_mode_enter is called: + * - called as part of context switch from z_arm_pendsv, in this case privileged stack is + * already setup and stack pointer points to privileged stack. + * - called directly from k_thread_user_mode_enter, in this case privileged stack is not + * setup and stack pointer points to user stack. + * + * When called from k_thread_user_mode_enter, we need to check and setup the privileged + * stack and then instruct z_arm_userspace_enter to change the PSP to the privileged stack. + * Note that we do not change the PSP in this function to avoid any conflict with compiler's + * sequence which has already pushed stuff on the user stack. + */ + if (0 == _current->arch.priv_stack_start) { + setup_priv_stack(_current); + sp_is_priv = 0; + } z_arm_userspace_enter(user_entry, p1, p2, p3, (uint32_t)_current->stack_info.start, _current->stack_info.size - - _current->stack_info.delta); + _current->stack_info.delta, + sp_is_priv); CODE_UNREACHABLE; } diff --git a/arch/arm/core/cortex_m/thread.c b/arch/arm/core/cortex_m/thread.c index 414eb865f93eb..70d627a52152f 100644 --- a/arch/arm/core/cortex_m/thread.c +++ b/arch/arm/core/cortex_m/thread.c @@ -45,6 +45,35 @@ K_THREAD_STACK_DECLARE(z_main_stack, CONFIG_MAIN_STACK_SIZE); #endif +#ifdef CONFIG_USERSPACE +static void setup_priv_stack(struct k_thread *thread) +{ + /* Set up privileged stack before entering user mode */ + thread->arch.priv_stack_start = (uint32_t)z_priv_stack_find(thread->stack_obj); + + /* CONFIG_PRIVILEGED_STACK_SIZE does not account for MPU_GUARD_ALIGN_AND_SIZE or + * MPU_GUARD_ALIGN_AND_SIZE_FLOAT. Therefore, we must compute priv_stack_end here before + * adjusting priv_stack_start for the mpu guard alignment + */ + thread->arch.priv_stack_end = thread->arch.priv_stack_start + CONFIG_PRIVILEGED_STACK_SIZE; + +#if defined(CONFIG_MPU_STACK_GUARD) + /* Stack guard area reserved at the bottom of the thread's + * privileged stack. Adjust the available (writable) stack + * buffer area accordingly. + */ +#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING) + thread->arch.priv_stack_start += + ((thread->arch.mode & Z_ARM_MODE_MPU_GUARD_FLOAT_Msk) != 0) + ? MPU_GUARD_ALIGN_AND_SIZE_FLOAT + : MPU_GUARD_ALIGN_AND_SIZE; +#else + thread->arch.priv_stack_start += MPU_GUARD_ALIGN_AND_SIZE; +#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */ +#endif /* CONFIG_MPU_STACK_GUARD */ +} +#endif + /* An initial context, to be "restored" by z_arm_pendsv(), is put at the other * end of the stack, and thus reusable by the stack when not needed anymore. * @@ -87,7 +116,10 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, char *sta iframe = Z_STACK_PTR_TO_FRAME(struct __basic_sf, stack_ptr); #if defined(CONFIG_USERSPACE) + thread->arch.priv_stack_start = 0; if ((thread->base.user_options & K_USER) != 0) { + setup_priv_stack(thread); + iframe = Z_STACK_PTR_TO_FRAME(struct __basic_sf, thread->arch.priv_stack_end); iframe->pc = (uint32_t)arch_user_mode_enter; } else { iframe->pc = (uint32_t)z_thread_entry; @@ -118,9 +150,6 @@ void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack, char *sta thread->arch.mode |= Z_ARM_MODE_MPU_GUARD_FLOAT_Msk; } #endif -#if defined(CONFIG_USERSPACE) - thread->arch.priv_stack_start = 0; -#endif #endif /* * initial values in all other registers/thread entries are @@ -215,9 +244,8 @@ uint32_t z_arm_mpu_stack_guard_and_fpu_adjust(struct k_thread *thread) #ifdef CONFIG_USERSPACE FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, void *p1, void *p2, void *p3) { + uint32_t sp_is_priv = 1; - /* Set up privileged stack before entering user mode */ - _current->arch.priv_stack_start = (uint32_t)z_priv_stack_find(_current->stack_obj); #if defined(CONFIG_MPU_STACK_GUARD) #if defined(CONFIG_THREAD_STACK_INFO) /* We're dropping to user mode which means the guard area is no @@ -234,23 +262,26 @@ FUNC_NORETURN void arch_user_mode_enter(k_thread_entry_t user_entry, void *p1, v _current->stack_info.start -= MPU_GUARD_ALIGN_AND_SIZE; _current->stack_info.size += MPU_GUARD_ALIGN_AND_SIZE; #endif /* CONFIG_THREAD_STACK_INFO */ +#endif /* CONFIG_MPU_STACK_GUARD */ - /* Stack guard area reserved at the bottom of the thread's - * privileged stack. Adjust the available (writable) stack - * buffer area accordingly. + /* 2 ways how arch_user_mode_enter is called: + * - called as part of context switch from z_arm_pendsv, in this case privileged stack is + * already setup and stack pointer points to privileged stack. + * - called directly from k_thread_user_mode_enter, in this case privileged stack is not + * setup and stack pointer points to user stack. + * + * When called from k_thread_user_mode_enter, we need to check and setup the privileged + * stack and then instruct z_arm_userspace_enter to change the PSP to the privileged stack. + * Note that we do not change the PSP in this function to avoid any conflict with compiler's + * sequence which has already pushed stuff on the user stack. */ -#if defined(CONFIG_FPU) && defined(CONFIG_FPU_SHARING) - _current->arch.priv_stack_start += - ((_current->arch.mode & Z_ARM_MODE_MPU_GUARD_FLOAT_Msk) != 0) - ? MPU_GUARD_ALIGN_AND_SIZE_FLOAT - : MPU_GUARD_ALIGN_AND_SIZE; -#else - _current->arch.priv_stack_start += MPU_GUARD_ALIGN_AND_SIZE; -#endif /* CONFIG_FPU && CONFIG_FPU_SHARING */ -#endif /* CONFIG_MPU_STACK_GUARD */ + if (0 == _current->arch.priv_stack_start) { + setup_priv_stack(_current); + sp_is_priv = 0; + } z_arm_userspace_enter(user_entry, p1, p2, p3, (uint32_t)_current->stack_info.start, - _current->stack_info.size - _current->stack_info.delta); + _current->stack_info.size - _current->stack_info.delta, sp_is_priv); CODE_UNREACHABLE; } diff --git a/arch/arm/core/offsets/offsets_aarch32.c b/arch/arm/core/offsets/offsets_aarch32.c index 693546630b05e..6a396fe33f7fc 100644 --- a/arch/arm/core/offsets/offsets_aarch32.c +++ b/arch/arm/core/offsets/offsets_aarch32.c @@ -45,8 +45,9 @@ GEN_OFFSET_SYM(_thread_arch_t, mode_exc_return); #endif #if defined(CONFIG_USERSPACE) GEN_OFFSET_SYM(_thread_arch_t, priv_stack_start); -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) GEN_OFFSET_SYM(_thread_arch_t, priv_stack_end); + +#if defined(CONFIG_CPU_AARCH32_CORTEX_R) GEN_OFFSET_SYM(_thread_arch_t, sp_usr); #endif #endif diff --git a/arch/arm/core/userspace.S b/arch/arm/core/userspace.S index 2a7d59be8ba3e..f72969f277b05 100644 --- a/arch/arm/core/userspace.S +++ b/arch/arm/core/userspace.S @@ -42,45 +42,41 @@ GDATA(_k_syscall_table) * * The function is invoked as: * z_arm_userspace_enter(user_entry, p1, p2, p3, - * stack_info.start, stack_info.size); + * stack_info.start, stack_info.size, + * sp_is_priv); */ SECTION_FUNC(TEXT,z_arm_userspace_enter) /* move user_entry to lr */ mov lr, r0 - /* prepare to set stack to privileged stack */ + /* load arguments from stack: + * r4 = user stack start + * r5 = user stack size + * r6 = sp_is_priv (1 if already on privileged stack) + */ + pop {r4, r5, r6} + + /* get current thread pointer */ ldr r0, =_kernel ldr r0, [r0, #_kernel_offset_to_current] + #if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) /* move p1 to ip */ mov ip, r1 - ldr r1, =_thread_offset_to_priv_stack_start - ldr r0, [r0, r1] /* priv stack ptr */ - ldr r1, =CONFIG_PRIVILEGED_STACK_SIZE - add r0, r0, r1 + ldr r1, =_thread_offset_to_priv_stack_end + ldr r0, [r0, r1] /* Restore p1 from ip */ mov r1, ip -#elif defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) - ldr r0, [r0, #_thread_offset_to_priv_stack_start] /* priv stack ptr */ - ldr ip, =CONFIG_PRIVILEGED_STACK_SIZE - add r0, r0, ip -#elif defined(CONFIG_CPU_AARCH32_CORTEX_R) - ldr r0, [r0, #_thread_offset_to_priv_stack_start] /* priv stack ptr */ - ldr ip, =CONFIG_PRIVILEGED_STACK_SIZE - add r0, r0, ip - - ldr ip, =_kernel - ldr ip, [ip, #_kernel_offset_to_current] - str r0, [ip, #_thread_offset_to_priv_stack_end] /* priv stack end */ +#elif defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) \ + || defined(CONFIG_CPU_AARCH32_CORTEX_R) + ldr r0, [r0, #_thread_offset_to_priv_stack_end] /* privileged stack ptr */ #endif - /* store current stack pointer to ip - * the current stack pointer is needed to retrieve - * stack_info.start and stack_info.size - */ - mov ip, sp + /* check if current stack is privileged and switch to it if not */ + cmp r6, #1 + beq 1f -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) +#if defined(CONFIG_CPU_AARCH32_CORTEX_R) || defined(CONFIG_CPU_AARCH32_CORTEX_A) mov sp, r0 #else /* set stack to privileged stack @@ -94,35 +90,28 @@ SECTION_FUNC(TEXT,z_arm_userspace_enter) msr PSP, r0 #endif +1: + /* push thread args and entrypoint to stack */ + push {r1,r2,r3,lr} + #if defined(CONFIG_BUILTIN_STACK_GUARD) /* At this point the privileged stack is not yet protected by PSPLIM. - * Since we have just switched to the top of the privileged stack, we + * Since we have switched to the top of the privileged stack, we * are safe, as long as the stack can accommodate the maximum exception * stack frame. */ - - /* set stack pointer limit to the start of the priv stack */ - ldr r0, =_kernel - ldr r0, [r0, #_kernel_offset_to_current] - ldr r0, [r0, #_thread_offset_to_priv_stack_start] /* priv stack ptr */ + ldr r1, =CONFIG_PRIVILEGED_STACK_SIZE + sub r0, r0, r1 /* Calculate start of privileged stack */ + /* set stack pointer limit to the start of the privileged stack */ msr PSPLIM, r0 #endif - /* push args to stack */ - push {r1,r2,r3,lr} -#if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) - mov r1, ip - push {r0,r1} -#elif defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) \ - || defined(CONFIG_CPU_AARCH32_CORTEX_R) - push {r0,ip} -#endif /* Re-program dynamic memory map. * * Important note: * z_arm_configure_dynamic_mpu_regions() may re-program the MPU Stack Guard - * to guard the privilege stack for overflows (if building with option + * to guard the privileged stack for overflows (if building with option * CONFIG_MPU_STACK_GUARD). There is a risk of actually overflowing the * stack while doing the re-programming. We minimize the risk by placing * this function immediately after we have switched to the privileged stack @@ -136,29 +125,10 @@ SECTION_FUNC(TEXT,z_arm_userspace_enter) ldr r0, [r0, #_kernel_offset_to_current] bl z_arm_configure_dynamic_mpu_regions -#if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) - pop {r0,r3} - - /* load up stack info from user stack */ - ldr r0, [r3] - ldr r3, [r3, #4] - mov ip, r3 - - push {r0,r3} -#elif defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) \ - || defined(CONFIG_CPU_AARCH32_CORTEX_R) - pop {r0,ip} - - /* load up stack info from user stack */ - ldr r0, [ip] - ldr ip, [ip, #4] - - push {r0,ip} -#endif - /* clear the user stack area to clean out privileged data */ /* from right past the guard right up to the end */ - mov r2, ip + mov r0, r4 + mov r2, r5 #ifdef CONFIG_INIT_STACKS ldr r1,=0xaaaaaaaa #else @@ -166,17 +136,18 @@ SECTION_FUNC(TEXT,z_arm_userspace_enter) #endif bl memset + /* At this point: + * r4 contains user stack start + * r5 contains user stack size + * calculate top of user stack in r0 (start+size) + */ #if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) - pop {r0, r1} - mov ip, r1 + adds r0, r4, r5 #elif defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) \ || defined(CONFIG_CPU_AARCH32_CORTEX_R) - pop {r0,ip} + add r0, r4, r5 #endif - /* r0 contains user stack start, ip contains user stack size */ - add r0, r0, ip /* calculate top of stack */ - /* pop remaining arguments from stack before switching stacks */ #if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) /* Use r4 to pop lr, then restore r4 */ @@ -189,7 +160,7 @@ SECTION_FUNC(TEXT,z_arm_userspace_enter) pop {r1,r2,r3,lr} #endif -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) +#if defined(CONFIG_CPU_AARCH32_CORTEX_R) || defined(CONFIG_CPU_AARCH32_CORTEX_A) /* * set stack to user stack. We are in SYSTEM state, so r13 and r14 are * shared with USER state @@ -224,10 +195,7 @@ SECTION_FUNC(TEXT,z_arm_userspace_enter) isb /* Set PSPLIM to guard the thread's user stack. */ - ldr r0, =_kernel - ldr r0, [r0, #_kernel_offset_to_current] - ldr r0, [r0, #_thread_offset_to_stack_info_start] - msr PSPLIM, r0 + msr PSPLIM, r4 pop {r0, ip} #endif diff --git a/arch/arm/include/cortex_a_r/kernel_arch_func.h b/arch/arm/include/cortex_a_r/kernel_arch_func.h index 7a100db07ddb1..2a4a89a0ae907 100644 --- a/arch/arm/include/cortex_a_r/kernel_arch_func.h +++ b/arch/arm/include/cortex_a_r/kernel_arch_func.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2019 Carlo Caione + * Copyright 2025 Arm Limited and/or its affiliates * * SPDX-License-Identifier: Apache-2.0 */ @@ -77,7 +78,8 @@ static ALWAYS_INLINE void arch_switch(void *switch_to, void **switched_from) extern FUNC_NORETURN void z_arm_userspace_enter(k_thread_entry_t user_entry, void *p1, void *p2, void *p3, uint32_t stack_end, - uint32_t stack_start); + uint32_t stack_start, + uint32_t sp_is_priv); extern void z_arm_fatal_error(unsigned int reason, const struct arch_esf *esf); diff --git a/arch/arm/include/cortex_m/kernel_arch_func.h b/arch/arm/include/cortex_m/kernel_arch_func.h index 5abe5870762cf..59290cc546308 100644 --- a/arch/arm/include/cortex_m/kernel_arch_func.h +++ b/arch/arm/include/cortex_m/kernel_arch_func.h @@ -1,5 +1,6 @@ /* * Copyright (c) 2019 Carlo Caione + * Copyright 2025 Arm Limited and/or its affiliates * * SPDX-License-Identifier: Apache-2.0 */ @@ -75,7 +76,8 @@ extern FUNC_NORETURN void z_arm_switch_to_main_no_multithreading(k_thread_entry_ #endif /* !CONFIG_MULTITHREADING */ extern FUNC_NORETURN void z_arm_userspace_enter(k_thread_entry_t user_entry, void *p1, void *p2, - void *p3, uint32_t stack_end, uint32_t stack_start); + void *p3, uint32_t stack_end, uint32_t stack_start, + uint32_t sp_is_priv); extern void z_arm_fatal_error(unsigned int reason, const struct arch_esf *esf); diff --git a/arch/arm/include/offsets_short_arch.h b/arch/arm/include/offsets_short_arch.h index ea6af4db92df3..350b8cb54c63b 100644 --- a/arch/arm/include/offsets_short_arch.h +++ b/arch/arm/include/offsets_short_arch.h @@ -45,10 +45,10 @@ #define _thread_offset_to_priv_stack_start \ (___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_start_OFFSET) -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) #define _thread_offset_to_priv_stack_end \ (___thread_t_arch_OFFSET + ___thread_arch_t_priv_stack_end_OFFSET) +#if defined(CONFIG_CPU_AARCH32_CORTEX_R) #define _thread_offset_to_sp_usr \ (___thread_t_arch_OFFSET + ___thread_arch_t_sp_usr_OFFSET) #endif diff --git a/include/zephyr/arch/arm/thread.h b/include/zephyr/arch/arm/thread.h index 139f606809f9d..a3cccbd9fae6e 100644 --- a/include/zephyr/arch/arm/thread.h +++ b/include/zephyr/arch/arm/thread.h @@ -128,8 +128,8 @@ struct _thread_arch { #if defined(CONFIG_USERSPACE) uint32_t priv_stack_start; -#if defined(CONFIG_CPU_AARCH32_CORTEX_R) uint32_t priv_stack_end; +#if defined(CONFIG_CPU_AARCH32_CORTEX_R) uint32_t sp_usr; #endif #endif