Like other architectures such as x86, arm64, riscv, powerpc and s390,
select THREAD_INFO_IN_TASK for LoongArch to move thread_info off the
stack into task_struct. This follows modern kernel standards and also
makes the system more secure.
With this patch, thread_info is included in task_struct at an offset
of 0 instead of being placed at the bottom of the kernel stack. Thus,
the $tp register points to both thread_info and task_struct.
To support this, introduce a per-CPU variable cpu_tasks to store the
pointer to the current task_struct. This decouples the recovery of the
$tp register from the stack pointer during exception entry.
Then initialize cpu_tasks for the primary and secondary CPUs during
arch-specific setup and SMP boot paths. To eliminate the dangerous
windows during the early initialization where the cpu_tasks remains
uninitialized, set_current() is invoked as early as possible in both
setup_arch() and start_secondary(). This ensures the $tp recovery
barrier is armed in case any early boot exceptions or kernel panics
occur.
Modify SAVE_SOME and handle_syscall to restore the $tp register from
cpu_tasks, and also use the la_abs absolute addressing for cpu_tasks
access in assembly to bypass the relocation limits within exception
handling sections. By advancing the preservation of u0 in SAVE_SOME,
we reuse the PERCPU_BASE_KS value in u0 for the cpu_tasks calculation,
effectively eliminating a duplicate csrrd instruction execution on SMP
platforms.
Update <asm/switch_to.h> and <kernel/switch.S> to fully support the
CONFIG_THREAD_INFO_IN_TASK feature.
Remove the obsolete next_ti argument from __switch_to(), which shifts
the remaining arguments ahead in the calling convention (sched_ra from
a3 to a2, and sched_cfa from a4 to a3). Under the new configuration,
__switch_to() now directly derives the thread pointer ($tp) from the
next task_struct pointer in a1.
To preserve the optimal and clean "move tp, a1" path for 64-bit kernels,
the thread pointer ($tp) is assigned directly from a1 in the core path.
For 32-bit kernels, where a1 carries a 2000-byte structural pointer bias
at entry, an explicit adjustment "PTR_ADDI tp, tp, -TASK_STRUCT_OFFSET"
is introduced at the function exit.
In the context of __switch_to(), local interrupts are disabled, and the
kernel is in a critical switching phase where handling any synchronous
exception is practically impossible and prohibited.
If any synchronous exception or watchpoint does trigger in this narrow
window, it constitutes a fatal double fault and the kernel is expected
to die/panic immediately anyway. Therefore, the temporary biased value
in $tp is safe and acceptable here.
Additionally, evaluate the stack lookup as a single load instruction
"LONG_LPTR t0, a1, (TASK_STACK - TASK_STRUCT_OFFSET)", this perfectly
satisfies both 32-bit and 64-bit kernels. Using the "next" pointer in
a1 as the base register, rather than $tp, effectively unchains the data
dependency (RAW hazard) from the preceding move instruction, maximizing
the instruction-level parallelism and superscalar execution efficiency
while naturally adapting the structural shift.
With CONFIG_THREAD_INFO_IN_TASK enabled, the kernel stack life cycle is
decoupled from task_struct and can be freed concurrently.
Currently, show_stacktrace() reads raw stack data via __get_addr() and
subsequently calls show_backtrace() to unwind the frame, without holding
any reference to the target task's stack. If show_stacktrace() is called
on a concurrently exiting task, it could attempt to read from a freed or
reallocated kernel stack. This introduces a severe use-after-free (UAF)
read risk or kernel panics.
Wrap the entire stack inspection process inside show_stacktrace() with
a try_get_task_stack() and put_task_stack() pair. This ensures the task
stack remains pinned safely during both the raw stack data dump loop and
the subsequent stack unwinding phase.
Also, ensure that the task pointer is initialized to "current" early if
it is NULL, so that try_get_task_stack() always operates on a valid task
reference.
Signed-off-by: Tiezhu Yang <yangtiezhu@loongson.cn>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
| arm64: | ok |
| csky: | TODO |
| hexagon: | TODO |
- | loongarch: | TODO |
+ | loongarch: | ok |
| m68k: | TODO |
| microblaze: | TODO |
| mips: | TODO |
select SYSCTL_ARCH_UNALIGN_NO_WARN
select SYSCTL_EXCEPTION_TRACE
select SWIOTLB if 64BIT
+ select THREAD_INFO_IN_TASK
select TRACE_IRQFLAGS_SUPPORT
select USE_PERCPU_NUMA_NODE_ID
select USER_STACKTRACE_SUPPORT
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_LOONGARCH_CURRENT_H
+#define __ASM_LOONGARCH_CURRENT_H
+
+#include <linux/compiler.h>
+
+#ifndef __ASSEMBLER__
+
+#include <asm/percpu.h>
+
+struct task_struct;
+
+DECLARE_PER_CPU(struct task_struct *, cpu_tasks);
+
+register struct task_struct *current_thread_pointer __asm__("$tp");
+
+static __always_inline struct task_struct *get_current(void)
+{
+ return current_thread_pointer;
+}
+
+#define current get_current()
+
+static __always_inline void set_current(struct task_struct *task)
+{
+ __this_cpu_write(cpu_tasks, task);
+}
+
+#endif /* __ASSEMBLER__ */
+
+#endif /* __ASM_LOONGARCH_CURRENT_H */
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H
-#include <asm/cmpxchg.h>
#include <asm/loongarch.h>
/*
struct seq_file;
struct secondary_data {
+ unsigned long task;
unsigned long stack;
- unsigned long thread_info;
+ unsigned long offset;
};
extern struct secondary_data cpuboot_data;
.cfi_rel_offset ra, PT_ERA
.endif
cfi_st tp, PT_R2, \docfi
+ cfi_st u0, PT_R21, \docfi
cfi_st fp, PT_R22, \docfi
/* Set thread_info if we're coming from user mode */
andi t0, t0, 0x3 /* extract pplv bit */
beqz t0, 9f
- LONG_LI tp, ~_THREAD_MASK
- and tp, tp, sp
- cfi_st u0, PT_R21, \docfi
csrrd u0, PERCPU_BASE_KS
+
+ la_abs t1, cpu_tasks
+#ifdef CONFIG_SMP
+ LONG_ADD t1, t1, u0
+#endif
+ LONG_L tp, t1, 0
9:
#ifdef CONFIG_KGDB
li.w t0, CSR_CRMD_WE
* __switch_to - switch execution of a task
* @prev: The task previously executed.
* @next: The task to begin executing.
- * @next_ti: task_thread_info(next).
* @sched_ra: __schedule return address.
* @sched_cfa: __schedule call frame address.
*
* the context of next. Returns prev.
*/
extern asmlinkage struct task_struct *__switch_to(struct task_struct *prev,
- struct task_struct *next, struct thread_info *next_ti,
- void *sched_ra, void *sched_cfa);
+ struct task_struct *next, void *sched_ra, void *sched_cfa);
/*
* For newly created kernel threads switch_to() will return to
lose_fpu_inatomic(1, prev); \
lose_lbt_inatomic(1, prev); \
hw_breakpoint_thread_switch(next); \
- (last) = __switch_to(prev, next, task_thread_info(next), \
+ set_current(next); \
+ (last) = __switch_to(prev, next, \
__builtin_return_address(0), __builtin_frame_address(0)); \
} while (0)
* must also be changed
*/
struct thread_info {
- struct task_struct *task; /* main task structure */
unsigned long flags; /* low level flags */
unsigned long tp_value; /* thread pointer */
__u32 cpu; /* current CPU */
*/
#define INIT_THREAD_INFO(tsk) \
{ \
- .task = &tsk, \
.flags = _TIF_FIXADE, \
.cpu = 0, \
.preempt_count = INIT_PREEMPT_COUNT, \
}
-/* How to get the thread information struct from C. */
-register struct thread_info *__current_thread_info __asm__("$tp");
-
-static inline struct thread_info *current_thread_info(void)
-{
- return __current_thread_info;
-}
-
register unsigned long current_stack_pointer __asm__("$sp");
#endif /* !__ASSEMBLER__ */
{
COMMENT("LoongArch task_struct offsets.");
OFFSET(TASK_STATE, task_struct, __state);
- OFFSET(TASK_THREAD_INFO, task_struct, stack);
+ OFFSET(TASK_STACK, task_struct, stack);
OFFSET(TASK_FLAGS, task_struct, flags);
OFFSET(TASK_MM, task_struct, mm);
OFFSET(TASK_PID, task_struct, pid);
static void __used output_thread_info_defines(void)
{
COMMENT("LoongArch thread_info offsets.");
- OFFSET(TI_TASK, thread_info, task);
OFFSET(TI_FLAGS, thread_info, flags);
OFFSET(TI_TP_VALUE, thread_info, tp_value);
OFFSET(TI_CPU, thread_info, cpu);
static void __used output_smpboot_defines(void)
{
COMMENT("Linux smp cpu boot offsets.");
+ OFFSET(CPU_BOOT_TASK, secondary_data, task);
OFFSET(CPU_BOOT_STACK, secondary_data, stack);
- OFFSET(CPU_BOOT_TINFO, secondary_data, thread_info);
+ OFFSET(CPU_BOOT_OFFSET, secondary_data, offset);
BLANK();
}
#endif
#endif
move u0, t0
- LONG_LI tp, ~_THREAD_MASK
- and tp, tp, sp
+ la_abs t1, cpu_tasks
+#ifdef CONFIG_SMP
+ LONG_ADD t1, t1, u0
+#endif
+ LONG_L tp, t1, 0
move a0, sp
bl do_syscall
/* GPR21 used for percpu base (runtime), initialized as 0 */
move u0, zero
- la.pcrel tp, init_thread_union
- /* Set the SP after an empty pt_regs. */
- PTR_LI sp, (_THREAD_SIZE - PT_SIZE)
- PTR_ADD sp, sp, tp
+ la.pcrel tp, init_task
+ la.pcrel t0, init_stack
+ PTR_LI t1, _THREAD_SIZE
+ PTR_ADD t0, t0, t1
+ PTR_ADDI sp, t0, -PT_SIZE
set_saved_sp sp, t0, t1
#ifdef CONFIG_RELOCATABLE
#ifdef CONFIG_RANDOMIZE_BASE
/* Repoint the sp into the new kernel */
- PTR_LI sp, (_THREAD_SIZE - PT_SIZE)
- PTR_ADD sp, sp, tp
+ LONG_LPTR t0, tp, TASK_STACK
+ PTR_LI t1, _THREAD_SIZE
+ PTR_ADD t0, t0, t1
+ PTR_ADDI sp, t0, -PT_SIZE
set_saved_sp sp, t0, t1
/* Jump to the new kernel: new_pc = current_pc + random_offset */
csrxchg t0, t1, LOONGARCH_CSR_IMPCTL1
#endif
la.pcrel t0, cpuboot_data
+ ld.d tp, t0, CPU_BOOT_TASK
ld.d sp, t0, CPU_BOOT_STACK
- ld.d tp, t0, CPU_BOOT_TINFO
+ ld.d u0, t0, CPU_BOOT_OFFSET
bl start_secondary
ASM_BUG()
EXPORT_SYMBOL(__stack_chk_guard);
#endif
+DEFINE_PER_CPU(struct task_struct *, cpu_tasks);
+
/*
* Idle related variables and functions
*/
reloc_offset += random_offset;
/* The current thread is now within the relocated kernel */
- __current_thread_info = RELOCATED_KASLR(__current_thread_info);
+ current_thread_pointer = RELOCATED_KASLR(current_thread_pointer);
update_reloc_offset(&reloc_offset, random_offset);
}
{
cpu_probe();
unwind_init();
+ set_current(current);
init_environ();
efi_init();
pr_info("Booting CPU#%d...\n", cpu);
entry = __pa_symbol((unsigned long)&smpboot_entry);
- cpuboot_data.stack = (unsigned long)__KSTK_TOS(idle);
- cpuboot_data.thread_info = (unsigned long)task_thread_info(idle);
+ cpuboot_data.task = (unsigned long)idle;
+ cpuboot_data.stack = (unsigned long)task_pt_regs(idle);
+ cpuboot_data.offset = per_cpu_offset(cpu);
csr_mail_send(entry, cpu_logical_map(cpu), 0);
set_my_cpu_offset(per_cpu_offset(cpu));
cpu_probe();
+ set_current(current);
constant_clockevent_init();
loongson_init_secondary();
/*
* task_struct *__switch_to(task_struct *prev, task_struct *next,
- * struct thread_info *next_ti, void *sched_ra, void *sched_cfa)
+ * void *sched_ra, void *sched_cfa)
*/
.align 5
SYM_FUNC_START(__switch_to)
LONG_SPTR t1, a0, (THREAD_CSRPRMD - TASK_STRUCT_OFFSET)
cpu_save_nonscratch a0
- LONG_SPTR a3, a0, (THREAD_SCHED_RA - TASK_STRUCT_OFFSET)
- LONG_SPTR a4, a0, (THREAD_SCHED_CFA - TASK_STRUCT_OFFSET)
+ LONG_SPTR a2, a0, (THREAD_SCHED_RA - TASK_STRUCT_OFFSET)
+ LONG_SPTR a3, a0, (THREAD_SCHED_CFA - TASK_STRUCT_OFFSET)
#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_SMP)
la t7, __stack_chk_guard
LONG_SPTR t8, t7, 0
#endif
- move tp, a2
+ move tp, a1
cpu_restore_nonscratch a1
- li.w t0, _THREAD_SIZE
- PTR_ADD t0, t0, tp
+ LONG_LPTR t0, a1, (TASK_STACK - TASK_STRUCT_OFFSET)
+ PTR_LI t1, _THREAD_SIZE
+ PTR_ADD t0, t0, t1
set_saved_sp t0, t1, t2
LONG_LPTR t1, a1, (THREAD_CSRPRMD - TASK_STRUCT_OFFSET)
#ifdef CONFIG_32BIT
PTR_ADDI a0, a0, -TASK_STRUCT_OFFSET
+ PTR_ADDI tp, tp, -TASK_STRUCT_OFFSET
#endif
jr ra
SYM_FUNC_END(__switch_to)
unsigned long stackdata;
unsigned long *sp = (unsigned long *)regs->regs[3];
+ if (!task)
+ task = current;
+
+ if (!try_get_task_stack(task))
+ return;
+
printk("%sStack :", loglvl);
i = 0;
while ((unsigned long) sp & (PAGE_SIZE - 1)) {
}
pr_cont("\n");
show_backtrace(task, regs, loglvl, user);
+
+ put_task_stack(task);
}
void show_stack(struct task_struct *task, unsigned long *sp, const char *loglvl)
projects / thirdparty / kernel / stable.git / commitdiff
