From: Ingo Molnar Date: Wed, 9 Apr 2025 21:11:22 +0000 (+0200) Subject: x86/fpu: Make task_struct::thread constant size X-Git-Tag: v6.16-rc1~195^2~28^2~27 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=cb7ca40a3882360ce87191793449d48df0b29184;p=thirdparty%2Flinux.git x86/fpu: Make task_struct::thread constant size Turn thread.fpu into a pointer. Since most FPU code internals work by passing around the FPU pointer already, the code generation impact is small. This allows us to remove the old kludge of task_struct being variable size: struct task_struct { ... /* * New fields for task_struct should be added above here, so that * they are included in the randomized portion of task_struct. */ randomized_struct_fields_end /* CPU-specific state of this task: */ struct thread_struct thread; /* * WARNING: on x86, 'thread_struct' contains a variable-sized * structure. It *MUST* be at the end of 'task_struct'. * * Do not put anything below here! */ }; ... which creates a number of problems, such as requiring thread_struct to be the last member of the struct - not allowing it to be struct-randomized, etc. But the primary motivation is to allow the decoupling of task_struct from hardware details ( in particular), and to eventually allow the per-task infrastructure: DECLARE_PER_TASK(type, name); ... per_task(current, name) = val; ... which requires task_struct to be a constant size struct. The fpu_thread_struct_whitelist() quirk to hardened usercopy can be removed, now that the FPU structure is not embedded in the task struct anymore, which reduces text footprint a bit. Fixed-by: Oleg Nesterov Signed-off-by: Ingo Molnar Cc: Andy Lutomirski Cc: Brian Gerst Cc: Chang S. Bae Cc: H. Peter Anvin Cc: Linus Torvalds Link: https://lore.kernel.org/r/20250409211127.3544993-4-mingo@kernel.org --- diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 2f631e0adea34..5ea7e5d2c4de0 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -516,21 +516,19 @@ struct thread_struct { #endif /* Floating point and extended processor state */ - struct fpu fpu; - /* - * WARNING: 'fpu' is dynamically-sized. It *MUST* be at - * the end. - */ + struct fpu *fpu; }; -#define x86_task_fpu(task) (&(task)->thread.fpu) - -extern void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size); +#define x86_task_fpu(task) ((task)->thread.fpu) -static inline void arch_thread_struct_whitelist(unsigned long *offset, - unsigned long *size) +/* + * X86 doesn't need any embedded-FPU-struct quirks: + */ +static inline void +arch_thread_struct_whitelist(unsigned long *offset, unsigned long *size) { - fpu_thread_struct_whitelist(offset, size); + *offset = 0; + *size = 0; } static inline void diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c index dc6d7f93c4465..853a738fdf2d3 100644 --- a/arch/x86/kernel/fpu/core.c +++ b/arch/x86/kernel/fpu/core.c @@ -593,8 +593,19 @@ static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp) int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, unsigned long ssp) { + /* + * We allocate the new FPU structure right after the end of the task struct. + * task allocation size already took this into account. + * + * This is safe because task_struct size is a multiple of cacheline size. + */ struct fpu *src_fpu = x86_task_fpu(current); - struct fpu *dst_fpu = x86_task_fpu(dst); + struct fpu *dst_fpu = (void *)dst + sizeof(*dst); + + BUILD_BUG_ON(sizeof(*dst) % SMP_CACHE_BYTES != 0); + BUG_ON(!src_fpu); + + dst->thread.fpu = dst_fpu; /* The new task's FPU state cannot be valid in the hardware. */ dst_fpu->last_cpu = -1; @@ -663,16 +674,6 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal, return 0; } -/* - * Whitelist the FPU register state embedded into task_struct for hardened - * usercopy. - */ -void fpu_thread_struct_whitelist(unsigned long *offset, unsigned long *size) -{ - *offset = offsetof(struct thread_struct, fpu.__fpstate.regs); - *size = fpu_kernel_cfg.default_size; -} - /* * Drops current FPU state: deactivates the fpregs and * the fpstate. NOTE: it still leaves previous contents diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c index ad5cb2943d37a..848ea79886bac 100644 --- a/arch/x86/kernel/fpu/init.c +++ b/arch/x86/kernel/fpu/init.c @@ -71,8 +71,15 @@ static bool __init fpu__probe_without_cpuid(void) return fsw == 0 && (fcw & 0x103f) == 0x003f; } +static struct fpu x86_init_fpu __attribute__ ((aligned (64))) __read_mostly; + static void __init fpu__init_system_early_generic(void) { + fpstate_reset(&x86_init_fpu); + current->thread.fpu = &x86_init_fpu; + set_thread_flag(TIF_NEED_FPU_LOAD); + x86_init_fpu.last_cpu = -1; + if (!boot_cpu_has(X86_FEATURE_CPUID) && !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) { if (fpu__probe_without_cpuid()) @@ -150,6 +157,8 @@ static void __init fpu__init_task_struct_size(void) { int task_size = sizeof(struct task_struct); + task_size += sizeof(struct fpu); + /* * Subtract off the static size of the register state. * It potentially has a bunch of padding. @@ -164,14 +173,9 @@ static void __init fpu__init_task_struct_size(void) /* * We dynamically size 'struct fpu', so we require that - * it be at the end of 'thread_struct' and that - * 'thread_struct' be at the end of 'task_struct'. If - * you hit a compile error here, check the structure to - * see if something got added to the end. + * 'state' be at the end of 'it: */ CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate); - CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu); - CHECK_MEMBER_AT_END_OF(struct task_struct, thread); arch_task_struct_size = task_size; } @@ -213,7 +217,6 @@ static void __init fpu__init_system_xstate_size_legacy(void) */ void __init fpu__init_system(void) { - fpstate_reset(x86_task_fpu(current)); fpu__init_system_early_generic(); /* diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 47694e3915066..3ce4cce46f3f5 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -103,7 +103,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) dst->thread.vm86 = NULL; #endif /* Drop the copied pointer to current's fpstate */ - x86_task_fpu(dst)->fpstate = NULL; + dst->thread.fpu = NULL; return 0; } diff --git a/include/linux/sched.h b/include/linux/sched.h index f96ac19828934..4ecc0c6b1cb0f 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1646,22 +1646,15 @@ struct task_struct { struct user_event_mm *user_event_mm; #endif - /* - * New fields for task_struct should be added above here, so that - * they are included in the randomized portion of task_struct. - */ - randomized_struct_fields_end - /* CPU-specific state of this task: */ struct thread_struct thread; /* - * WARNING: on x86, 'thread_struct' contains a variable-sized - * structure. It *MUST* be at the end of 'task_struct'. - * - * Do not put anything below here! + * New fields for task_struct should be added above here, so that + * they are included in the randomized portion of task_struct. */ -}; + randomized_struct_fields_end +} __attribute__ ((aligned (64))); #define TASK_REPORT_IDLE (TASK_REPORT + 1) #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)