x86/percpu: Move current_task to percpu hot section

No functional change.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://lore.kernel.org/r/20250303165246.2175811-10-brgerst@gmail.com

diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 3d1b123c2ee32b9688db0f8ddccdf0d1731e34ca..dea7d8b854f0e137a5f7b2e7f20604381525c458 100644 (file)
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -12,22 +12,17 @@
 
 struct task_struct;
 
-struct pcpu_hot {
-       struct task_struct      *current_task;
-};
-
-DECLARE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot);
-
-/* const-qualified alias to pcpu_hot, aliased by linker. */
-DECLARE_PER_CPU_CACHE_HOT(const struct pcpu_hot __percpu_seg_override,
-                       const_pcpu_hot);
+DECLARE_PER_CPU_CACHE_HOT(struct task_struct *, current_task);
+/* const-qualified alias provided by the linker. */
+DECLARE_PER_CPU_CACHE_HOT(struct task_struct * const __percpu_seg_override,
+                         const_current_task);
 
 static __always_inline struct task_struct *get_current(void)
 {
        if (IS_ENABLED(CONFIG_USE_X86_SEG_SUPPORT))
-               return this_cpu_read_const(const_pcpu_hot.current_task);
+               return this_cpu_read_const(const_current_task);
 
-       return this_cpu_read_stable(pcpu_hot.current_task);
+       return this_cpu_read_stable(current_task);
 }
 
 #define current get_current()

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 41517a7f7f1c59f0f543b2df0d9f50bb542e935b..8cc9548205b002bd63cf250f17af7bbf800d5a17 100644 (file)
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -551,7 +551,7 @@ do {                                                                        \
  * it is accessed while this_cpu_read_stable() allows the value to be cached.
  * this_cpu_read_stable() is more efficient and can be used if its value
  * is guaranteed to be valid across CPUs.  The current users include
- * pcpu_hot.current_task and cpu_current_top_of_stack, both of which are
+ * current_task and cpu_current_top_of_stack, both of which are
  * actually per-thread variables implemented as per-CPU variables and
  * thus stable for the duration of the respective task.
  */

diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 54ace808defd221dcd411d50c39bf6ab715f2ff3..ad4ea6fb3b6cb9755f7ea783aea5fc2d095fdbba 100644 (file)
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -107,7 +107,6 @@ static void __used common(void)
        OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
        OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
        OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
-       OFFSET(X86_current_task, pcpu_hot, current_task);
 #if IS_ENABLED(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64)
        /* Offset for fields in aria_ctx */
        BLANK();

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 51653e01a7161fe64327424fecef46d6078de128..3f45fdd48bdb91d6cdeb93edeb4a721528ef0c72 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2064,11 +2064,9 @@ static __init int setup_setcpuid(char *arg)
 }
 __setup("setcpuid=", setup_setcpuid);
 
-DEFINE_PER_CPU_CACHE_HOT(struct pcpu_hot, pcpu_hot) = {
-       .current_task   = &init_task,
-};
-EXPORT_PER_CPU_SYMBOL(pcpu_hot);
-EXPORT_PER_CPU_SYMBOL(const_pcpu_hot);
+DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
+EXPORT_PER_CPU_SYMBOL(current_task);
+EXPORT_PER_CPU_SYMBOL(const_current_task);
 
 DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
 EXPORT_PER_CPU_SYMBOL(__preempt_count);

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 2843b0a56198da25ba687a2e350d16f73a767ac5..fefe2a25cf025415c1d202081c1616e1182abbad 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -322,7 +322,7 @@ SYM_INNER_LABEL(common_startup_64, SYM_L_LOCAL)
         *
         * RDX contains the per-cpu offset
         */
-       movq    pcpu_hot + X86_current_task(%rdx), %rax
+       movq    current_task(%rdx), %rax
        movq    TASK_threadsp(%rax), %rsp
 
        /*
@@ -433,7 +433,7 @@ SYM_CODE_START(soft_restart_cpu)
        UNWIND_HINT_END_OF_STACK
 
        /* Find the idle task stack */
-       movq    PER_CPU_VAR(pcpu_hot + X86_current_task), %rcx
+       movq    PER_CPU_VAR(current_task), %rcx
        movq    TASK_threadsp(%rcx), %rsp
 
        jmp     .Ljump_to_C_code

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 8ec44acb863be9cb8a165789df13833ab618f12a..4636ef3599737c1e810ee421d87c575305f690ce 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -206,7 +206,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        if (prev->gs | next->gs)
                loadsegment(gs, next->gs);
 
-       raw_cpu_write(pcpu_hot.current_task, next_p);
+       raw_cpu_write(current_task, next_p);
 
        switch_fpu_finish(next_p);
 

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index d8f4bcef8ee4a24411b22c01470a99438f448f52..7196ca7048be0cfbb57d13f0f142ea4f90b48634 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -668,7 +668,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
        /*
         * Switch the PDA and FPU contexts.
         */
-       raw_cpu_write(pcpu_hot.current_task, next_p);
+       raw_cpu_write(current_task, next_p);
        raw_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
 
        switch_fpu_finish(next_p);

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c3a26e60e3c40780a5aa65facb215888557bea00..42ca131bebbc33c2eaf689b89e89eacd7181211a 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -822,7 +822,7 @@ int common_cpu_up(unsigned int cpu, struct task_struct *idle)
        /* Just in case we booted with a single CPU. */
        alternatives_enable_smp();
 
-       per_cpu(pcpu_hot.current_task, cpu) = idle;
+       per_cpu(current_task, cpu) = idle;
        cpu_init_stack_canary(cpu, idle);
 
        /* Initialize the interrupt stack(s) */

diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 475f6717f27ad36dad42f710147dc42cf5f3aea4..31f9102b107f156b8e1f5567e960b73f6a97b9f6 100644 (file)
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -43,7 +43,7 @@ ENTRY(phys_startup_64)
 #endif
 
 jiffies = jiffies_64;
-const_pcpu_hot = pcpu_hot;
+const_current_task = current_task;
 const_cpu_current_top_of_stack = cpu_current_top_of_stack;
 
 #if defined(CONFIG_X86_64)

diff --git a/scripts/gdb/linux/cpus.py b/scripts/gdb/linux/cpus.py
index 13eb8b3901b8fc5e2500138c7d204ac5f7c9858c..8f7c4fb78c2ccd944c12d7307a9dd6564ff08d04 100644 (file)
--- a/scripts/gdb/linux/cpus.py
+++ b/scripts/gdb/linux/cpus.py
@@ -164,7 +164,7 @@ def get_current_task(cpu):
             var_ptr = gdb.parse_and_eval("(struct task_struct *)cpu_tasks[0].task")
             return var_ptr.dereference()
         else:
-            var_ptr = gdb.parse_and_eval("&pcpu_hot.current_task")
+            var_ptr = gdb.parse_and_eval("&current_task")
             return per_cpu(var_ptr, cpu).dereference()
     elif utils.is_target_arch("aarch64"):
         current_task_addr = gdb.parse_and_eval("(unsigned long)$SP_EL0")