Merge tag 'objtool-core-2023-04-27' of git://git.kernel.org/pub/scm/linux/kernel...

[thirdparty/linux.git] / arch / x86 / kernel / head_64.S

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 8d8d25c648274191667d31a5cfa325fe6a594fc1..a5df3e994f04f10f2c9fb814bf6f952a6452ae7a 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -61,23 +61,15 @@ SYM_CODE_START_NOALIGN(startup_64)
         * tables and then reload them.
         */
 
-       /* Set up the stack for verify_cpu(), similar to initial_stack below */
-       leaq    (__end_init_task - FRAME_SIZE)(%rip), %rsp
+       /* Set up the stack for verify_cpu() */
+       leaq    (__end_init_task - PTREGS_SIZE)(%rip), %rsp
 
        leaq    _text(%rip), %rdi
 
-       /*
-        * initial_gs points to initial fixed_percpu_data struct with storage for
-        * the stack protector canary. Global pointer fixups are needed at this
-        * stage, so apply them as is done in fixup_pointer(), and initialize %gs
-        * such that the canary can be accessed at %gs:40 for subsequent C calls.
-        */
+       /* Setup GSBASE to allow stack canary access for C code */
        movl    $MSR_GS_BASE, %ecx
-       movq    initial_gs(%rip), %rax
-       movq    $_text, %rdx
-       subq    %rdx, %rax
-       addq    %rdi, %rax
-       movq    %rax, %rdx
+       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
+       movl    %edx, %eax
        shrq    $32,  %rdx
        wrmsr
 
@@ -241,13 +233,36 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        UNWIND_HINT_END_OF_STACK
        ANNOTATE_NOENDBR // above
 
+#ifdef CONFIG_SMP
+       movl    smpboot_control(%rip), %ecx
+
+       /* Get the per cpu offset for the given CPU# which is in ECX */
+       movq    __per_cpu_offset(,%rcx,8), %rdx
+#else
+       xorl    %edx, %edx /* zero-extended to clear all of RDX */
+#endif /* CONFIG_SMP */
+
+       /*
+        * Setup a boot time stack - Any secondary CPU will have lost its stack
+        * by now because the cr3-switch above unmaps the real-mode stack.
+        *
+        * RDX contains the per-cpu offset
+        */
+       movq    pcpu_hot + X86_current_task(%rdx), %rax
+       movq    TASK_threadsp(%rax), %rsp
+
        /*
         * We must switch to a new descriptor in kernel space for the GDT
         * because soon the kernel won't have access anymore to the userspace
         * addresses where we're currently running on. We have to do that here
         * because in 32bit we couldn't load a 64bit linear address.
         */
-       lgdt    early_gdt_descr(%rip)
+       subq    $16, %rsp
+       movw    $(GDT_SIZE-1), (%rsp)
+       leaq    gdt_page(%rdx), %rax
+       movq    %rax, 2(%rsp)
+       lgdt    (%rsp)
+       addq    $16, %rsp
 
        /* set up data segments */
        xorl %eax,%eax
@@ -271,16 +286,13 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
         * the per cpu areas are set up.
         */
        movl    $MSR_GS_BASE,%ecx
-       movl    initial_gs(%rip),%eax
-       movl    initial_gs+4(%rip),%edx
+#ifndef CONFIG_SMP
+       leaq    INIT_PER_CPU_VAR(fixed_percpu_data)(%rip), %rdx
+#endif
+       movl    %edx, %eax
+       shrq    $32, %rdx
        wrmsr
 
-       /*
-        * Setup a boot time stack - Any secondary CPU will have lost its stack
-        * by now because the cr3-switch above unmaps the real-mode stack
-        */
-       movq initial_stack(%rip), %rsp
-
        /* Setup and Load IDT */
        pushq   %rsi
        call    early_setup_idt
@@ -372,7 +384,11 @@ SYM_CODE_END(secondary_startup_64)
 SYM_CODE_START(start_cpu0)
        ANNOTATE_NOENDBR
        UNWIND_HINT_END_OF_STACK
-       movq    initial_stack(%rip), %rsp
+
+       /* Find the idle task stack */
+       movq    PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
+       movq    TASK_threadsp(%rcx), %rsp
+
        jmp     .Ljump_to_C_code
 SYM_CODE_END(start_cpu0)
 #endif
@@ -414,16 +430,9 @@ SYM_CODE_END(vc_boot_ghcb)
        __REFDATA
        .balign 8
 SYM_DATA(initial_code, .quad x86_64_start_kernel)
-SYM_DATA(initial_gs,   .quad INIT_PER_CPU_VAR(fixed_percpu_data))
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 SYM_DATA(initial_vc_handler,   .quad handle_vc_boot_ghcb)
 #endif
-
-/*
- * The FRAME_SIZE gap is a convention which helps the in-kernel unwinder
- * reliably detect the end of the stack.
- */
-SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE)
        __FINITDATA
 
        __INIT
@@ -652,8 +661,7 @@ SYM_DATA_END(level1_fixmap_pgt)
        .data
        .align 16
 
-SYM_DATA(early_gdt_descr,              .word GDT_ENTRIES*8-1)
-SYM_DATA_LOCAL(early_gdt_descr_base,   .quad INIT_PER_CPU_VAR(gdt_page))
+SYM_DATA(smpboot_control,              .long 0)
 
        .align 16
 /* This must match the first entry in level2_kernel_pgt */