--- /dev/null
+From 8217ad0a435ff06d651d7298ea8ae8d72388179e Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:15 +0200
+Subject: decompress: Use 8 byte alignment
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 8217ad0a435ff06d651d7298ea8ae8d72388179e upstream.
+
+The ZSTD decompressor requires malloc() allocations to be 8 byte
+aligned, so ensure that this the case.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-19-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ include/linux/decompress/mm.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+--- a/include/linux/decompress/mm.h
++++ b/include/linux/decompress/mm.h
+@@ -48,7 +48,7 @@ MALLOC_VISIBLE void *malloc(int size)
+ if (!malloc_ptr)
+ malloc_ptr = free_mem_ptr;
+
+- malloc_ptr = (malloc_ptr + 3) & ~3; /* Align */
++ malloc_ptr = (malloc_ptr + 7) & ~7; /* Align */
+
+ p = (void *)malloc_ptr;
+ malloc_ptr += size;
--- /dev/null
+From 9cf42bca30e98a1c6c9e8abf876940a551eaa3d1 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 2 Aug 2022 11:00:16 +0200
+Subject: efi: libstub: use EFI_LOADER_CODE region when moving the kernel in memory
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 9cf42bca30e98a1c6c9e8abf876940a551eaa3d1 upstream.
+
+The EFI spec is not very clear about which permissions are being given
+when allocating pages of a certain type. However, it is quite obvious
+that EFI_LOADER_CODE is more likely to permit execution than
+EFI_LOADER_DATA, which becomes relevant once we permit booting the
+kernel proper with the firmware's 1:1 mapping still active.
+
+Ostensibly, recent systems such as the Surface Pro X grant executable
+permissions to EFI_LOADER_CODE regions but not EFI_LOADER_DATA regions.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ drivers/firmware/efi/libstub/alignedmem.c | 5 +++--
+ drivers/firmware/efi/libstub/arm64-stub.c | 6 ++++--
+ drivers/firmware/efi/libstub/efistub.h | 6 ++++--
+ drivers/firmware/efi/libstub/mem.c | 3 ++-
+ drivers/firmware/efi/libstub/randomalloc.c | 5 +++--
+ 5 files changed, 16 insertions(+), 9 deletions(-)
+
+--- a/drivers/firmware/efi/libstub/alignedmem.c
++++ b/drivers/firmware/efi/libstub/alignedmem.c
+@@ -22,7 +22,8 @@
+ * Return: status code
+ */
+ efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr,
+- unsigned long max, unsigned long align)
++ unsigned long max, unsigned long align,
++ int memory_type)
+ {
+ efi_physical_addr_t alloc_addr;
+ efi_status_t status;
+@@ -36,7 +37,7 @@ efi_status_t efi_allocate_pages_aligned(
+ slack = align / EFI_PAGE_SIZE - 1;
+
+ status = efi_bs_call(allocate_pages, EFI_ALLOCATE_MAX_ADDRESS,
+- EFI_LOADER_DATA, size / EFI_PAGE_SIZE + slack,
++ memory_type, size / EFI_PAGE_SIZE + slack,
+ &alloc_addr);
+ if (status != EFI_SUCCESS)
+ return status;
+--- a/drivers/firmware/efi/libstub/arm64-stub.c
++++ b/drivers/firmware/efi/libstub/arm64-stub.c
+@@ -180,7 +180,8 @@ efi_status_t handle_kernel_image(unsigne
+ * locate the kernel at a randomized offset in physical memory.
+ */
+ status = efi_random_alloc(*reserve_size, min_kimg_align,
+- reserve_addr, phys_seed);
++ reserve_addr, phys_seed,
++ EFI_LOADER_CODE);
+ if (status != EFI_SUCCESS)
+ efi_warn("efi_random_alloc() failed: 0x%lx\n", status);
+ } else {
+@@ -201,7 +202,8 @@ efi_status_t handle_kernel_image(unsigne
+ }
+
+ status = efi_allocate_pages_aligned(*reserve_size, reserve_addr,
+- ULONG_MAX, min_kimg_align);
++ ULONG_MAX, min_kimg_align,
++ EFI_LOADER_CODE);
+
+ if (status != EFI_SUCCESS) {
+ efi_err("Failed to relocate kernel\n");
+--- a/drivers/firmware/efi/libstub/efistub.h
++++ b/drivers/firmware/efi/libstub/efistub.h
+@@ -880,7 +880,8 @@ void efi_get_virtmap(efi_memory_desc_t *
+ efi_status_t efi_get_random_bytes(unsigned long size, u8 *out);
+
+ efi_status_t efi_random_alloc(unsigned long size, unsigned long align,
+- unsigned long *addr, unsigned long random_seed);
++ unsigned long *addr, unsigned long random_seed,
++ int memory_type);
+
+ efi_status_t efi_random_get_seed(void);
+
+@@ -907,7 +908,8 @@ efi_status_t efi_allocate_pages(unsigned
+ unsigned long max);
+
+ efi_status_t efi_allocate_pages_aligned(unsigned long size, unsigned long *addr,
+- unsigned long max, unsigned long align);
++ unsigned long max, unsigned long align,
++ int memory_type);
+
+ efi_status_t efi_low_alloc_above(unsigned long size, unsigned long align,
+ unsigned long *addr, unsigned long min);
+--- a/drivers/firmware/efi/libstub/mem.c
++++ b/drivers/firmware/efi/libstub/mem.c
+@@ -91,7 +91,8 @@ efi_status_t efi_allocate_pages(unsigned
+
+ if (EFI_ALLOC_ALIGN > EFI_PAGE_SIZE)
+ return efi_allocate_pages_aligned(size, addr, max,
+- EFI_ALLOC_ALIGN);
++ EFI_ALLOC_ALIGN,
++ EFI_LOADER_DATA);
+
+ alloc_addr = ALIGN_DOWN(max + 1, EFI_ALLOC_ALIGN) - 1;
+ status = efi_bs_call(allocate_pages, EFI_ALLOCATE_MAX_ADDRESS,
+--- a/drivers/firmware/efi/libstub/randomalloc.c
++++ b/drivers/firmware/efi/libstub/randomalloc.c
+@@ -53,7 +53,8 @@ static unsigned long get_entry_num_slots
+ efi_status_t efi_random_alloc(unsigned long size,
+ unsigned long align,
+ unsigned long *addr,
+- unsigned long random_seed)
++ unsigned long random_seed,
++ int memory_type)
+ {
+ unsigned long total_slots = 0, target_slot;
+ unsigned long total_mirrored_slots = 0;
+@@ -118,7 +119,7 @@ efi_status_t efi_random_alloc(unsigned l
+ pages = size / EFI_PAGE_SIZE;
+
+ status = efi_bs_call(allocate_pages, EFI_ALLOCATE_ADDRESS,
+- EFI_LOADER_DATA, pages, &target);
++ memory_type, pages, &target);
+ if (status == EFI_SUCCESS)
+ *addr = target;
+ break;
--- /dev/null
+From bad267f9e18f8e9e628abd1811d2899b1735a4e1 Mon Sep 17 00:00:00 2001
+From: Johan Hovold <johan+linaro@kernel.org>
+Date: Thu, 19 Jan 2023 17:42:54 +0100
+Subject: efi: verify that variable services are supported
+
+From: Johan Hovold <johan+linaro@kernel.org>
+
+commit bad267f9e18f8e9e628abd1811d2899b1735a4e1 upstream.
+
+Current Qualcomm UEFI firmware does not implement the variable services
+but not all revisions clear the corresponding bits in the RT_PROP table
+services mask and instead the corresponding calls return
+EFI_UNSUPPORTED.
+
+This leads to efi core registering the generic efivar ops even when the
+variable services are not supported or when they are accessed through
+some other interface (e.g. Google SMI or the upcoming Qualcomm SCM
+implementation).
+
+Instead of playing games with init call levels to make sure that the
+custom implementations are registered after the generic one, make sure
+that get_next_variable() is actually supported before registering the
+generic ops.
+
+Signed-off-by: Johan Hovold <johan+linaro@kernel.org>
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ drivers/firmware/efi/efi.c | 22 ++++++++++++++++++++++
+ 1 file changed, 22 insertions(+)
+
+--- a/drivers/firmware/efi/efi.c
++++ b/drivers/firmware/efi/efi.c
+@@ -185,8 +185,27 @@ static const struct attribute_group efi_
+ static struct efivars generic_efivars;
+ static struct efivar_operations generic_ops;
+
++static bool generic_ops_supported(void)
++{
++ unsigned long name_size;
++ efi_status_t status;
++ efi_char16_t name;
++ efi_guid_t guid;
++
++ name_size = sizeof(name);
++
++ status = efi.get_next_variable(&name_size, &name, &guid);
++ if (status == EFI_UNSUPPORTED)
++ return false;
++
++ return true;
++}
++
+ static int generic_ops_register(void)
+ {
++ if (!generic_ops_supported())
++ return 0;
++
+ generic_ops.get_variable = efi.get_variable;
+ generic_ops.get_next_variable = efi.get_next_variable;
+ generic_ops.query_variable_store = efi_query_variable_store;
+@@ -200,6 +219,9 @@ static int generic_ops_register(void)
+
+ static void generic_ops_unregister(void)
+ {
++ if (!generic_ops.get_variable)
++ return;
++
+ efivars_unregister(&generic_efivars);
+ }
+
mptcp-fix-possible-deadlock-in-subflow-diag.patch
rdma-core-refactor-rdma_bind_addr.patch
rdma-core-update-cma-destination-address-on-rdma_resolve_addr.patch
+efi-libstub-use-efi_loader_code-region-when-moving-the-kernel-in-memory.patch
+x86-boot-compressed-rename-efi_thunk_64.s-to-efi-mixed.s.patch
+x86-boot-compressed-move-32-bit-entrypoint-code-into-.text-section.patch
+x86-boot-compressed-move-bootargs-parsing-out-of-32-bit-startup-code.patch
+x86-boot-compressed-move-efi32_pe_entry-into-.text-section.patch
+x86-boot-compressed-move-efi32_entry-out-of-head_64.s.patch
+x86-boot-compressed-move-efi32_pe_entry-out-of-head_64.s.patch
+x86-boot-compressed-efi-merge-multiple-definitions-of-image_offset-into-one.patch
+x86-boot-compressed-simplify-idt-gdt-preserve-restore-in-the-efi-thunk.patch
+x86-boot-compressed-avoid-touching-ecx-in-startup32_set_idt_entry.patch
+x86-boot-compressed-pull-global-variable-reference-into-startup32_load_idt.patch
+x86-boot-compressed-move-startup32_load_idt-into-.text-section.patch
+x86-boot-compressed-move-startup32_load_idt-out-of-head_64.s.patch
+x86-boot-compressed-move-startup32_check_sev_cbit-into-.text.patch
+x86-boot-compressed-move-startup32_check_sev_cbit-out-of-head_64.s.patch
+x86-boot-compressed-adhere-to-calling-convention-in-get_sev_encryption_bit.patch
+x86-boot-compressed-only-build-mem_encrypt.s-if-amd_mem_encrypt-y.patch
+efi-verify-that-variable-services-are-supported.patch
+x86-efi-make-the-deprecated-efi-handover-protocol-optional.patch
+x86-boot-robustify-calling-startup_-32-64-from-the-decompressor-code.patch
+x86-efistub-branch-straight-to-kernel-entry-point-from-c-code.patch
+x86-decompressor-store-boot_params-pointer-in-callee-save-register.patch
+x86-decompressor-assign-paging-related-global-variables-earlier.patch
+x86-decompressor-call-trampoline-as-a-normal-function.patch
+x86-decompressor-use-standard-calling-convention-for-trampoline.patch
+x86-decompressor-avoid-the-need-for-a-stack-in-the-32-bit-trampoline.patch
+x86-decompressor-call-trampoline-directly-from-c-code.patch
+x86-decompressor-only-call-the-trampoline-when-changing-paging-levels.patch
+x86-decompressor-pass-pgtable-address-to-trampoline-directly.patch
+x86-decompressor-merge-trampoline-cleanup-with-switching-code.patch
+x86-decompressor-move-global-symbol-references-to-c-code.patch
+decompress-use-8-byte-alignment.patch
--- /dev/null
+From 30c9ca16a5271ba6f8ad9c86507ff1c789c94677 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:15 +0100
+Subject: x86/boot/compressed: Adhere to calling convention in get_sev_encryption_bit()
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 30c9ca16a5271ba6f8ad9c86507ff1c789c94677 upstream.
+
+Make get_sev_encryption_bit() follow the ordinary i386 calling
+convention, and only call it if CONFIG_AMD_MEM_ENCRYPT is actually
+enabled. This clarifies the calling code, and makes it more
+maintainable.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-16-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 5 +++--
+ arch/x86/boot/compressed/mem_encrypt.S | 10 ----------
+ 2 files changed, 3 insertions(+), 12 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -180,12 +180,13 @@ SYM_FUNC_START(startup_32)
+ */
+ /*
+ * If SEV is active then set the encryption mask in the page tables.
+- * This will insure that when the kernel is copied and decompressed
++ * This will ensure that when the kernel is copied and decompressed
+ * it will be done so encrypted.
+ */
+- call get_sev_encryption_bit
+ xorl %edx, %edx
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
++ call get_sev_encryption_bit
++ xorl %edx, %edx
+ testl %eax, %eax
+ jz 1f
+ subl $32, %eax /* Encryption bit is always above bit 31 */
+--- a/arch/x86/boot/compressed/mem_encrypt.S
++++ b/arch/x86/boot/compressed/mem_encrypt.S
+@@ -18,12 +18,7 @@
+ .text
+ .code32
+ SYM_FUNC_START(get_sev_encryption_bit)
+- xor %eax, %eax
+-
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+ push %ebx
+- push %ecx
+- push %edx
+
+ movl $0x80000000, %eax /* CPUID to check the highest leaf */
+ cpuid
+@@ -54,12 +49,7 @@ SYM_FUNC_START(get_sev_encryption_bit)
+ xor %eax, %eax
+
+ .Lsev_exit:
+- pop %edx
+- pop %ecx
+ pop %ebx
+-
+-#endif /* CONFIG_AMD_MEM_ENCRYPT */
+-
+ RET
+ SYM_FUNC_END(get_sev_encryption_bit)
+
--- /dev/null
+From 6aac80a8da46d70f2ae7ff97c9f45a15c7c9b3ef Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:09 +0100
+Subject: x86/boot/compressed: Avoid touching ECX in startup32_set_idt_entry()
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 6aac80a8da46d70f2ae7ff97c9f45a15c7c9b3ef upstream.
+
+Avoid touching register %ecx in startup32_set_idt_entry(), by folding
+the MOV, SHL and ORL instructions into a single ORL which no longer
+requires a temp register.
+
+This permits ECX to be used as a function argument in a subsequent
+patch.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-10-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 8 ++------
+ 1 file changed, 2 insertions(+), 6 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -749,7 +749,6 @@ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLO
+ */
+ SYM_FUNC_START(startup32_set_idt_entry)
+ push %ebx
+- push %ecx
+
+ /* IDT entry address to %ebx */
+ leal rva(boot32_idt)(%ebp), %ebx
+@@ -758,10 +757,8 @@ SYM_FUNC_START(startup32_set_idt_entry)
+
+ /* Build IDT entry, lower 4 bytes */
+ movl %eax, %edx
+- andl $0x0000ffff, %edx # Target code segment offset [15:0]
+- movl $__KERNEL32_CS, %ecx # Target code segment selector
+- shl $16, %ecx
+- orl %ecx, %edx
++ andl $0x0000ffff, %edx # Target code segment offset [15:0]
++ orl $(__KERNEL32_CS << 16), %edx # Target code segment selector
+
+ /* Store lower 4 bytes to IDT */
+ movl %edx, (%ebx)
+@@ -774,7 +771,6 @@ SYM_FUNC_START(startup32_set_idt_entry)
+ /* Store upper 4 bytes to IDT */
+ movl %edx, 4(%ebx)
+
+- pop %ecx
+ pop %ebx
+ RET
+ SYM_FUNC_END(startup32_set_idt_entry)
--- /dev/null
+From 4b52016247aeaa55ca3e3bc2e03cd91114c145c2 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:07 +0100
+Subject: x86/boot/compressed, efi: Merge multiple definitions of image_offset into one
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 4b52016247aeaa55ca3e3bc2e03cd91114c145c2 upstream.
+
+There is no need for head_32.S and head_64.S both declaring a copy of
+the global 'image_offset' variable, so drop those and make the extern C
+declaration the definition.
+
+When image_offset is moved to the .c file, it needs to be placed
+particularly in the .data section because it lands by default in the
+.bss section which is cleared too late, in .Lrelocated, before the first
+access to it and thus garbage gets read, leading to SEV guests exploding
+in early boot.
+
+This happens only when the SEV guest kernel is loaded through grub. If
+supplied with qemu's -kernel command line option, that memory is always
+cleared upfront by qemu and all is fine there.
+
+ [ bp: Expand commit message with SEV aspect. ]
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-8-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_32.S | 4 ----
+ arch/x86/boot/compressed/head_64.S | 4 ----
+ drivers/firmware/efi/libstub/x86-stub.c | 2 +-
+ 3 files changed, 1 insertion(+), 9 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_32.S
++++ b/arch/x86/boot/compressed/head_32.S
+@@ -208,10 +208,6 @@ SYM_DATA_START_LOCAL(gdt)
+ .quad 0x00cf92000000ffff /* __KERNEL_DS */
+ SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt_end)
+
+-#ifdef CONFIG_EFI_STUB
+-SYM_DATA(image_offset, .long 0)
+-#endif
+-
+ /*
+ * Stack and heap for uncompression
+ */
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -734,10 +734,6 @@ SYM_DATA_START(boot32_idt)
+ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLOBAL, boot32_idt_end)
+ #endif
+
+-#ifdef CONFIG_EFI_STUB
+-SYM_DATA(image_offset, .long 0)
+-#endif
+-
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
+ __HEAD
+ .code32
+--- a/drivers/firmware/efi/libstub/x86-stub.c
++++ b/drivers/firmware/efi/libstub/x86-stub.c
+@@ -23,7 +23,7 @@
+
+ const efi_system_table_t *efi_system_table;
+ const efi_dxe_services_table_t *efi_dxe_table;
+-extern u32 image_offset;
++u32 image_offset __section(".data");
+ static efi_loaded_image_t *image = NULL;
+
+ static efi_status_t
--- /dev/null
+From e2ab9eab324cdf240de89741e4a1aa79919f0196 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:02 +0100
+Subject: x86/boot/compressed: Move 32-bit entrypoint code into .text section
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit e2ab9eab324cdf240de89741e4a1aa79919f0196 upstream.
+
+Move the code that stores the arguments passed to the EFI entrypoint
+into the .text section, so that it can be moved into a separate
+compilation unit in a subsequent patch.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-3-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 48 ++++++++++++++++++++++++++-----------
+ 1 file changed, 34 insertions(+), 14 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -303,24 +303,41 @@ SYM_FUNC_START(efi32_stub_entry)
+ popl %ecx
+ popl %edx
+ popl %esi
++ jmp efi32_entry
++SYM_FUNC_END(efi32_stub_entry)
+
++ .text
++/*
++ * This is the common EFI stub entry point for mixed mode.
++ *
++ * Arguments: %ecx image handle
++ * %edx EFI system table pointer
++ * %esi struct bootparams pointer (or NULL when not using
++ * the EFI handover protocol)
++ *
++ * Since this is the point of no return for ordinary execution, no registers
++ * are considered live except for the function parameters. [Note that the EFI
++ * stub may still exit and return to the firmware using the Exit() EFI boot
++ * service.]
++ */
++SYM_FUNC_START_LOCAL(efi32_entry)
+ call 1f
+-1: pop %ebp
+- subl $ rva(1b), %ebp
+-
+- movl %esi, rva(efi32_boot_args+8)(%ebp)
+-SYM_INNER_LABEL(efi32_pe_stub_entry, SYM_L_LOCAL)
+- movl %ecx, rva(efi32_boot_args)(%ebp)
+- movl %edx, rva(efi32_boot_args+4)(%ebp)
+- movb $0, rva(efi_is64)(%ebp)
++1: pop %ebx
+
+ /* Save firmware GDTR and code/data selectors */
+- sgdtl rva(efi32_boot_gdt)(%ebp)
+- movw %cs, rva(efi32_boot_cs)(%ebp)
+- movw %ds, rva(efi32_boot_ds)(%ebp)
++ sgdtl (efi32_boot_gdt - 1b)(%ebx)
++ movw %cs, (efi32_boot_cs - 1b)(%ebx)
++ movw %ds, (efi32_boot_ds - 1b)(%ebx)
+
+ /* Store firmware IDT descriptor */
+- sidtl rva(efi32_boot_idt)(%ebp)
++ sidtl (efi32_boot_idt - 1b)(%ebx)
++
++ /* Store boot arguments */
++ leal (efi32_boot_args - 1b)(%ebx), %ebx
++ movl %ecx, 0(%ebx)
++ movl %edx, 4(%ebx)
++ movl %esi, 8(%ebx)
++ movb $0x0, 12(%ebx) // efi_is64
+
+ /* Disable paging */
+ movl %cr0, %eax
+@@ -328,7 +345,8 @@ SYM_INNER_LABEL(efi32_pe_stub_entry, SYM
+ movl %eax, %cr0
+
+ jmp startup_32
+-SYM_FUNC_END(efi32_stub_entry)
++SYM_FUNC_END(efi32_entry)
++ __HEAD
+ #endif
+
+ .code64
+@@ -847,7 +865,9 @@ SYM_FUNC_START(efi32_pe_entry)
+ */
+ subl %esi, %ebx
+ movl %ebx, rva(image_offset)(%ebp) // save image_offset
+- jmp efi32_pe_stub_entry
++ xorl %esi, %esi
++ jmp efi32_entry // pass %ecx, %edx, %esi
++ // no other registers remain live
+
+ 2: popl %edi // restore callee-save registers
+ popl %ebx
--- /dev/null
+From 5c3a85f35b583259cf5ca0344cd79c8899ba1bb7 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:03 +0100
+Subject: x86/boot/compressed: Move bootargs parsing out of 32-bit startup code
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 5c3a85f35b583259cf5ca0344cd79c8899ba1bb7 upstream.
+
+Move the logic that chooses between the different EFI entrypoints out of
+the 32-bit boot path, and into a 64-bit helper that can perform the same
+task much more cleanly. While at it, document the mixed mode boot flow
+in a code comment.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-4-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/efi_mixed.S | 43 +++++++++++++++++++++++++++++++++++
+ arch/x86/boot/compressed/head_64.S | 24 +++----------------
+ 2 files changed, 47 insertions(+), 20 deletions(-)
+
+--- a/arch/x86/boot/compressed/efi_mixed.S
++++ b/arch/x86/boot/compressed/efi_mixed.S
+@@ -22,6 +22,49 @@
+
+ .code64
+ .text
++/*
++ * When booting in 64-bit mode on 32-bit EFI firmware, startup_64_mixed_mode()
++ * is the first thing that runs after switching to long mode. Depending on
++ * whether the EFI handover protocol or the compat entry point was used to
++ * enter the kernel, it will either branch to the 64-bit EFI handover
++ * entrypoint at offset 0x390 in the image, or to the 64-bit EFI PE/COFF
++ * entrypoint efi_pe_entry(). In the former case, the bootloader must provide a
++ * struct bootparams pointer as the third argument, so the presence of such a
++ * pointer is used to disambiguate.
++ *
++ * +--------------+
++ * +------------------+ +------------+ +------>| efi_pe_entry |
++ * | efi32_pe_entry |---->| | | +-----------+--+
++ * +------------------+ | | +------+----------------+ |
++ * | startup_32 |---->| startup_64_mixed_mode | |
++ * +------------------+ | | +------+----------------+ V
++ * | efi32_stub_entry |---->| | | +------------------+
++ * +------------------+ +------------+ +---->| efi64_stub_entry |
++ * +-------------+----+
++ * +------------+ +----------+ |
++ * | startup_64 |<----| efi_main |<--------------+
++ * +------------+ +----------+
++ */
++SYM_FUNC_START(startup_64_mixed_mode)
++ lea efi32_boot_args(%rip), %rdx
++ mov 0(%rdx), %edi
++ mov 4(%rdx), %esi
++ mov 8(%rdx), %edx // saved bootparams pointer
++ test %edx, %edx
++ jnz efi64_stub_entry
++ /*
++ * efi_pe_entry uses MS calling convention, which requires 32 bytes of
++ * shadow space on the stack even if all arguments are passed in
++ * registers. We also need an additional 8 bytes for the space that
++ * would be occupied by the return address, and this also results in
++ * the correct stack alignment for entry.
++ */
++ sub $40, %rsp
++ mov %rdi, %rcx // MS calling convention
++ mov %rsi, %rdx
++ jmp efi_pe_entry
++SYM_FUNC_END(startup_64_mixed_mode)
++
+ SYM_FUNC_START(__efi64_thunk)
+ push %rbp
+ push %rbx
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -261,25 +261,9 @@ SYM_FUNC_START(startup_32)
+ */
+ leal rva(startup_64)(%ebp), %eax
+ #ifdef CONFIG_EFI_MIXED
+- movl rva(efi32_boot_args)(%ebp), %edi
+- testl %edi, %edi
+- jz 1f
+- leal rva(efi64_stub_entry)(%ebp), %eax
+- movl rva(efi32_boot_args+4)(%ebp), %esi
+- movl rva(efi32_boot_args+8)(%ebp), %edx // saved bootparams pointer
+- testl %edx, %edx
+- jnz 1f
+- /*
+- * efi_pe_entry uses MS calling convention, which requires 32 bytes of
+- * shadow space on the stack even if all arguments are passed in
+- * registers. We also need an additional 8 bytes for the space that
+- * would be occupied by the return address, and this also results in
+- * the correct stack alignment for entry.
+- */
+- subl $40, %esp
+- leal rva(efi_pe_entry)(%ebp), %eax
+- movl %edi, %ecx // MS calling convention
+- movl %esi, %edx
++ cmpb $1, rva(efi_is64)(%ebp)
++ je 1f
++ leal rva(startup_64_mixed_mode)(%ebp), %eax
+ 1:
+ #endif
+ /* Check if the C-bit position is correct when SEV is active */
+@@ -795,7 +779,7 @@ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLO
+ SYM_DATA(image_offset, .long 0)
+ #endif
+ #ifdef CONFIG_EFI_MIXED
+-SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0)
++SYM_DATA(efi32_boot_args, .long 0, 0, 0)
+ SYM_DATA(efi_is64, .byte 1)
+
+ #define ST32_boottime 60 // offsetof(efi_system_table_32_t, boottime)
--- /dev/null
+From 73a6dec80e2acedaef3ca603d4b5799049f6e9f8 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:05 +0100
+Subject: x86/boot/compressed: Move efi32_entry out of head_64.S
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 73a6dec80e2acedaef3ca603d4b5799049f6e9f8 upstream.
+
+Move the efi32_entry() routine out of head_64.S and into efi-mixed.S,
+which reduces clutter in the complicated startup routines. It also
+permits linkage of some symbols used by code to be made local.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-6-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/efi_mixed.S | 57 ++++++++++++++++++++++++++++-------
+ arch/x86/boot/compressed/head_64.S | 45 ---------------------------
+ 2 files changed, 47 insertions(+), 55 deletions(-)
+
+--- a/arch/x86/boot/compressed/efi_mixed.S
++++ b/arch/x86/boot/compressed/efi_mixed.S
+@@ -105,7 +105,7 @@ SYM_FUNC_START(__efi64_thunk)
+ /*
+ * Switch to IDT and GDT with 32-bit segments. This is the firmware GDT
+ * and IDT that was installed when the kernel started executing. The
+- * pointers were saved at the EFI stub entry point in head_64.S.
++ * pointers were saved by the efi32_entry() routine below.
+ *
+ * Pass the saved DS selector to the 32-bit code, and use far return to
+ * restore the saved CS selector.
+@@ -217,22 +217,59 @@ SYM_FUNC_START_LOCAL(efi_enter32)
+ lret
+ SYM_FUNC_END(efi_enter32)
+
++/*
++ * This is the common EFI stub entry point for mixed mode.
++ *
++ * Arguments: %ecx image handle
++ * %edx EFI system table pointer
++ * %esi struct bootparams pointer (or NULL when not using
++ * the EFI handover protocol)
++ *
++ * Since this is the point of no return for ordinary execution, no registers
++ * are considered live except for the function parameters. [Note that the EFI
++ * stub may still exit and return to the firmware using the Exit() EFI boot
++ * service.]
++ */
++SYM_FUNC_START(efi32_entry)
++ call 1f
++1: pop %ebx
++
++ /* Save firmware GDTR and code/data selectors */
++ sgdtl (efi32_boot_gdt - 1b)(%ebx)
++ movw %cs, (efi32_boot_cs - 1b)(%ebx)
++ movw %ds, (efi32_boot_ds - 1b)(%ebx)
++
++ /* Store firmware IDT descriptor */
++ sidtl (efi32_boot_idt - 1b)(%ebx)
++
++ /* Store boot arguments */
++ leal (efi32_boot_args - 1b)(%ebx), %ebx
++ movl %ecx, 0(%ebx)
++ movl %edx, 4(%ebx)
++ movl %esi, 8(%ebx)
++ movb $0x0, 12(%ebx) // efi_is64
++
++ /* Disable paging */
++ movl %cr0, %eax
++ btrl $X86_CR0_PG_BIT, %eax
++ movl %eax, %cr0
++
++ jmp startup_32
++SYM_FUNC_END(efi32_entry)
++
+ .data
+ .balign 8
+-SYM_DATA_START(efi32_boot_gdt)
++SYM_DATA_START_LOCAL(efi32_boot_gdt)
+ .word 0
+ .quad 0
+ SYM_DATA_END(efi32_boot_gdt)
+
+-SYM_DATA_START(efi32_boot_idt)
++SYM_DATA_START_LOCAL(efi32_boot_idt)
+ .word 0
+ .quad 0
+ SYM_DATA_END(efi32_boot_idt)
+
+-SYM_DATA_START(efi32_boot_cs)
+- .word 0
+-SYM_DATA_END(efi32_boot_cs)
+-
+-SYM_DATA_START(efi32_boot_ds)
+- .word 0
+-SYM_DATA_END(efi32_boot_ds)
++SYM_DATA_LOCAL(efi32_boot_cs, .word 0)
++SYM_DATA_LOCAL(efi32_boot_ds, .word 0)
++SYM_DATA_LOCAL(efi32_boot_args, .long 0, 0, 0)
++SYM_DATA(efi_is64, .byte 1)
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -289,48 +289,6 @@ SYM_FUNC_START(efi32_stub_entry)
+ popl %esi
+ jmp efi32_entry
+ SYM_FUNC_END(efi32_stub_entry)
+-
+- .text
+-/*
+- * This is the common EFI stub entry point for mixed mode.
+- *
+- * Arguments: %ecx image handle
+- * %edx EFI system table pointer
+- * %esi struct bootparams pointer (or NULL when not using
+- * the EFI handover protocol)
+- *
+- * Since this is the point of no return for ordinary execution, no registers
+- * are considered live except for the function parameters. [Note that the EFI
+- * stub may still exit and return to the firmware using the Exit() EFI boot
+- * service.]
+- */
+-SYM_FUNC_START_LOCAL(efi32_entry)
+- call 1f
+-1: pop %ebx
+-
+- /* Save firmware GDTR and code/data selectors */
+- sgdtl (efi32_boot_gdt - 1b)(%ebx)
+- movw %cs, (efi32_boot_cs - 1b)(%ebx)
+- movw %ds, (efi32_boot_ds - 1b)(%ebx)
+-
+- /* Store firmware IDT descriptor */
+- sidtl (efi32_boot_idt - 1b)(%ebx)
+-
+- /* Store boot arguments */
+- leal (efi32_boot_args - 1b)(%ebx), %ebx
+- movl %ecx, 0(%ebx)
+- movl %edx, 4(%ebx)
+- movl %esi, 8(%ebx)
+- movb $0x0, 12(%ebx) // efi_is64
+-
+- /* Disable paging */
+- movl %cr0, %eax
+- btrl $X86_CR0_PG_BIT, %eax
+- movl %eax, %cr0
+-
+- jmp startup_32
+-SYM_FUNC_END(efi32_entry)
+- __HEAD
+ #endif
+
+ .code64
+@@ -779,9 +737,6 @@ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLO
+ SYM_DATA(image_offset, .long 0)
+ #endif
+ #ifdef CONFIG_EFI_MIXED
+-SYM_DATA(efi32_boot_args, .long 0, 0, 0)
+-SYM_DATA(efi_is64, .byte 1)
+-
+ #define ST32_boottime 60 // offsetof(efi_system_table_32_t, boottime)
+ #define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol)
+ #define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base)
--- /dev/null
+From 91592b5c0c2f076ff9d8cc0c14aa563448ac9fc4 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:04 +0100
+Subject: x86/boot/compressed: Move efi32_pe_entry into .text section
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 91592b5c0c2f076ff9d8cc0c14aa563448ac9fc4 upstream.
+
+Move efi32_pe_entry() into the .text section, so that it can be moved
+out of head_64.S and into a separate compilation unit in a subsequent
+patch.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-5-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 11 +++++------
+ 1 file changed, 5 insertions(+), 6 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -786,7 +786,7 @@ SYM_DATA(efi_is64, .byte 1)
+ #define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol)
+ #define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base)
+
+- __HEAD
++ .text
+ .code32
+ SYM_FUNC_START(efi32_pe_entry)
+ /*
+@@ -808,12 +808,11 @@ SYM_FUNC_START(efi32_pe_entry)
+
+ call 1f
+ 1: pop %ebx
+- subl $ rva(1b), %ebx
+
+ /* Get the loaded image protocol pointer from the image handle */
+ leal -4(%ebp), %eax
+ pushl %eax // &loaded_image
+- leal rva(loaded_image_proto)(%ebx), %eax
++ leal (loaded_image_proto - 1b)(%ebx), %eax
+ pushl %eax // pass the GUID address
+ pushl 8(%ebp) // pass the image handle
+
+@@ -842,13 +841,13 @@ SYM_FUNC_START(efi32_pe_entry)
+ movl 12(%ebp), %edx // sys_table
+ movl -4(%ebp), %esi // loaded_image
+ movl LI32_image_base(%esi), %esi // loaded_image->image_base
+- movl %ebx, %ebp // startup_32 for efi32_pe_stub_entry
++ leal (startup_32 - 1b)(%ebx), %ebp // runtime address of startup_32
+ /*
+ * We need to set the image_offset variable here since startup_32() will
+ * use it before we get to the 64-bit efi_pe_entry() in C code.
+ */
+- subl %esi, %ebx
+- movl %ebx, rva(image_offset)(%ebp) // save image_offset
++ subl %esi, %ebp // calculate image_offset
++ movl %ebp, (image_offset - 1b)(%ebx) // save image_offset
+ xorl %esi, %esi
+ jmp efi32_entry // pass %ecx, %edx, %esi
+ // no other registers remain live
--- /dev/null
+From 7f22ca396778fea9332d83ec2359dbe8396e9a06 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:06 +0100
+Subject: x86/boot/compressed: Move efi32_pe_entry() out of head_64.S
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 7f22ca396778fea9332d83ec2359dbe8396e9a06 upstream.
+
+Move the implementation of efi32_pe_entry() into efi-mixed.S, which is a
+more suitable location that only gets built if EFI mixed mode is
+actually enabled.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-7-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/efi_mixed.S | 82 ++++++++++++++++++++++++++++++++
+ arch/x86/boot/compressed/head_64.S | 87 -----------------------------------
+ 2 files changed, 83 insertions(+), 86 deletions(-)
+
+--- a/arch/x86/boot/compressed/efi_mixed.S
++++ b/arch/x86/boot/compressed/efi_mixed.S
+@@ -257,6 +257,88 @@ SYM_FUNC_START(efi32_entry)
+ jmp startup_32
+ SYM_FUNC_END(efi32_entry)
+
++#define ST32_boottime 60 // offsetof(efi_system_table_32_t, boottime)
++#define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol)
++#define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base)
++
++/*
++ * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
++ * efi_system_table_32_t *sys_table)
++ */
++SYM_FUNC_START(efi32_pe_entry)
++ pushl %ebp
++ movl %esp, %ebp
++ pushl %eax // dummy push to allocate loaded_image
++
++ pushl %ebx // save callee-save registers
++ pushl %edi
++
++ call verify_cpu // check for long mode support
++ testl %eax, %eax
++ movl $0x80000003, %eax // EFI_UNSUPPORTED
++ jnz 2f
++
++ call 1f
++1: pop %ebx
++
++ /* Get the loaded image protocol pointer from the image handle */
++ leal -4(%ebp), %eax
++ pushl %eax // &loaded_image
++ leal (loaded_image_proto - 1b)(%ebx), %eax
++ pushl %eax // pass the GUID address
++ pushl 8(%ebp) // pass the image handle
++
++ /*
++ * Note the alignment of the stack frame.
++ * sys_table
++ * handle <-- 16-byte aligned on entry by ABI
++ * return address
++ * frame pointer
++ * loaded_image <-- local variable
++ * saved %ebx <-- 16-byte aligned here
++ * saved %edi
++ * &loaded_image
++ * &loaded_image_proto
++ * handle <-- 16-byte aligned for call to handle_protocol
++ */
++
++ movl 12(%ebp), %eax // sys_table
++ movl ST32_boottime(%eax), %eax // sys_table->boottime
++ call *BS32_handle_protocol(%eax) // sys_table->boottime->handle_protocol
++ addl $12, %esp // restore argument space
++ testl %eax, %eax
++ jnz 2f
++
++ movl 8(%ebp), %ecx // image_handle
++ movl 12(%ebp), %edx // sys_table
++ movl -4(%ebp), %esi // loaded_image
++ movl LI32_image_base(%esi), %esi // loaded_image->image_base
++ leal (startup_32 - 1b)(%ebx), %ebp // runtime address of startup_32
++ /*
++ * We need to set the image_offset variable here since startup_32() will
++ * use it before we get to the 64-bit efi_pe_entry() in C code.
++ */
++ subl %esi, %ebp // calculate image_offset
++ movl %ebp, (image_offset - 1b)(%ebx) // save image_offset
++ xorl %esi, %esi
++ jmp efi32_entry // pass %ecx, %edx, %esi
++ // no other registers remain live
++
++2: popl %edi // restore callee-save registers
++ popl %ebx
++ leave
++ RET
++SYM_FUNC_END(efi32_pe_entry)
++
++ .section ".rodata"
++ /* EFI loaded image protocol GUID */
++ .balign 4
++SYM_DATA_START_LOCAL(loaded_image_proto)
++ .long 0x5b1b31a1
++ .word 0x9562, 0x11d2
++ .byte 0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b
++SYM_DATA_END(loaded_image_proto)
++
+ .data
+ .balign 8
+ SYM_DATA_START_LOCAL(efi32_boot_gdt)
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -689,6 +689,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmo
+ jmp 1b
+ SYM_FUNC_END(.Lno_longmode)
+
++ .globl verify_cpu
+ #include "../../kernel/verify_cpu.S"
+
+ .data
+@@ -736,92 +737,6 @@ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLO
+ #ifdef CONFIG_EFI_STUB
+ SYM_DATA(image_offset, .long 0)
+ #endif
+-#ifdef CONFIG_EFI_MIXED
+-#define ST32_boottime 60 // offsetof(efi_system_table_32_t, boottime)
+-#define BS32_handle_protocol 88 // offsetof(efi_boot_services_32_t, handle_protocol)
+-#define LI32_image_base 32 // offsetof(efi_loaded_image_32_t, image_base)
+-
+- .text
+- .code32
+-SYM_FUNC_START(efi32_pe_entry)
+-/*
+- * efi_status_t efi32_pe_entry(efi_handle_t image_handle,
+- * efi_system_table_32_t *sys_table)
+- */
+-
+- pushl %ebp
+- movl %esp, %ebp
+- pushl %eax // dummy push to allocate loaded_image
+-
+- pushl %ebx // save callee-save registers
+- pushl %edi
+-
+- call verify_cpu // check for long mode support
+- testl %eax, %eax
+- movl $0x80000003, %eax // EFI_UNSUPPORTED
+- jnz 2f
+-
+- call 1f
+-1: pop %ebx
+-
+- /* Get the loaded image protocol pointer from the image handle */
+- leal -4(%ebp), %eax
+- pushl %eax // &loaded_image
+- leal (loaded_image_proto - 1b)(%ebx), %eax
+- pushl %eax // pass the GUID address
+- pushl 8(%ebp) // pass the image handle
+-
+- /*
+- * Note the alignment of the stack frame.
+- * sys_table
+- * handle <-- 16-byte aligned on entry by ABI
+- * return address
+- * frame pointer
+- * loaded_image <-- local variable
+- * saved %ebx <-- 16-byte aligned here
+- * saved %edi
+- * &loaded_image
+- * &loaded_image_proto
+- * handle <-- 16-byte aligned for call to handle_protocol
+- */
+-
+- movl 12(%ebp), %eax // sys_table
+- movl ST32_boottime(%eax), %eax // sys_table->boottime
+- call *BS32_handle_protocol(%eax) // sys_table->boottime->handle_protocol
+- addl $12, %esp // restore argument space
+- testl %eax, %eax
+- jnz 2f
+-
+- movl 8(%ebp), %ecx // image_handle
+- movl 12(%ebp), %edx // sys_table
+- movl -4(%ebp), %esi // loaded_image
+- movl LI32_image_base(%esi), %esi // loaded_image->image_base
+- leal (startup_32 - 1b)(%ebx), %ebp // runtime address of startup_32
+- /*
+- * We need to set the image_offset variable here since startup_32() will
+- * use it before we get to the 64-bit efi_pe_entry() in C code.
+- */
+- subl %esi, %ebp // calculate image_offset
+- movl %ebp, (image_offset - 1b)(%ebx) // save image_offset
+- xorl %esi, %esi
+- jmp efi32_entry // pass %ecx, %edx, %esi
+- // no other registers remain live
+-
+-2: popl %edi // restore callee-save registers
+- popl %ebx
+- leave
+- RET
+-SYM_FUNC_END(efi32_pe_entry)
+-
+- .section ".rodata"
+- /* EFI loaded image protocol GUID */
+- .balign 4
+-SYM_DATA_START_LOCAL(loaded_image_proto)
+- .long 0x5b1b31a1
+- .word 0x9562, 0x11d2
+- .byte 0x8e, 0x3f, 0x00, 0xa0, 0xc9, 0x69, 0x72, 0x3b
+-SYM_DATA_END(loaded_image_proto)
+-#endif
+
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
+ __HEAD
--- /dev/null
+From b5d854cd4b6a314edd6c15dabc4233b84a0f8e5e Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:13 +0100
+Subject: x86/boot/compressed: Move startup32_check_sev_cbit() into .text
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit b5d854cd4b6a314edd6c15dabc4233b84a0f8e5e upstream.
+
+Move startup32_check_sev_cbit() into the .text section and turn it into
+an ordinary function using the ordinary 32-bit calling convention,
+instead of saving/restoring the registers that are known to be live at
+the only call site. This improves maintainability, and makes it possible
+to move this function out of head_64.S and into a separate compilation
+unit that is specific to memory encryption.
+
+Note that this requires the call site to be moved before the mixed mode
+check, as %eax will be live otherwise.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-14-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 35 +++++++++++++++++++----------------
+ 1 file changed, 19 insertions(+), 16 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -251,6 +251,11 @@ SYM_FUNC_START(startup_32)
+ movl $__BOOT_TSS, %eax
+ ltr %ax
+
++#ifdef CONFIG_AMD_MEM_ENCRYPT
++ /* Check if the C-bit position is correct when SEV is active */
++ call startup32_check_sev_cbit
++#endif
++
+ /*
+ * Setup for the jump to 64bit mode
+ *
+@@ -268,8 +273,6 @@ SYM_FUNC_START(startup_32)
+ leal rva(startup_64_mixed_mode)(%ebp), %eax
+ 1:
+ #endif
+- /* Check if the C-bit position is correct when SEV is active */
+- call startup32_check_sev_cbit
+
+ pushl $__KERNEL_CS
+ pushl %eax
+@@ -740,16 +743,17 @@ SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBA
+ * succeed. An incorrect C-bit position will map all memory unencrypted, so that
+ * the compare will use the encrypted random data and fail.
+ */
+- __HEAD
+-SYM_FUNC_START(startup32_check_sev_cbit)
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
+- pushl %eax
++ .text
++SYM_FUNC_START(startup32_check_sev_cbit)
+ pushl %ebx
+- pushl %ecx
+- pushl %edx
++ pushl %ebp
++
++ call 0f
++0: popl %ebp
+
+ /* Check for non-zero sev_status */
+- movl rva(sev_status)(%ebp), %eax
++ movl (sev_status - 0b)(%ebp), %eax
+ testl %eax, %eax
+ jz 4f
+
+@@ -764,17 +768,18 @@ SYM_FUNC_START(startup32_check_sev_cbit)
+ jnc 2b
+
+ /* Store to memory and keep it in the registers */
+- movl %eax, rva(sev_check_data)(%ebp)
+- movl %ebx, rva(sev_check_data+4)(%ebp)
++ leal (sev_check_data - 0b)(%ebp), %ebp
++ movl %eax, 0(%ebp)
++ movl %ebx, 4(%ebp)
+
+ /* Enable paging to see if encryption is active */
+ movl %cr0, %edx /* Backup %cr0 in %edx */
+ movl $(X86_CR0_PG | X86_CR0_PE), %ecx /* Enable Paging and Protected mode */
+ movl %ecx, %cr0
+
+- cmpl %eax, rva(sev_check_data)(%ebp)
++ cmpl %eax, 0(%ebp)
+ jne 3f
+- cmpl %ebx, rva(sev_check_data+4)(%ebp)
++ cmpl %ebx, 4(%ebp)
+ jne 3f
+
+ movl %edx, %cr0 /* Restore previous %cr0 */
+@@ -786,13 +791,11 @@ SYM_FUNC_START(startup32_check_sev_cbit)
+ jmp 3b
+
+ 4:
+- popl %edx
+- popl %ecx
++ popl %ebp
+ popl %ebx
+- popl %eax
+-#endif
+ RET
+ SYM_FUNC_END(startup32_check_sev_cbit)
++#endif
+
+ /*
+ * Stack and heap for uncompression
--- /dev/null
+From 9d7eaae6a071ff1f718e0aa5e610bb712f8cc632 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:14 +0100
+Subject: x86/boot/compressed: Move startup32_check_sev_cbit() out of head_64.S
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 9d7eaae6a071ff1f718e0aa5e610bb712f8cc632 upstream.
+
+Now that the startup32_check_sev_cbit() routine can execute from
+anywhere and behaves like an ordinary function, it can be moved where it
+belongs.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-15-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 71 ---------------------------------
+ arch/x86/boot/compressed/mem_encrypt.S | 68 +++++++++++++++++++++++++++++++
+ 2 files changed, 68 insertions(+), 71 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -727,77 +727,6 @@ SYM_DATA_START(boot_idt)
+ SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBAL, boot_idt_end)
+
+ /*
+- * Check for the correct C-bit position when the startup_32 boot-path is used.
+- *
+- * The check makes use of the fact that all memory is encrypted when paging is
+- * disabled. The function creates 64 bits of random data using the RDRAND
+- * instruction. RDRAND is mandatory for SEV guests, so always available. If the
+- * hypervisor violates that the kernel will crash right here.
+- *
+- * The 64 bits of random data are stored to a memory location and at the same
+- * time kept in the %eax and %ebx registers. Since encryption is always active
+- * when paging is off the random data will be stored encrypted in main memory.
+- *
+- * Then paging is enabled. When the C-bit position is correct all memory is
+- * still mapped encrypted and comparing the register values with memory will
+- * succeed. An incorrect C-bit position will map all memory unencrypted, so that
+- * the compare will use the encrypted random data and fail.
+- */
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+- .text
+-SYM_FUNC_START(startup32_check_sev_cbit)
+- pushl %ebx
+- pushl %ebp
+-
+- call 0f
+-0: popl %ebp
+-
+- /* Check for non-zero sev_status */
+- movl (sev_status - 0b)(%ebp), %eax
+- testl %eax, %eax
+- jz 4f
+-
+- /*
+- * Get two 32-bit random values - Don't bail out if RDRAND fails
+- * because it is better to prevent forward progress if no random value
+- * can be gathered.
+- */
+-1: rdrand %eax
+- jnc 1b
+-2: rdrand %ebx
+- jnc 2b
+-
+- /* Store to memory and keep it in the registers */
+- leal (sev_check_data - 0b)(%ebp), %ebp
+- movl %eax, 0(%ebp)
+- movl %ebx, 4(%ebp)
+-
+- /* Enable paging to see if encryption is active */
+- movl %cr0, %edx /* Backup %cr0 in %edx */
+- movl $(X86_CR0_PG | X86_CR0_PE), %ecx /* Enable Paging and Protected mode */
+- movl %ecx, %cr0
+-
+- cmpl %eax, 0(%ebp)
+- jne 3f
+- cmpl %ebx, 4(%ebp)
+- jne 3f
+-
+- movl %edx, %cr0 /* Restore previous %cr0 */
+-
+- jmp 4f
+-
+-3: /* Check failed - hlt the machine */
+- hlt
+- jmp 3b
+-
+-4:
+- popl %ebp
+- popl %ebx
+- RET
+-SYM_FUNC_END(startup32_check_sev_cbit)
+-#endif
+-
+-/*
+ * Stack and heap for uncompression
+ */
+ .bss
+--- a/arch/x86/boot/compressed/mem_encrypt.S
++++ b/arch/x86/boot/compressed/mem_encrypt.S
+@@ -243,6 +243,74 @@ SYM_FUNC_START(startup32_load_idt)
+ RET
+ SYM_FUNC_END(startup32_load_idt)
+
++/*
++ * Check for the correct C-bit position when the startup_32 boot-path is used.
++ *
++ * The check makes use of the fact that all memory is encrypted when paging is
++ * disabled. The function creates 64 bits of random data using the RDRAND
++ * instruction. RDRAND is mandatory for SEV guests, so always available. If the
++ * hypervisor violates that the kernel will crash right here.
++ *
++ * The 64 bits of random data are stored to a memory location and at the same
++ * time kept in the %eax and %ebx registers. Since encryption is always active
++ * when paging is off the random data will be stored encrypted in main memory.
++ *
++ * Then paging is enabled. When the C-bit position is correct all memory is
++ * still mapped encrypted and comparing the register values with memory will
++ * succeed. An incorrect C-bit position will map all memory unencrypted, so that
++ * the compare will use the encrypted random data and fail.
++ */
++SYM_FUNC_START(startup32_check_sev_cbit)
++ pushl %ebx
++ pushl %ebp
++
++ call 0f
++0: popl %ebp
++
++ /* Check for non-zero sev_status */
++ movl (sev_status - 0b)(%ebp), %eax
++ testl %eax, %eax
++ jz 4f
++
++ /*
++ * Get two 32-bit random values - Don't bail out if RDRAND fails
++ * because it is better to prevent forward progress if no random value
++ * can be gathered.
++ */
++1: rdrand %eax
++ jnc 1b
++2: rdrand %ebx
++ jnc 2b
++
++ /* Store to memory and keep it in the registers */
++ leal (sev_check_data - 0b)(%ebp), %ebp
++ movl %eax, 0(%ebp)
++ movl %ebx, 4(%ebp)
++
++ /* Enable paging to see if encryption is active */
++ movl %cr0, %edx /* Backup %cr0 in %edx */
++ movl $(X86_CR0_PG | X86_CR0_PE), %ecx /* Enable Paging and Protected mode */
++ movl %ecx, %cr0
++
++ cmpl %eax, 0(%ebp)
++ jne 3f
++ cmpl %ebx, 4(%ebp)
++ jne 3f
++
++ movl %edx, %cr0 /* Restore previous %cr0 */
++
++ jmp 4f
++
++3: /* Check failed - hlt the machine */
++ hlt
++ jmp 3b
++
++4:
++ popl %ebp
++ popl %ebx
++ RET
++SYM_FUNC_END(startup32_check_sev_cbit)
++
+ .code64
+
+ #include "../../kernel/sev_verify_cbit.S"
--- /dev/null
+From c6355995ba471d7ad574174e593192ce805c7e1a Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:11 +0100
+Subject: x86/boot/compressed: Move startup32_load_idt() into .text section
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit c6355995ba471d7ad574174e593192ce805c7e1a upstream.
+
+Convert startup32_load_idt() into an ordinary function and move it into
+the .text section. This involves turning the rva() immediates into ones
+derived from a local label, and preserving/restoring the %ebp and %ebx
+as per the calling convention.
+
+Also move the #ifdef to the only existing call site. This makes it clear
+that the function call does nothing if support for memory encryption is
+not compiled in.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-12-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 31 ++++++++++++++++++++-----------
+ 1 file changed, 20 insertions(+), 11 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -118,7 +118,9 @@ SYM_FUNC_START(startup_32)
+ 1:
+
+ /* Setup Exception handling for SEV-ES */
++#ifdef CONFIG_AMD_MEM_ENCRYPT
+ call startup32_load_idt
++#endif
+
+ /* Make sure cpu supports long mode. */
+ call verify_cpu
+@@ -732,10 +734,8 @@ SYM_DATA_START(boot32_idt)
+ .quad 0
+ .endr
+ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLOBAL, boot32_idt_end)
+-#endif
+
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+- __HEAD
++ .text
+ .code32
+ /*
+ * Write an IDT entry into boot32_idt
+@@ -768,24 +768,32 @@ SYM_FUNC_START_LOCAL(startup32_set_idt_e
+
+ RET
+ SYM_FUNC_END(startup32_set_idt_entry)
+-#endif
+
+ SYM_FUNC_START(startup32_load_idt)
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+- leal rva(boot32_idt)(%ebp), %ecx
++ push %ebp
++ push %ebx
++
++ call 1f
++1: pop %ebp
++
++ leal (boot32_idt - 1b)(%ebp), %ebx
+
+ /* #VC handler */
+- leal rva(startup32_vc_handler)(%ebp), %eax
++ leal (startup32_vc_handler - 1b)(%ebp), %eax
+ movl $X86_TRAP_VC, %edx
++ movl %ebx, %ecx
+ call startup32_set_idt_entry
+
+ /* Load IDT */
+- leal rva(boot32_idt)(%ebp), %eax
+- movl %eax, rva(boot32_idt_desc+2)(%ebp)
+- lidt rva(boot32_idt_desc)(%ebp)
+-#endif
++ leal (boot32_idt_desc - 1b)(%ebp), %ecx
++ movl %ebx, 2(%ecx)
++ lidt (%ecx)
++
++ pop %ebx
++ pop %ebp
+ RET
+ SYM_FUNC_END(startup32_load_idt)
++#endif
+
+ /*
+ * Check for the correct C-bit position when the startup_32 boot-path is used.
+@@ -804,6 +812,7 @@ SYM_FUNC_END(startup32_load_idt)
+ * succeed. An incorrect C-bit position will map all memory unencrypted, so that
+ * the compare will use the encrypted random data and fail.
+ */
++ __HEAD
+ SYM_FUNC_START(startup32_check_sev_cbit)
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
+ pushl %eax
--- /dev/null
+From 9ea813be3d345dfb8ac5bf6fbb29e6a63647a39d Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:12 +0100
+Subject: x86/boot/compressed: Move startup32_load_idt() out of head_64.S
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 9ea813be3d345dfb8ac5bf6fbb29e6a63647a39d upstream.
+
+Now that startup32_load_idt() has been refactored into an ordinary
+callable function, move it into mem-encrypt.S where it belongs.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-13-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 72 ---------------------------------
+ arch/x86/boot/compressed/mem_encrypt.S | 72 ++++++++++++++++++++++++++++++++-
+ 2 files changed, 71 insertions(+), 73 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -723,78 +723,6 @@ SYM_DATA_START(boot_idt)
+ .endr
+ SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBAL, boot_idt_end)
+
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+-SYM_DATA_START(boot32_idt_desc)
+- .word boot32_idt_end - boot32_idt - 1
+- .long 0
+-SYM_DATA_END(boot32_idt_desc)
+- .balign 8
+-SYM_DATA_START(boot32_idt)
+- .rept 32
+- .quad 0
+- .endr
+-SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLOBAL, boot32_idt_end)
+-
+- .text
+- .code32
+-/*
+- * Write an IDT entry into boot32_idt
+- *
+- * Parameters:
+- *
+- * %eax: Handler address
+- * %edx: Vector number
+- * %ecx: IDT address
+- */
+-SYM_FUNC_START_LOCAL(startup32_set_idt_entry)
+- /* IDT entry address to %ecx */
+- leal (%ecx, %edx, 8), %ecx
+-
+- /* Build IDT entry, lower 4 bytes */
+- movl %eax, %edx
+- andl $0x0000ffff, %edx # Target code segment offset [15:0]
+- orl $(__KERNEL32_CS << 16), %edx # Target code segment selector
+-
+- /* Store lower 4 bytes to IDT */
+- movl %edx, (%ecx)
+-
+- /* Build IDT entry, upper 4 bytes */
+- movl %eax, %edx
+- andl $0xffff0000, %edx # Target code segment offset [31:16]
+- orl $0x00008e00, %edx # Present, Type 32-bit Interrupt Gate
+-
+- /* Store upper 4 bytes to IDT */
+- movl %edx, 4(%ecx)
+-
+- RET
+-SYM_FUNC_END(startup32_set_idt_entry)
+-
+-SYM_FUNC_START(startup32_load_idt)
+- push %ebp
+- push %ebx
+-
+- call 1f
+-1: pop %ebp
+-
+- leal (boot32_idt - 1b)(%ebp), %ebx
+-
+- /* #VC handler */
+- leal (startup32_vc_handler - 1b)(%ebp), %eax
+- movl $X86_TRAP_VC, %edx
+- movl %ebx, %ecx
+- call startup32_set_idt_entry
+-
+- /* Load IDT */
+- leal (boot32_idt_desc - 1b)(%ebp), %ecx
+- movl %ebx, 2(%ecx)
+- lidt (%ecx)
+-
+- pop %ebx
+- pop %ebp
+- RET
+-SYM_FUNC_END(startup32_load_idt)
+-#endif
+-
+ /*
+ * Check for the correct C-bit position when the startup_32 boot-path is used.
+ *
+--- a/arch/x86/boot/compressed/mem_encrypt.S
++++ b/arch/x86/boot/compressed/mem_encrypt.S
+@@ -12,6 +12,8 @@
+ #include <asm/processor-flags.h>
+ #include <asm/msr.h>
+ #include <asm/asm-offsets.h>
++#include <asm/segment.h>
++#include <asm/trapnr.h>
+
+ .text
+ .code32
+@@ -98,7 +100,7 @@ SYM_CODE_START_LOCAL(sev_es_req_cpuid)
+ jmp 1b
+ SYM_CODE_END(sev_es_req_cpuid)
+
+-SYM_CODE_START(startup32_vc_handler)
++SYM_CODE_START_LOCAL(startup32_vc_handler)
+ pushl %eax
+ pushl %ebx
+ pushl %ecx
+@@ -184,6 +186,63 @@ SYM_CODE_START(startup32_vc_handler)
+ jmp .Lfail
+ SYM_CODE_END(startup32_vc_handler)
+
++/*
++ * Write an IDT entry into boot32_idt
++ *
++ * Parameters:
++ *
++ * %eax: Handler address
++ * %edx: Vector number
++ * %ecx: IDT address
++ */
++SYM_FUNC_START_LOCAL(startup32_set_idt_entry)
++ /* IDT entry address to %ecx */
++ leal (%ecx, %edx, 8), %ecx
++
++ /* Build IDT entry, lower 4 bytes */
++ movl %eax, %edx
++ andl $0x0000ffff, %edx # Target code segment offset [15:0]
++ orl $(__KERNEL32_CS << 16), %edx # Target code segment selector
++
++ /* Store lower 4 bytes to IDT */
++ movl %edx, (%ecx)
++
++ /* Build IDT entry, upper 4 bytes */
++ movl %eax, %edx
++ andl $0xffff0000, %edx # Target code segment offset [31:16]
++ orl $0x00008e00, %edx # Present, Type 32-bit Interrupt Gate
++
++ /* Store upper 4 bytes to IDT */
++ movl %edx, 4(%ecx)
++
++ RET
++SYM_FUNC_END(startup32_set_idt_entry)
++
++SYM_FUNC_START(startup32_load_idt)
++ push %ebp
++ push %ebx
++
++ call 1f
++1: pop %ebp
++
++ leal (boot32_idt - 1b)(%ebp), %ebx
++
++ /* #VC handler */
++ leal (startup32_vc_handler - 1b)(%ebp), %eax
++ movl $X86_TRAP_VC, %edx
++ movl %ebx, %ecx
++ call startup32_set_idt_entry
++
++ /* Load IDT */
++ leal (boot32_idt_desc - 1b)(%ebp), %ecx
++ movl %ebx, 2(%ecx)
++ lidt (%ecx)
++
++ pop %ebx
++ pop %ebp
++ RET
++SYM_FUNC_END(startup32_load_idt)
++
+ .code64
+
+ #include "../../kernel/sev_verify_cbit.S"
+@@ -195,4 +254,15 @@ SYM_CODE_END(startup32_vc_handler)
+ SYM_DATA(sme_me_mask, .quad 0)
+ SYM_DATA(sev_status, .quad 0)
+ SYM_DATA(sev_check_data, .quad 0)
++
++SYM_DATA_START_LOCAL(boot32_idt)
++ .rept 32
++ .quad 0
++ .endr
++SYM_DATA_END(boot32_idt)
++
++SYM_DATA_START_LOCAL(boot32_idt_desc)
++ .word . - boot32_idt - 1
++ .long 0
++SYM_DATA_END(boot32_idt_desc)
+ #endif
--- /dev/null
+From 61de13df95901bc58456bc5acdbd3c18c66cf859 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:16 +0100
+Subject: x86/boot/compressed: Only build mem_encrypt.S if AMD_MEM_ENCRYPT=y
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 61de13df95901bc58456bc5acdbd3c18c66cf859 upstream.
+
+Avoid building the mem_encrypt.o object if memory encryption support is
+not enabled to begin with.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-17-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/Makefile | 2 +-
+ arch/x86/boot/compressed/mem_encrypt.S | 2 --
+ 2 files changed, 1 insertion(+), 3 deletions(-)
+
+--- a/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/boot/compressed/Makefile
+@@ -100,7 +100,7 @@ vmlinux-objs-$(CONFIG_RANDOMIZE_BASE) +=
+ ifdef CONFIG_X86_64
+ vmlinux-objs-y += $(obj)/ident_map_64.o
+ vmlinux-objs-y += $(obj)/idt_64.o $(obj)/idt_handlers_64.o
+- vmlinux-objs-y += $(obj)/mem_encrypt.o
++ vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/mem_encrypt.o
+ vmlinux-objs-y += $(obj)/pgtable_64.o
+ vmlinux-objs-$(CONFIG_AMD_MEM_ENCRYPT) += $(obj)/sev.o
+ endif
+--- a/arch/x86/boot/compressed/mem_encrypt.S
++++ b/arch/x86/boot/compressed/mem_encrypt.S
+@@ -307,7 +307,6 @@ SYM_FUNC_END(startup32_check_sev_cbit)
+
+ .data
+
+-#ifdef CONFIG_AMD_MEM_ENCRYPT
+ .balign 8
+ SYM_DATA(sme_me_mask, .quad 0)
+ SYM_DATA(sev_status, .quad 0)
+@@ -323,4 +322,3 @@ SYM_DATA_START_LOCAL(boot32_idt_desc)
+ .word . - boot32_idt - 1
+ .long 0
+ SYM_DATA_END(boot32_idt_desc)
+-#endif
--- /dev/null
+From d73a257f7f86871c3aac24dc20538e3983096647 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:10 +0100
+Subject: x86/boot/compressed: Pull global variable reference into startup32_load_idt()
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit d73a257f7f86871c3aac24dc20538e3983096647 upstream.
+
+In preparation for moving startup32_load_idt() out of head_64.S and
+turning it into an ordinary function using the ordinary 32-bit calling
+convention, pull the global variable reference to boot32_idt up into
+startup32_load_idt() so that startup32_set_idt_entry() does not need to
+discover its own runtime physical address, which will no longer be
+correlated with startup_32 once this code is moved into .text.
+
+While at it, give startup32_set_idt_entry() static linkage.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-11-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 20 ++++++++------------
+ 1 file changed, 8 insertions(+), 12 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -744,16 +744,11 @@ SYM_DATA_END_LABEL(boot32_idt, SYM_L_GLO
+ *
+ * %eax: Handler address
+ * %edx: Vector number
+- *
+- * Physical offset is expected in %ebp
++ * %ecx: IDT address
+ */
+-SYM_FUNC_START(startup32_set_idt_entry)
+- push %ebx
+-
+- /* IDT entry address to %ebx */
+- leal rva(boot32_idt)(%ebp), %ebx
+- shl $3, %edx
+- addl %edx, %ebx
++SYM_FUNC_START_LOCAL(startup32_set_idt_entry)
++ /* IDT entry address to %ecx */
++ leal (%ecx, %edx, 8), %ecx
+
+ /* Build IDT entry, lower 4 bytes */
+ movl %eax, %edx
+@@ -761,7 +756,7 @@ SYM_FUNC_START(startup32_set_idt_entry)
+ orl $(__KERNEL32_CS << 16), %edx # Target code segment selector
+
+ /* Store lower 4 bytes to IDT */
+- movl %edx, (%ebx)
++ movl %edx, (%ecx)
+
+ /* Build IDT entry, upper 4 bytes */
+ movl %eax, %edx
+@@ -769,15 +764,16 @@ SYM_FUNC_START(startup32_set_idt_entry)
+ orl $0x00008e00, %edx # Present, Type 32-bit Interrupt Gate
+
+ /* Store upper 4 bytes to IDT */
+- movl %edx, 4(%ebx)
++ movl %edx, 4(%ecx)
+
+- pop %ebx
+ RET
+ SYM_FUNC_END(startup32_set_idt_entry)
+ #endif
+
+ SYM_FUNC_START(startup32_load_idt)
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
++ leal rva(boot32_idt)(%ebp), %ecx
++
+ /* #VC handler */
+ leal rva(startup32_vc_handler)(%ebp), %eax
+ movl $X86_TRAP_VC, %edx
--- /dev/null
+From cb8bda8ad4438b4bcfcf89697fc84803fb210017 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:01 +0100
+Subject: x86/boot/compressed: Rename efi_thunk_64.S to efi-mixed.S
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit cb8bda8ad4438b4bcfcf89697fc84803fb210017 upstream.
+
+In preparation for moving the mixed mode specific code out of head_64.S,
+rename the existing file to clarify that it contains more than just the
+mixed mode thunk.
+
+While at it, clean up the Makefile rules that add it to the build.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-2-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/Makefile | 6 +++---
+ arch/x86/boot/compressed/{efi_thunk_64.S => efi_mixed.S} | 0
+ arch/x86/boot/compressed/Makefile | 6
+ arch/x86/boot/compressed/efi_mixed.S | 195 ++++++++++++++++++++++++++++++++
+ arch/x86/boot/compressed/efi_thunk_64.S | 195 --------------------------------
+ 3 files changed, 198 insertions(+), 198 deletions(-)
+ rename arch/x86/boot/compressed/{efi_thunk_64.S => efi_mixed.S} (100%)
+
+--- a/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/boot/compressed/Makefile
+@@ -108,11 +108,11 @@ endif
+ vmlinux-objs-$(CONFIG_ACPI) += $(obj)/acpi.o
+ vmlinux-objs-$(CONFIG_INTEL_TDX_GUEST) += $(obj)/tdx.o $(obj)/tdcall.o
+
+-vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_thunk_$(BITS).o
+ vmlinux-objs-$(CONFIG_EFI) += $(obj)/efi.o
+-efi-obj-$(CONFIG_EFI_STUB) = $(objtree)/drivers/firmware/efi/libstub/lib.a
++vmlinux-objs-$(CONFIG_EFI_MIXED) += $(obj)/efi_mixed.o
++vmlinux-objs-$(CONFIG_EFI_STUB) += $(objtree)/drivers/firmware/efi/libstub/lib.a
+
+-$(obj)/vmlinux: $(vmlinux-objs-y) $(efi-obj-y) FORCE
++$(obj)/vmlinux: $(vmlinux-objs-y) FORCE
+ $(call if_changed,ld)
+
+ OBJCOPYFLAGS_vmlinux.bin := -R .comment -S
+--- /dev/null
++++ b/arch/x86/boot/compressed/efi_mixed.S
+@@ -0,0 +1,195 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
++ *
++ * Early support for invoking 32-bit EFI services from a 64-bit kernel.
++ *
++ * Because this thunking occurs before ExitBootServices() we have to
++ * restore the firmware's 32-bit GDT and IDT before we make EFI service
++ * calls.
++ *
++ * On the plus side, we don't have to worry about mangling 64-bit
++ * addresses into 32-bits because we're executing with an identity
++ * mapped pagetable and haven't transitioned to 64-bit virtual addresses
++ * yet.
++ */
++
++#include <linux/linkage.h>
++#include <asm/msr.h>
++#include <asm/page_types.h>
++#include <asm/processor-flags.h>
++#include <asm/segment.h>
++
++ .code64
++ .text
++SYM_FUNC_START(__efi64_thunk)
++ push %rbp
++ push %rbx
++
++ movl %ds, %eax
++ push %rax
++ movl %es, %eax
++ push %rax
++ movl %ss, %eax
++ push %rax
++
++ /* Copy args passed on stack */
++ movq 0x30(%rsp), %rbp
++ movq 0x38(%rsp), %rbx
++ movq 0x40(%rsp), %rax
++
++ /*
++ * Convert x86-64 ABI params to i386 ABI
++ */
++ subq $64, %rsp
++ movl %esi, 0x0(%rsp)
++ movl %edx, 0x4(%rsp)
++ movl %ecx, 0x8(%rsp)
++ movl %r8d, 0xc(%rsp)
++ movl %r9d, 0x10(%rsp)
++ movl %ebp, 0x14(%rsp)
++ movl %ebx, 0x18(%rsp)
++ movl %eax, 0x1c(%rsp)
++
++ leaq 0x20(%rsp), %rbx
++ sgdt (%rbx)
++
++ addq $16, %rbx
++ sidt (%rbx)
++
++ leaq 1f(%rip), %rbp
++
++ /*
++ * Switch to IDT and GDT with 32-bit segments. This is the firmware GDT
++ * and IDT that was installed when the kernel started executing. The
++ * pointers were saved at the EFI stub entry point in head_64.S.
++ *
++ * Pass the saved DS selector to the 32-bit code, and use far return to
++ * restore the saved CS selector.
++ */
++ leaq efi32_boot_idt(%rip), %rax
++ lidt (%rax)
++ leaq efi32_boot_gdt(%rip), %rax
++ lgdt (%rax)
++
++ movzwl efi32_boot_ds(%rip), %edx
++ movzwq efi32_boot_cs(%rip), %rax
++ pushq %rax
++ leaq efi_enter32(%rip), %rax
++ pushq %rax
++ lretq
++
++1: addq $64, %rsp
++ movq %rdi, %rax
++
++ pop %rbx
++ movl %ebx, %ss
++ pop %rbx
++ movl %ebx, %es
++ pop %rbx
++ movl %ebx, %ds
++ /* Clear out 32-bit selector from FS and GS */
++ xorl %ebx, %ebx
++ movl %ebx, %fs
++ movl %ebx, %gs
++
++ /*
++ * Convert 32-bit status code into 64-bit.
++ */
++ roll $1, %eax
++ rorq $1, %rax
++
++ pop %rbx
++ pop %rbp
++ RET
++SYM_FUNC_END(__efi64_thunk)
++
++ .code32
++/*
++ * EFI service pointer must be in %edi.
++ *
++ * The stack should represent the 32-bit calling convention.
++ */
++SYM_FUNC_START_LOCAL(efi_enter32)
++ /* Load firmware selector into data and stack segment registers */
++ movl %edx, %ds
++ movl %edx, %es
++ movl %edx, %fs
++ movl %edx, %gs
++ movl %edx, %ss
++
++ /* Reload pgtables */
++ movl %cr3, %eax
++ movl %eax, %cr3
++
++ /* Disable paging */
++ movl %cr0, %eax
++ btrl $X86_CR0_PG_BIT, %eax
++ movl %eax, %cr0
++
++ /* Disable long mode via EFER */
++ movl $MSR_EFER, %ecx
++ rdmsr
++ btrl $_EFER_LME, %eax
++ wrmsr
++
++ call *%edi
++
++ /* We must preserve return value */
++ movl %eax, %edi
++
++ /*
++ * Some firmware will return with interrupts enabled. Be sure to
++ * disable them before we switch GDTs and IDTs.
++ */
++ cli
++
++ lidtl (%ebx)
++ subl $16, %ebx
++
++ lgdtl (%ebx)
++
++ movl %cr4, %eax
++ btsl $(X86_CR4_PAE_BIT), %eax
++ movl %eax, %cr4
++
++ movl %cr3, %eax
++ movl %eax, %cr3
++
++ movl $MSR_EFER, %ecx
++ rdmsr
++ btsl $_EFER_LME, %eax
++ wrmsr
++
++ xorl %eax, %eax
++ lldt %ax
++
++ pushl $__KERNEL_CS
++ pushl %ebp
++
++ /* Enable paging */
++ movl %cr0, %eax
++ btsl $X86_CR0_PG_BIT, %eax
++ movl %eax, %cr0
++ lret
++SYM_FUNC_END(efi_enter32)
++
++ .data
++ .balign 8
++SYM_DATA_START(efi32_boot_gdt)
++ .word 0
++ .quad 0
++SYM_DATA_END(efi32_boot_gdt)
++
++SYM_DATA_START(efi32_boot_idt)
++ .word 0
++ .quad 0
++SYM_DATA_END(efi32_boot_idt)
++
++SYM_DATA_START(efi32_boot_cs)
++ .word 0
++SYM_DATA_END(efi32_boot_cs)
++
++SYM_DATA_START(efi32_boot_ds)
++ .word 0
++SYM_DATA_END(efi32_boot_ds)
+--- a/arch/x86/boot/compressed/efi_thunk_64.S
++++ /dev/null
+@@ -1,195 +0,0 @@
+-/* SPDX-License-Identifier: GPL-2.0 */
+-/*
+- * Copyright (C) 2014, 2015 Intel Corporation; author Matt Fleming
+- *
+- * Early support for invoking 32-bit EFI services from a 64-bit kernel.
+- *
+- * Because this thunking occurs before ExitBootServices() we have to
+- * restore the firmware's 32-bit GDT and IDT before we make EFI service
+- * calls.
+- *
+- * On the plus side, we don't have to worry about mangling 64-bit
+- * addresses into 32-bits because we're executing with an identity
+- * mapped pagetable and haven't transitioned to 64-bit virtual addresses
+- * yet.
+- */
+-
+-#include <linux/linkage.h>
+-#include <asm/msr.h>
+-#include <asm/page_types.h>
+-#include <asm/processor-flags.h>
+-#include <asm/segment.h>
+-
+- .code64
+- .text
+-SYM_FUNC_START(__efi64_thunk)
+- push %rbp
+- push %rbx
+-
+- movl %ds, %eax
+- push %rax
+- movl %es, %eax
+- push %rax
+- movl %ss, %eax
+- push %rax
+-
+- /* Copy args passed on stack */
+- movq 0x30(%rsp), %rbp
+- movq 0x38(%rsp), %rbx
+- movq 0x40(%rsp), %rax
+-
+- /*
+- * Convert x86-64 ABI params to i386 ABI
+- */
+- subq $64, %rsp
+- movl %esi, 0x0(%rsp)
+- movl %edx, 0x4(%rsp)
+- movl %ecx, 0x8(%rsp)
+- movl %r8d, 0xc(%rsp)
+- movl %r9d, 0x10(%rsp)
+- movl %ebp, 0x14(%rsp)
+- movl %ebx, 0x18(%rsp)
+- movl %eax, 0x1c(%rsp)
+-
+- leaq 0x20(%rsp), %rbx
+- sgdt (%rbx)
+-
+- addq $16, %rbx
+- sidt (%rbx)
+-
+- leaq 1f(%rip), %rbp
+-
+- /*
+- * Switch to IDT and GDT with 32-bit segments. This is the firmware GDT
+- * and IDT that was installed when the kernel started executing. The
+- * pointers were saved at the EFI stub entry point in head_64.S.
+- *
+- * Pass the saved DS selector to the 32-bit code, and use far return to
+- * restore the saved CS selector.
+- */
+- leaq efi32_boot_idt(%rip), %rax
+- lidt (%rax)
+- leaq efi32_boot_gdt(%rip), %rax
+- lgdt (%rax)
+-
+- movzwl efi32_boot_ds(%rip), %edx
+- movzwq efi32_boot_cs(%rip), %rax
+- pushq %rax
+- leaq efi_enter32(%rip), %rax
+- pushq %rax
+- lretq
+-
+-1: addq $64, %rsp
+- movq %rdi, %rax
+-
+- pop %rbx
+- movl %ebx, %ss
+- pop %rbx
+- movl %ebx, %es
+- pop %rbx
+- movl %ebx, %ds
+- /* Clear out 32-bit selector from FS and GS */
+- xorl %ebx, %ebx
+- movl %ebx, %fs
+- movl %ebx, %gs
+-
+- /*
+- * Convert 32-bit status code into 64-bit.
+- */
+- roll $1, %eax
+- rorq $1, %rax
+-
+- pop %rbx
+- pop %rbp
+- RET
+-SYM_FUNC_END(__efi64_thunk)
+-
+- .code32
+-/*
+- * EFI service pointer must be in %edi.
+- *
+- * The stack should represent the 32-bit calling convention.
+- */
+-SYM_FUNC_START_LOCAL(efi_enter32)
+- /* Load firmware selector into data and stack segment registers */
+- movl %edx, %ds
+- movl %edx, %es
+- movl %edx, %fs
+- movl %edx, %gs
+- movl %edx, %ss
+-
+- /* Reload pgtables */
+- movl %cr3, %eax
+- movl %eax, %cr3
+-
+- /* Disable paging */
+- movl %cr0, %eax
+- btrl $X86_CR0_PG_BIT, %eax
+- movl %eax, %cr0
+-
+- /* Disable long mode via EFER */
+- movl $MSR_EFER, %ecx
+- rdmsr
+- btrl $_EFER_LME, %eax
+- wrmsr
+-
+- call *%edi
+-
+- /* We must preserve return value */
+- movl %eax, %edi
+-
+- /*
+- * Some firmware will return with interrupts enabled. Be sure to
+- * disable them before we switch GDTs and IDTs.
+- */
+- cli
+-
+- lidtl (%ebx)
+- subl $16, %ebx
+-
+- lgdtl (%ebx)
+-
+- movl %cr4, %eax
+- btsl $(X86_CR4_PAE_BIT), %eax
+- movl %eax, %cr4
+-
+- movl %cr3, %eax
+- movl %eax, %cr3
+-
+- movl $MSR_EFER, %ecx
+- rdmsr
+- btsl $_EFER_LME, %eax
+- wrmsr
+-
+- xorl %eax, %eax
+- lldt %ax
+-
+- pushl $__KERNEL_CS
+- pushl %ebp
+-
+- /* Enable paging */
+- movl %cr0, %eax
+- btsl $X86_CR0_PG_BIT, %eax
+- movl %eax, %cr0
+- lret
+-SYM_FUNC_END(efi_enter32)
+-
+- .data
+- .balign 8
+-SYM_DATA_START(efi32_boot_gdt)
+- .word 0
+- .quad 0
+-SYM_DATA_END(efi32_boot_gdt)
+-
+-SYM_DATA_START(efi32_boot_idt)
+- .word 0
+- .quad 0
+-SYM_DATA_END(efi32_boot_idt)
+-
+-SYM_DATA_START(efi32_boot_cs)
+- .word 0
+-SYM_DATA_END(efi32_boot_cs)
+-
+-SYM_DATA_START(efi32_boot_ds)
+- .word 0
+-SYM_DATA_END(efi32_boot_ds)
--- /dev/null
+From 630f337f0c4fd80390e8600adcab31550aea33df Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:08 +0100
+Subject: x86/boot/compressed: Simplify IDT/GDT preserve/restore in the EFI thunk
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 630f337f0c4fd80390e8600adcab31550aea33df upstream.
+
+Tweak the asm and remove some redundant instructions. While at it,
+fix the associated comment for style and correctness.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-9-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/efi_mixed.S | 20 +++++++-------------
+ 1 file changed, 7 insertions(+), 13 deletions(-)
+
+--- a/arch/x86/boot/compressed/efi_mixed.S
++++ b/arch/x86/boot/compressed/efi_mixed.S
+@@ -96,24 +96,20 @@ SYM_FUNC_START(__efi64_thunk)
+
+ leaq 0x20(%rsp), %rbx
+ sgdt (%rbx)
+-
+- addq $16, %rbx
+- sidt (%rbx)
++ sidt 16(%rbx)
+
+ leaq 1f(%rip), %rbp
+
+ /*
+- * Switch to IDT and GDT with 32-bit segments. This is the firmware GDT
+- * and IDT that was installed when the kernel started executing. The
+- * pointers were saved by the efi32_entry() routine below.
++ * Switch to IDT and GDT with 32-bit segments. These are the firmware
++ * GDT and IDT that were installed when the kernel started executing.
++ * The pointers were saved by the efi32_entry() routine below.
+ *
+ * Pass the saved DS selector to the 32-bit code, and use far return to
+ * restore the saved CS selector.
+ */
+- leaq efi32_boot_idt(%rip), %rax
+- lidt (%rax)
+- leaq efi32_boot_gdt(%rip), %rax
+- lgdt (%rax)
++ lidt efi32_boot_idt(%rip)
++ lgdt efi32_boot_gdt(%rip)
+
+ movzwl efi32_boot_ds(%rip), %edx
+ movzwq efi32_boot_cs(%rip), %rax
+@@ -187,9 +183,7 @@ SYM_FUNC_START_LOCAL(efi_enter32)
+ */
+ cli
+
+- lidtl (%ebx)
+- subl $16, %ebx
+-
++ lidtl 16(%ebx)
+ lgdtl (%ebx)
+
+ movl %cr4, %eax
--- /dev/null
+From 7734a0f31e99c433df3063bbb7e8ee5a16a2cb82 Mon Sep 17 00:00:00 2001
+From: Alexander Lobakin <alexandr.lobakin@intel.com>
+Date: Mon, 9 Jan 2023 18:04:02 +0100
+Subject: x86/boot: Robustify calling startup_{32,64}() from the decompressor code
+
+From: Alexander Lobakin <alexandr.lobakin@intel.com>
+
+commit 7734a0f31e99c433df3063bbb7e8ee5a16a2cb82 upstream.
+
+After commit ce697ccee1a8 ("kbuild: remove head-y syntax"), I
+started digging whether x86 is ready for removing this old cruft.
+Removing its objects from the list makes the kernel unbootable.
+This applies only to bzImage, vmlinux still works correctly.
+The reason is that with no strict object order determined by the
+linker arguments, not the linker script, startup_64 can be placed
+not right at the beginning of the kernel.
+Here's vmlinux.map's beginning before removing:
+
+ ffffffff81000000 vmlinux.o:(.head.text)
+ ffffffff81000000 startup_64
+ ffffffff81000070 secondary_startup_64
+ ffffffff81000075 secondary_startup_64_no_verify
+ ffffffff81000160 verify_cpu
+
+and after:
+
+ ffffffff81000000 vmlinux.o:(.head.text)
+ ffffffff81000000 pvh_start_xen
+ ffffffff81000080 startup_64
+ ffffffff810000f0 secondary_startup_64
+ ffffffff810000f5 secondary_startup_64_no_verify
+
+Not a problem itself, but the self-extractor code has the address of
+that function hardcoded the beginning, not looking onto the ELF
+header, which always contains the address of startup_{32,64}().
+
+So, instead of doing an "act of blind faith", just take the address
+from the ELF header and extract a relative offset to the entry
+point. The decompressor function already returns a pointer to the
+beginning of the kernel to the Asm code, which then jumps to it,
+so add that offset to the return value.
+This doesn't change anything for now, but allows to resign from the
+"head object list" for x86 and makes sure valid Kbuild or any other
+improvements won't break anything here in general.
+
+Signed-off-by: Alexander Lobakin <alexandr.lobakin@intel.com>
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Tested-by: Jiri Slaby <jirislaby@kernel.org>
+Cc: "H. Peter Anvin" <hpa@zytor.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Link: https://lore.kernel.org/r/20230109170403.4117105-2-alexandr.lobakin@intel.com
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_32.S | 2 +-
+ arch/x86/boot/compressed/head_64.S | 2 +-
+ arch/x86/boot/compressed/misc.c | 18 +++++++++++-------
+ 3 files changed, 13 insertions(+), 9 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_32.S
++++ b/arch/x86/boot/compressed/head_32.S
+@@ -187,7 +187,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ leal boot_heap@GOTOFF(%ebx), %eax
+ pushl %eax /* heap area */
+ pushl %esi /* real mode pointer */
+- call extract_kernel /* returns kernel location in %eax */
++ call extract_kernel /* returns kernel entry point in %eax */
+ addl $24, %esp
+
+ /*
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -580,7 +580,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ movl input_len(%rip), %ecx /* input_len */
+ movq %rbp, %r8 /* output target address */
+ movl output_len(%rip), %r9d /* decompressed length, end of relocs */
+- call extract_kernel /* returns kernel location in %rax */
++ call extract_kernel /* returns kernel entry point in %rax */
+ popq %rsi
+
+ /*
+--- a/arch/x86/boot/compressed/misc.c
++++ b/arch/x86/boot/compressed/misc.c
+@@ -277,7 +277,7 @@ static inline void handle_relocations(vo
+ { }
+ #endif
+
+-static void parse_elf(void *output)
++static size_t parse_elf(void *output)
+ {
+ #ifdef CONFIG_X86_64
+ Elf64_Ehdr ehdr;
+@@ -293,10 +293,8 @@ static void parse_elf(void *output)
+ if (ehdr.e_ident[EI_MAG0] != ELFMAG0 ||
+ ehdr.e_ident[EI_MAG1] != ELFMAG1 ||
+ ehdr.e_ident[EI_MAG2] != ELFMAG2 ||
+- ehdr.e_ident[EI_MAG3] != ELFMAG3) {
++ ehdr.e_ident[EI_MAG3] != ELFMAG3)
+ error("Kernel is not a valid ELF file");
+- return;
+- }
+
+ debug_putstr("Parsing ELF... ");
+
+@@ -328,6 +326,8 @@ static void parse_elf(void *output)
+ }
+
+ free(phdrs);
++
++ return ehdr.e_entry - LOAD_PHYSICAL_ADDR;
+ }
+
+ /*
+@@ -356,6 +356,7 @@ asmlinkage __visible void *extract_kerne
+ const unsigned long kernel_total_size = VO__end - VO__text;
+ unsigned long virt_addr = LOAD_PHYSICAL_ADDR;
+ unsigned long needed_size;
++ size_t entry_offset;
+
+ /* Retain x86 boot parameters pointer passed from startup_32/64. */
+ boot_params = rmode;
+@@ -456,14 +457,17 @@ asmlinkage __visible void *extract_kerne
+ debug_putstr("\nDecompressing Linux... ");
+ __decompress(input_data, input_len, NULL, NULL, output, output_len,
+ NULL, error);
+- parse_elf(output);
++ entry_offset = parse_elf(output);
+ handle_relocations(output, output_len, virt_addr);
+- debug_putstr("done.\nBooting the kernel.\n");
++
++ debug_putstr("done.\nBooting the kernel (entry_offset: 0x");
++ debug_puthex(entry_offset);
++ debug_putstr(").\n");
+
+ /* Disable exception handling before booting the kernel */
+ cleanup_exception_handling();
+
+- return output;
++ return output + entry_offset;
+ }
+
+ void fortify_panic(const char *name)
--- /dev/null
+From 00c6b0978ec182f1a672095930872168b9d5b1e2 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:05 +0200
+Subject: x86/decompressor: Assign paging related global variables earlier
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 00c6b0978ec182f1a672095930872168b9d5b1e2 upstream.
+
+There is no need to defer the assignment of the paging related global
+variables 'pgdir_shift' and 'ptrs_per_p4d' until after the trampoline is
+cleaned up, so assign them as soon as it is clear that 5-level paging
+will be enabled.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-9-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/misc.h | 2 --
+ arch/x86/boot/compressed/pgtable_64.c | 14 +++++---------
+ 2 files changed, 5 insertions(+), 11 deletions(-)
+
+--- a/arch/x86/boot/compressed/misc.h
++++ b/arch/x86/boot/compressed/misc.h
+@@ -170,9 +170,7 @@ static inline int count_immovable_mem_re
+ #endif
+
+ /* ident_map_64.c */
+-#ifdef CONFIG_X86_5LEVEL
+ extern unsigned int __pgtable_l5_enabled, pgdir_shift, ptrs_per_p4d;
+-#endif
+ extern void kernel_add_identity_map(unsigned long start, unsigned long end);
+
+ /* Used by PAGE_KERN* macros: */
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -130,6 +130,11 @@ struct paging_config paging_prepare(void
+ native_cpuid_eax(0) >= 7 &&
+ (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
+ paging_config.l5_required = 1;
++
++ /* Initialize variables for 5-level paging */
++ __pgtable_l5_enabled = 1;
++ pgdir_shift = 48;
++ ptrs_per_p4d = 512;
+ }
+
+ paging_config.trampoline_start = find_trampoline_placement();
+@@ -206,13 +211,4 @@ void cleanup_trampoline(void *pgtable)
+
+ /* Restore trampoline memory */
+ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
+-
+- /* Initialize variables for 5-level paging */
+-#ifdef CONFIG_X86_5LEVEL
+- if (__read_cr4() & X86_CR4_LA57) {
+- __pgtable_l5_enabled = 1;
+- pgdir_shift = 48;
+- ptrs_per_p4d = 512;
+- }
+-#endif
+ }
--- /dev/null
+From bd328aa01ff77a45aeffea5fc4521854291db11f Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:08 +0200
+Subject: x86/decompressor: Avoid the need for a stack in the 32-bit trampoline
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit bd328aa01ff77a45aeffea5fc4521854291db11f upstream.
+
+The 32-bit trampoline no longer uses the stack for anything except
+performing a far return back to long mode, and preserving the caller's
+stack pointer value. Currently, the trampoline stack is placed in the
+same page that carries the trampoline code, which means this page must
+be mapped writable and executable, and the stack is therefore executable
+as well.
+
+Replace the far return with a far jump, so that the return address can
+be pre-calculated and patched into the code before it is called. This
+removes the need for a 32-bit addressable stack entirely, and in a later
+patch, this will be taken advantage of by removing writable permissions
+from (and adding executable permissions to) the trampoline code page
+when booting via the EFI stub.
+
+Note that the value of RSP still needs to be preserved explicitly across
+the switch into 32-bit mode, as the register may get truncated to 32
+bits.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-12-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 45 ++++++++++++++++++++--------------
+ arch/x86/boot/compressed/pgtable.h | 4 +--
+ arch/x86/boot/compressed/pgtable_64.c | 12 ++++++++-
+ 3 files changed, 40 insertions(+), 21 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -558,6 +558,7 @@ SYM_FUNC_END(.Lrelocated)
+ * trampoline memory. A non-zero second argument (ESI) means that the
+ * trampoline needs to enable 5-level paging.
+ */
++ .section ".rodata", "a", @progbits
+ SYM_CODE_START(trampoline_32bit_src)
+ /*
+ * Preserve live 64-bit registers on the stack: this is necessary
+@@ -568,13 +569,9 @@ SYM_CODE_START(trampoline_32bit_src)
+ pushq %rbp
+ pushq %rbx
+
+- /* Set up 32-bit addressable stack and push the old RSP value */
+- leaq (TRAMPOLINE_32BIT_STACK_END - 8)(%rcx), %rbx
+- movq %rsp, (%rbx)
+- movq %rbx, %rsp
+-
+- /* Take the address of the trampoline exit code */
+- leaq .Lret(%rip), %rbx
++ /* Preserve top half of RSP in a legacy mode GPR to avoid truncation */
++ movq %rsp, %rbx
++ shrq $32, %rbx
+
+ /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
+ pushq $__KERNEL32_CS
+@@ -582,9 +579,17 @@ SYM_CODE_START(trampoline_32bit_src)
+ pushq %rax
+ lretq
+
++ /*
++ * The 32-bit code below will do a far jump back to long mode and end
++ * up here after reconfiguring the number of paging levels. First, the
++ * stack pointer needs to be restored to its full 64-bit value before
++ * the callee save register contents can be popped from the stack.
++ */
+ .Lret:
++ shlq $32, %rbx
++ orq %rbx, %rsp
++
+ /* Restore the preserved 64-bit registers */
+- movq (%rsp), %rsp
+ popq %rbx
+ popq %rbp
+ popq %r15
+@@ -592,11 +597,6 @@ SYM_CODE_START(trampoline_32bit_src)
+
+ .code32
+ 0:
+- /* Set up data and stack segments */
+- movl $__KERNEL_DS, %eax
+- movl %eax, %ds
+- movl %eax, %ss
+-
+ /* Disable paging */
+ movl %cr0, %eax
+ btrl $X86_CR0_PG_BIT, %eax
+@@ -651,18 +651,26 @@ SYM_CODE_START(trampoline_32bit_src)
+ 1:
+ movl %eax, %cr4
+
+- /* Prepare the stack for far return to Long Mode */
+- pushl $__KERNEL_CS
+- pushl %ebx
+-
+ /* Enable paging again. */
+ movl %cr0, %eax
+ btsl $X86_CR0_PG_BIT, %eax
+ movl %eax, %cr0
+
+- lret
++ /*
++ * Return to the 64-bit calling code using LJMP rather than LRET, to
++ * avoid the need for a 32-bit addressable stack. The destination
++ * address will be adjusted after the template code is copied into a
++ * 32-bit addressable buffer.
++ */
++.Ljmp: ljmpl $__KERNEL_CS, $(.Lret - trampoline_32bit_src)
+ SYM_CODE_END(trampoline_32bit_src)
+
++/*
++ * This symbol is placed right after trampoline_32bit_src() so its address can
++ * be used to infer the size of the trampoline code.
++ */
++SYM_DATA(trampoline_ljmp_imm_offset, .word .Ljmp + 1 - trampoline_32bit_src)
++
+ /*
+ * The trampoline code has a size limit.
+ * Make sure we fail to compile if the trampoline code grows
+@@ -670,6 +678,7 @@ SYM_CODE_END(trampoline_32bit_src)
+ */
+ .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE
+
++ .text
+ SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmode)
+ /* This isn't an x86-64 CPU, so hang intentionally, we cannot continue */
+ 1:
+--- a/arch/x86/boot/compressed/pgtable.h
++++ b/arch/x86/boot/compressed/pgtable.h
+@@ -8,13 +8,13 @@
+ #define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE
+ #define TRAMPOLINE_32BIT_CODE_SIZE 0xA0
+
+-#define TRAMPOLINE_32BIT_STACK_END TRAMPOLINE_32BIT_SIZE
+-
+ #ifndef __ASSEMBLER__
+
+ extern unsigned long *trampoline_32bit;
+
+ extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+
++extern const u16 trampoline_ljmp_imm_offset;
++
+ #endif /* __ASSEMBLER__ */
+ #endif /* BOOT_COMPRESSED_PAGETABLE_H */
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -109,6 +109,7 @@ static unsigned long find_trampoline_pla
+ struct paging_config paging_prepare(void *rmode)
+ {
+ struct paging_config paging_config = {};
++ void *tramp_code;
+
+ /* Initialize boot_params. Required for cmdline_find_option_bool(). */
+ boot_params = rmode;
+@@ -148,10 +149,19 @@ struct paging_config paging_prepare(void
+ memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
+
+ /* Copy trampoline code in place */
+- memcpy(trampoline_32bit + TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
++ tramp_code = memcpy(trampoline_32bit +
++ TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
+ &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
+
+ /*
++ * Avoid the need for a stack in the 32-bit trampoline code, by using
++ * LJMP rather than LRET to return back to long mode. LJMP takes an
++ * immediate absolute address, which needs to be adjusted based on the
++ * placement of the trampoline.
++ */
++ *(u32 *)(tramp_code + trampoline_ljmp_imm_offset) += (unsigned long)tramp_code;
++
++ /*
+ * The code below prepares page table in trampoline memory.
+ *
+ * The new page table will be used by trampoline code for switching
--- /dev/null
+From e8972a76aa90c05a0078043413f806c02fcb3487 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:06 +0200
+Subject: x86/decompressor: Call trampoline as a normal function
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit e8972a76aa90c05a0078043413f806c02fcb3487 upstream.
+
+Move the long return to switch to 32-bit mode into the trampoline code
+so it can be called as an ordinary function. This will allow it to be
+called directly from C code in a subsequent patch.
+
+While at it, reorganize the code somewhat to keep the prologue and
+epilogue of the function together, making the code a bit easier to
+follow. Also, given that the trampoline is now entered in 64-bit mode, a
+simple RIP-relative reference can be used to take the address of the
+exit point.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-10-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 79 ++++++++++++++++---------------------
+ arch/x86/boot/compressed/pgtable.h | 2
+ 2 files changed, 36 insertions(+), 45 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -450,39 +450,8 @@ SYM_CODE_START(startup_64)
+ /* Save the trampoline address in RCX */
+ movq %rax, %rcx
+
+- /* Set up 32-bit addressable stack */
+- leaq TRAMPOLINE_32BIT_STACK_END(%rcx), %rsp
+-
+- /*
+- * Preserve live 64-bit registers on the stack: this is necessary
+- * because the architecture does not guarantee that GPRs will retain
+- * their full 64-bit values across a 32-bit mode switch.
+- */
+- pushq %r15
+- pushq %rbp
+- pushq %rbx
+-
+- /*
+- * Push the 64-bit address of trampoline_return() onto the new stack.
+- * It will be used by the trampoline to return to the main code. Due to
+- * the 32-bit mode switch, it cannot be kept it in a register either.
+- */
+- leaq trampoline_return(%rip), %rdi
+- pushq %rdi
+-
+- /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
+- pushq $__KERNEL32_CS
+ leaq TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax
+- pushq %rax
+- lretq
+-trampoline_return:
+- /* Restore live 64-bit registers */
+- popq %rbx
+- popq %rbp
+- popq %r15
+-
+- /* Restore the stack, the 32-bit trampoline uses its own stack */
+- leaq rva(boot_stack_end)(%rbx), %rsp
++ call *%rax
+
+ /*
+ * cleanup_trampoline() would restore trampoline memory.
+@@ -579,7 +548,6 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ jmp *%rax
+ SYM_FUNC_END(.Lrelocated)
+
+- .code32
+ /*
+ * This is the 32-bit trampoline that will be copied over to low memory.
+ *
+@@ -588,6 +556,39 @@ SYM_FUNC_END(.Lrelocated)
+ * Non zero RDX means trampoline needs to enable 5-level paging.
+ */
+ SYM_CODE_START(trampoline_32bit_src)
++ /*
++ * Preserve live 64-bit registers on the stack: this is necessary
++ * because the architecture does not guarantee that GPRs will retain
++ * their full 64-bit values across a 32-bit mode switch.
++ */
++ pushq %r15
++ pushq %rbp
++ pushq %rbx
++
++ /* Set up 32-bit addressable stack and push the old RSP value */
++ leaq (TRAMPOLINE_32BIT_STACK_END - 8)(%rcx), %rbx
++ movq %rsp, (%rbx)
++ movq %rbx, %rsp
++
++ /* Take the address of the trampoline exit code */
++ leaq .Lret(%rip), %rbx
++
++ /* Switch to compatibility mode (CS.L = 0 CS.D = 1) via far return */
++ pushq $__KERNEL32_CS
++ leaq 0f(%rip), %rax
++ pushq %rax
++ lretq
++
++.Lret:
++ /* Restore the preserved 64-bit registers */
++ movq (%rsp), %rsp
++ popq %rbx
++ popq %rbp
++ popq %r15
++ retq
++
++ .code32
++0:
+ /* Set up data and stack segments */
+ movl $__KERNEL_DS, %eax
+ movl %eax, %ds
+@@ -651,12 +652,9 @@ SYM_CODE_START(trampoline_32bit_src)
+ 1:
+ movl %eax, %cr4
+
+- /* Calculate address of paging_enabled() once we are executing in the trampoline */
+- leal .Lpaging_enabled - trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_OFFSET(%ecx), %eax
+-
+ /* Prepare the stack for far return to Long Mode */
+ pushl $__KERNEL_CS
+- pushl %eax
++ pushl %ebx
+
+ /* Enable paging again. */
+ movl %cr0, %eax
+@@ -666,12 +664,6 @@ SYM_CODE_START(trampoline_32bit_src)
+ lret
+ SYM_CODE_END(trampoline_32bit_src)
+
+- .code64
+-SYM_FUNC_START_LOCAL_NOALIGN(.Lpaging_enabled)
+- /* Return from the trampoline */
+- retq
+-SYM_FUNC_END(.Lpaging_enabled)
+-
+ /*
+ * The trampoline code has a size limit.
+ * Make sure we fail to compile if the trampoline code grows
+@@ -679,7 +671,6 @@ SYM_FUNC_END(.Lpaging_enabled)
+ */
+ .org trampoline_32bit_src + TRAMPOLINE_32BIT_CODE_SIZE
+
+- .code32
+ SYM_FUNC_START_LOCAL_NOALIGN(.Lno_longmode)
+ /* This isn't an x86-64 CPU, so hang intentionally, we cannot continue */
+ 1:
+--- a/arch/x86/boot/compressed/pgtable.h
++++ b/arch/x86/boot/compressed/pgtable.h
+@@ -6,7 +6,7 @@
+ #define TRAMPOLINE_32BIT_PGTABLE_OFFSET 0
+
+ #define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE
+-#define TRAMPOLINE_32BIT_CODE_SIZE 0x80
++#define TRAMPOLINE_32BIT_CODE_SIZE 0xA0
+
+ #define TRAMPOLINE_32BIT_STACK_END TRAMPOLINE_32BIT_SIZE
+
--- /dev/null
+From 64ef578b6b6866bec012544416946533444036c8 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:09 +0200
+Subject: x86/decompressor: Call trampoline directly from C code
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 64ef578b6b6866bec012544416946533444036c8 upstream.
+
+Instead of returning to the asm calling code to invoke the trampoline,
+call it straight from the C code that sets it up. That way, the struct
+return type is no longer needed for returning two values, and the call
+can be made conditional more cleanly in a subsequent patch.
+
+This means that all callee save 64-bit registers need to be preserved
+and restored, as their contents may not survive the legacy mode switch.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-13-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 31 +++++++++++++------------------
+ arch/x86/boot/compressed/pgtable_64.c | 32 +++++++++++++-------------------
+ 2 files changed, 26 insertions(+), 37 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -433,25 +433,14 @@ SYM_CODE_START(startup_64)
+ #endif
+
+ /*
+- * paging_prepare() sets up the trampoline and checks if we need to
+- * enable 5-level paging.
+- *
+- * paging_prepare() returns a two-quadword structure which lands
+- * into RDX:RAX:
+- * - Address of the trampoline is returned in RAX.
+- * - Non zero RDX means trampoline needs to enable 5-level
+- * paging.
++ * configure_5level_paging() updates the number of paging levels using
++ * a trampoline in 32-bit addressable memory if the current number does
++ * not match the desired number.
+ *
+ * Pass the boot_params pointer as the first argument.
+ */
+ movq %r15, %rdi
+- call paging_prepare
+-
+- /* Pass the trampoline address and boolean flag as args #1 and #2 */
+- movq %rax, %rdi
+- movq %rdx, %rsi
+- leaq TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax
+- call *%rax
++ call configure_5level_paging
+
+ /*
+ * cleanup_trampoline() would restore trampoline memory.
+@@ -561,11 +550,14 @@ SYM_FUNC_END(.Lrelocated)
+ .section ".rodata", "a", @progbits
+ SYM_CODE_START(trampoline_32bit_src)
+ /*
+- * Preserve live 64-bit registers on the stack: this is necessary
+- * because the architecture does not guarantee that GPRs will retain
+- * their full 64-bit values across a 32-bit mode switch.
++ * Preserve callee save 64-bit registers on the stack: this is
++ * necessary because the architecture does not guarantee that GPRs will
++ * retain their full 64-bit values across a 32-bit mode switch.
+ */
+ pushq %r15
++ pushq %r14
++ pushq %r13
++ pushq %r12
+ pushq %rbp
+ pushq %rbx
+
+@@ -592,6 +584,9 @@ SYM_CODE_START(trampoline_32bit_src)
+ /* Restore the preserved 64-bit registers */
+ popq %rbx
+ popq %rbp
++ popq %r12
++ popq %r13
++ popq %r14
+ popq %r15
+ retq
+
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -16,11 +16,6 @@ unsigned int __section(".data") pgdir_sh
+ unsigned int __section(".data") ptrs_per_p4d = 1;
+ #endif
+
+-struct paging_config {
+- unsigned long trampoline_start;
+- unsigned long l5_required;
+-};
+-
+ /* Buffer to preserve trampoline memory */
+ static char trampoline_save[TRAMPOLINE_32BIT_SIZE];
+
+@@ -29,7 +24,7 @@ static char trampoline_save[TRAMPOLINE_3
+ * purposes.
+ *
+ * Avoid putting the pointer into .bss as it will be cleared between
+- * paging_prepare() and extract_kernel().
++ * configure_5level_paging() and extract_kernel().
+ */
+ unsigned long *trampoline_32bit __section(".data");
+
+@@ -106,13 +101,13 @@ static unsigned long find_trampoline_pla
+ return bios_start - TRAMPOLINE_32BIT_SIZE;
+ }
+
+-struct paging_config paging_prepare(void *rmode)
++asmlinkage void configure_5level_paging(struct boot_params *bp)
+ {
+- struct paging_config paging_config = {};
+- void *tramp_code;
++ void (*toggle_la57)(void *trampoline, bool enable_5lvl);
++ bool l5_required = false;
+
+ /* Initialize boot_params. Required for cmdline_find_option_bool(). */
+- boot_params = rmode;
++ boot_params = bp;
+
+ /*
+ * Check if LA57 is desired and supported.
+@@ -130,7 +125,7 @@ struct paging_config paging_prepare(void
+ !cmdline_find_option_bool("no5lvl") &&
+ native_cpuid_eax(0) >= 7 &&
+ (native_cpuid_ecx(7) & (1 << (X86_FEATURE_LA57 & 31)))) {
+- paging_config.l5_required = 1;
++ l5_required = true;
+
+ /* Initialize variables for 5-level paging */
+ __pgtable_l5_enabled = 1;
+@@ -138,9 +133,7 @@ struct paging_config paging_prepare(void
+ ptrs_per_p4d = 512;
+ }
+
+- paging_config.trampoline_start = find_trampoline_placement();
+-
+- trampoline_32bit = (unsigned long *)paging_config.trampoline_start;
++ trampoline_32bit = (unsigned long *)find_trampoline_placement();
+
+ /* Preserve trampoline memory */
+ memcpy(trampoline_save, trampoline_32bit, TRAMPOLINE_32BIT_SIZE);
+@@ -149,7 +142,7 @@ struct paging_config paging_prepare(void
+ memset(trampoline_32bit, 0, TRAMPOLINE_32BIT_SIZE);
+
+ /* Copy trampoline code in place */
+- tramp_code = memcpy(trampoline_32bit +
++ toggle_la57 = memcpy(trampoline_32bit +
+ TRAMPOLINE_32BIT_CODE_OFFSET / sizeof(unsigned long),
+ &trampoline_32bit_src, TRAMPOLINE_32BIT_CODE_SIZE);
+
+@@ -159,7 +152,8 @@ struct paging_config paging_prepare(void
+ * immediate absolute address, which needs to be adjusted based on the
+ * placement of the trampoline.
+ */
+- *(u32 *)(tramp_code + trampoline_ljmp_imm_offset) += (unsigned long)tramp_code;
++ *(u32 *)((u8 *)toggle_la57 + trampoline_ljmp_imm_offset) +=
++ (unsigned long)toggle_la57;
+
+ /*
+ * The code below prepares page table in trampoline memory.
+@@ -175,10 +169,10 @@ struct paging_config paging_prepare(void
+ * We are not going to use the page table in trampoline memory if we
+ * are already in the desired paging mode.
+ */
+- if (paging_config.l5_required == !!(native_read_cr4() & X86_CR4_LA57))
++ if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
+ goto out;
+
+- if (paging_config.l5_required) {
++ if (l5_required) {
+ /*
+ * For 4- to 5-level paging transition, set up current CR3 as
+ * the first and the only entry in a new top-level page table.
+@@ -201,7 +195,7 @@ struct paging_config paging_prepare(void
+ }
+
+ out:
+- return paging_config;
++ toggle_la57(trampoline_32bit, l5_required);
+ }
+
+ void cleanup_trampoline(void *pgtable)
--- /dev/null
+From 03dda95137d3247564854ad9032c0354273a159d Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:12 +0200
+Subject: x86/decompressor: Merge trampoline cleanup with switching code
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 03dda95137d3247564854ad9032c0354273a159d upstream.
+
+Now that the trampoline setup code and the actual invocation of it are
+all done from the C routine, the trampoline cleanup can be merged into
+it as well, instead of returning to asm just to call another C function.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-16-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 14 ++++----------
+ arch/x86/boot/compressed/pgtable_64.c | 18 ++++--------------
+ 2 files changed, 8 insertions(+), 24 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -433,20 +433,14 @@ SYM_CODE_START(startup_64)
+ * a trampoline in 32-bit addressable memory if the current number does
+ * not match the desired number.
+ *
+- * Pass the boot_params pointer as the first argument.
++ * Pass the boot_params pointer as the first argument. The second
++ * argument is the relocated address of the page table to use instead
++ * of the page table in trampoline memory (if required).
+ */
+ movq %r15, %rdi
++ leaq rva(top_pgtable)(%rbx), %rsi
+ call configure_5level_paging
+
+- /*
+- * cleanup_trampoline() would restore trampoline memory.
+- *
+- * RDI is address of the page table to use instead of page table
+- * in trampoline memory (if required).
+- */
+- leaq rva(top_pgtable)(%rbx), %rdi
+- call cleanup_trampoline
+-
+ /* Zero EFLAGS */
+ pushq $0
+ popfq
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -101,7 +101,7 @@ static unsigned long find_trampoline_pla
+ return bios_start - TRAMPOLINE_32BIT_SIZE;
+ }
+
+-asmlinkage void configure_5level_paging(struct boot_params *bp)
++asmlinkage void configure_5level_paging(struct boot_params *bp, void *pgtable)
+ {
+ void (*toggle_la57)(void *cr3);
+ bool l5_required = false;
+@@ -191,22 +191,12 @@ asmlinkage void configure_5level_paging(
+ }
+
+ toggle_la57(trampoline_32bit);
+-}
+-
+-void cleanup_trampoline(void *pgtable)
+-{
+- void *trampoline_pgtable;
+-
+- trampoline_pgtable = trampoline_32bit;
+
+ /*
+- * Move the top level page table out of trampoline memory,
+- * if it's there.
++ * Move the top level page table out of trampoline memory.
+ */
+- if ((void *)__native_read_cr3() == trampoline_pgtable) {
+- memcpy(pgtable, trampoline_pgtable, PAGE_SIZE);
+- native_write_cr3((unsigned long)pgtable);
+- }
++ memcpy(pgtable, trampoline_32bit, PAGE_SIZE);
++ native_write_cr3((unsigned long)pgtable);
+
+ /* Restore trampoline memory */
+ memcpy(trampoline_32bit, trampoline_save, TRAMPOLINE_32BIT_SIZE);
--- /dev/null
+From 24388292e2d7fae79a0d4183cc91716b851299cf Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:16 +0200
+Subject: x86/decompressor: Move global symbol references to C code
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 24388292e2d7fae79a0d4183cc91716b851299cf upstream.
+
+It is no longer necessary to be cautious when referring to global
+variables in the position independent decompressor code, now that it is
+built using PIE codegen and makes an assertion in the linker script that
+no GOT entries exist (which would require adjustment for the actual
+runtime load address of the decompressor binary).
+
+This means global variables can be referenced directly from C code,
+instead of having to pass their runtime addresses into C routines from
+asm code, which needs to happen at each call site. Do so for the code
+that will be called directly from the EFI stub after a subsequent patch,
+and avoid the need to duplicate this logic a third time.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-20-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_32.S | 8 --------
+ arch/x86/boot/compressed/head_64.S | 10 ++--------
+ arch/x86/boot/compressed/misc.c | 16 +++++++++-------
+ 3 files changed, 11 insertions(+), 23 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_32.S
++++ b/arch/x86/boot/compressed/head_32.S
+@@ -179,13 +179,7 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ */
+ /* push arguments for extract_kernel: */
+
+- pushl output_len@GOTOFF(%ebx) /* decompressed length, end of relocs */
+ pushl %ebp /* output address */
+- pushl input_len@GOTOFF(%ebx) /* input_len */
+- leal input_data@GOTOFF(%ebx), %eax
+- pushl %eax /* input_data */
+- leal boot_heap@GOTOFF(%ebx), %eax
+- pushl %eax /* heap area */
+ pushl %esi /* real mode pointer */
+ call extract_kernel /* returns kernel entry point in %eax */
+ addl $24, %esp
+@@ -213,8 +207,6 @@ SYM_DATA_END_LABEL(gdt, SYM_L_LOCAL, gdt
+ */
+ .bss
+ .balign 4
+-boot_heap:
+- .fill BOOT_HEAP_SIZE, 1, 0
+ boot_stack:
+ .fill BOOT_STACK_SIZE, 1, 0
+ boot_stack_end:
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -511,13 +511,9 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ /*
+ * Do the extraction, and jump to the new kernel..
+ */
+- /* pass struct boot_params pointer */
++ /* pass struct boot_params pointer and output target address */
+ movq %r15, %rdi
+- leaq boot_heap(%rip), %rsi /* malloc area for uncompression */
+- leaq input_data(%rip), %rdx /* input_data */
+- movl input_len(%rip), %ecx /* input_len */
+- movq %rbp, %r8 /* output target address */
+- movl output_len(%rip), %r9d /* decompressed length, end of relocs */
++ movq %rbp, %rsi
+ call extract_kernel /* returns kernel entry point in %rax */
+
+ /*
+@@ -675,8 +671,6 @@ SYM_DATA_END_LABEL(boot_idt, SYM_L_GLOBA
+ */
+ .bss
+ .balign 4
+-SYM_DATA_LOCAL(boot_heap, .fill BOOT_HEAP_SIZE, 1, 0)
+-
+ SYM_DATA_START_LOCAL(boot_stack)
+ .fill BOOT_STACK_SIZE, 1, 0
+ .balign 16
+--- a/arch/x86/boot/compressed/misc.c
++++ b/arch/x86/boot/compressed/misc.c
+@@ -330,6 +330,11 @@ static size_t parse_elf(void *output)
+ return ehdr.e_entry - LOAD_PHYSICAL_ADDR;
+ }
+
++static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4);
++
++extern unsigned char input_data[];
++extern unsigned int input_len, output_len;
++
+ /*
+ * The compressed kernel image (ZO), has been moved so that its position
+ * is against the end of the buffer used to hold the uncompressed kernel
+@@ -347,14 +352,11 @@ static size_t parse_elf(void *output)
+ * |-------uncompressed kernel image---------|
+ *
+ */
+-asmlinkage __visible void *extract_kernel(void *rmode, memptr heap,
+- unsigned char *input_data,
+- unsigned long input_len,
+- unsigned char *output,
+- unsigned long output_len)
++asmlinkage __visible void *extract_kernel(void *rmode, unsigned char *output)
+ {
+ const unsigned long kernel_total_size = VO__end - VO__text;
+ unsigned long virt_addr = LOAD_PHYSICAL_ADDR;
++ memptr heap = (memptr)boot_heap;
+ unsigned long needed_size;
+ size_t entry_offset;
+
+@@ -412,7 +414,7 @@ asmlinkage __visible void *extract_kerne
+ * entries. This ensures the full mapped area is usable RAM
+ * and doesn't include any reserved areas.
+ */
+- needed_size = max(output_len, kernel_total_size);
++ needed_size = max_t(unsigned long, output_len, kernel_total_size);
+ #ifdef CONFIG_X86_64
+ needed_size = ALIGN(needed_size, MIN_KERNEL_ALIGN);
+ #endif
+@@ -443,7 +445,7 @@ asmlinkage __visible void *extract_kerne
+ #ifdef CONFIG_X86_64
+ if (heap > 0x3fffffffffffUL)
+ error("Destination address too large");
+- if (virt_addr + max(output_len, kernel_total_size) > KERNEL_IMAGE_SIZE)
++ if (virt_addr + needed_size > KERNEL_IMAGE_SIZE)
+ error("Destination virtual address is beyond the kernel mapping area");
+ #else
+ if (heap > ((-__PAGE_OFFSET-(128<<20)-1) & 0x7fffffff))
--- /dev/null
+From f97b67a773cd84bd8b55c0a0ec32448a87fc56bb Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:10 +0200
+Subject: x86/decompressor: Only call the trampoline when changing paging levels
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit f97b67a773cd84bd8b55c0a0ec32448a87fc56bb upstream.
+
+Since the current and desired number of paging levels are known when the
+trampoline is being prepared, avoid calling the trampoline at all if it
+is clear that calling it is not going to result in a change to the
+number of paging levels.
+
+Given that the CPU is already running in long mode, the PAE and LA57
+settings are necessarily consistent with the currently active page
+tables, and other fields in CR4 will be initialized by the startup code
+in the kernel proper. So limit the manipulation of CR4 to toggling the
+LA57 bit, which is the only thing that really needs doing at this point
+in the boot. This also means that there is no need to pass the value of
+l5_required to toggle_la57(), as it will not be called unless CR4.LA57
+needs to toggle.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-14-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 45 +++-------------------------------
+ arch/x86/boot/compressed/pgtable_64.c | 22 ++++++----------
+ 2 files changed, 13 insertions(+), 54 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -390,10 +390,6 @@ SYM_CODE_START(startup_64)
+ * For the trampoline, we need the top page table to reside in lower
+ * memory as we don't have a way to load 64-bit values into CR3 in
+ * 32-bit mode.
+- *
+- * We go though the trampoline even if we don't have to: if we're
+- * already in a desired paging mode. This way the trampoline code gets
+- * tested on every boot.
+ */
+
+ /* Make sure we have GDT with 32-bit code segment */
+@@ -544,8 +540,7 @@ SYM_FUNC_END(.Lrelocated)
+ *
+ * Return address is at the top of the stack (might be above 4G).
+ * The first argument (EDI) contains the 32-bit addressable base of the
+- * trampoline memory. A non-zero second argument (ESI) means that the
+- * trampoline needs to enable 5-level paging.
++ * trampoline memory.
+ */
+ .section ".rodata", "a", @progbits
+ SYM_CODE_START(trampoline_32bit_src)
+@@ -597,25 +592,10 @@ SYM_CODE_START(trampoline_32bit_src)
+ btrl $X86_CR0_PG_BIT, %eax
+ movl %eax, %cr0
+
+- /* Check what paging mode we want to be in after the trampoline */
+- testl %esi, %esi
+- jz 1f
+-
+- /* We want 5-level paging: don't touch CR3 if it already points to 5-level page tables */
+- movl %cr4, %eax
+- testl $X86_CR4_LA57, %eax
+- jnz 3f
+- jmp 2f
+-1:
+- /* We want 4-level paging: don't touch CR3 if it already points to 4-level page tables */
+- movl %cr4, %eax
+- testl $X86_CR4_LA57, %eax
+- jz 3f
+-2:
+ /* Point CR3 to the trampoline's new top level page table */
+ leal TRAMPOLINE_32BIT_PGTABLE_OFFSET(%edi), %eax
+ movl %eax, %cr3
+-3:
++
+ /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
+ movl $MSR_EFER, %ecx
+ rdmsr
+@@ -624,26 +604,9 @@ SYM_CODE_START(trampoline_32bit_src)
+ jc 1f
+ wrmsr
+ 1:
+-#ifdef CONFIG_X86_MCE
+- /*
+- * Preserve CR4.MCE if the kernel will enable #MC support.
+- * Clearing MCE may fault in some environments (that also force #MC
+- * support). Any machine check that occurs before #MC support is fully
+- * configured will crash the system regardless of the CR4.MCE value set
+- * here.
+- */
++ /* Toggle CR4.LA57 */
+ movl %cr4, %eax
+- andl $X86_CR4_MCE, %eax
+-#else
+- movl $0, %eax
+-#endif
+-
+- /* Enable PAE and LA57 (if required) paging modes */
+- orl $X86_CR4_PAE, %eax
+- testl %esi, %esi
+- jz 1f
+- orl $X86_CR4_LA57, %eax
+-1:
++ btcl $X86_CR4_LA57_BIT, %eax
+ movl %eax, %cr4
+
+ /* Enable paging again. */
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -103,7 +103,7 @@ static unsigned long find_trampoline_pla
+
+ asmlinkage void configure_5level_paging(struct boot_params *bp)
+ {
+- void (*toggle_la57)(void *trampoline, bool enable_5lvl);
++ void (*toggle_la57)(void *trampoline);
+ bool l5_required = false;
+
+ /* Initialize boot_params. Required for cmdline_find_option_bool(). */
+@@ -133,6 +133,13 @@ asmlinkage void configure_5level_paging(
+ ptrs_per_p4d = 512;
+ }
+
++ /*
++ * The trampoline will not be used if the paging mode is already set to
++ * the desired one.
++ */
++ if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
++ return;
++
+ trampoline_32bit = (unsigned long *)find_trampoline_placement();
+
+ /* Preserve trampoline memory */
+@@ -160,18 +167,8 @@ asmlinkage void configure_5level_paging(
+ *
+ * The new page table will be used by trampoline code for switching
+ * from 4- to 5-level paging or vice versa.
+- *
+- * If switching is not required, the page table is unused: trampoline
+- * code wouldn't touch CR3.
+ */
+
+- /*
+- * We are not going to use the page table in trampoline memory if we
+- * are already in the desired paging mode.
+- */
+- if (l5_required == !!(native_read_cr4() & X86_CR4_LA57))
+- goto out;
+-
+ if (l5_required) {
+ /*
+ * For 4- to 5-level paging transition, set up current CR3 as
+@@ -194,8 +191,7 @@ asmlinkage void configure_5level_paging(
+ (void *)src, PAGE_SIZE);
+ }
+
+-out:
+- toggle_la57(trampoline_32bit, l5_required);
++ toggle_la57(trampoline_32bit);
+ }
+
+ void cleanup_trampoline(void *pgtable)
--- /dev/null
+From cb83cece57e1889109dd73ea08ee338668c9d1b8 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:11 +0200
+Subject: x86/decompressor: Pass pgtable address to trampoline directly
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit cb83cece57e1889109dd73ea08ee338668c9d1b8 upstream.
+
+The only remaining use of the trampoline address by the trampoline
+itself is deriving the page table address from it, and this involves
+adding an offset of 0x0. So simplify this, and pass the new CR3 value
+directly.
+
+This makes the fact that the page table happens to be at the start of
+the trampoline allocation an implementation detail of the caller.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-15-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 8 ++++----
+ arch/x86/boot/compressed/pgtable.h | 2 --
+ arch/x86/boot/compressed/pgtable_64.c | 9 ++++-----
+ 3 files changed, 8 insertions(+), 11 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -539,8 +539,9 @@ SYM_FUNC_END(.Lrelocated)
+ * running in 64-bit mode.
+ *
+ * Return address is at the top of the stack (might be above 4G).
+- * The first argument (EDI) contains the 32-bit addressable base of the
+- * trampoline memory.
++ * The first argument (EDI) contains the address of the temporary PGD level
++ * page table in 32-bit addressable memory which will be programmed into
++ * register CR3.
+ */
+ .section ".rodata", "a", @progbits
+ SYM_CODE_START(trampoline_32bit_src)
+@@ -593,8 +594,7 @@ SYM_CODE_START(trampoline_32bit_src)
+ movl %eax, %cr0
+
+ /* Point CR3 to the trampoline's new top level page table */
+- leal TRAMPOLINE_32BIT_PGTABLE_OFFSET(%edi), %eax
+- movl %eax, %cr3
++ movl %edi, %cr3
+
+ /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
+ movl $MSR_EFER, %ecx
+--- a/arch/x86/boot/compressed/pgtable.h
++++ b/arch/x86/boot/compressed/pgtable.h
+@@ -3,8 +3,6 @@
+
+ #define TRAMPOLINE_32BIT_SIZE (2 * PAGE_SIZE)
+
+-#define TRAMPOLINE_32BIT_PGTABLE_OFFSET 0
+-
+ #define TRAMPOLINE_32BIT_CODE_OFFSET PAGE_SIZE
+ #define TRAMPOLINE_32BIT_CODE_SIZE 0xA0
+
+--- a/arch/x86/boot/compressed/pgtable_64.c
++++ b/arch/x86/boot/compressed/pgtable_64.c
+@@ -103,7 +103,7 @@ static unsigned long find_trampoline_pla
+
+ asmlinkage void configure_5level_paging(struct boot_params *bp)
+ {
+- void (*toggle_la57)(void *trampoline);
++ void (*toggle_la57)(void *cr3);
+ bool l5_required = false;
+
+ /* Initialize boot_params. Required for cmdline_find_option_bool(). */
+@@ -174,7 +174,7 @@ asmlinkage void configure_5level_paging(
+ * For 4- to 5-level paging transition, set up current CR3 as
+ * the first and the only entry in a new top-level page table.
+ */
+- trampoline_32bit[TRAMPOLINE_32BIT_PGTABLE_OFFSET] = __native_read_cr3() | _PAGE_TABLE_NOENC;
++ *trampoline_32bit = __native_read_cr3() | _PAGE_TABLE_NOENC;
+ } else {
+ unsigned long src;
+
+@@ -187,8 +187,7 @@ asmlinkage void configure_5level_paging(
+ * may be above 4G.
+ */
+ src = *(unsigned long *)__native_read_cr3() & PAGE_MASK;
+- memcpy(trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long),
+- (void *)src, PAGE_SIZE);
++ memcpy(trampoline_32bit, (void *)src, PAGE_SIZE);
+ }
+
+ toggle_la57(trampoline_32bit);
+@@ -198,7 +197,7 @@ void cleanup_trampoline(void *pgtable)
+ {
+ void *trampoline_pgtable;
+
+- trampoline_pgtable = trampoline_32bit + TRAMPOLINE_32BIT_PGTABLE_OFFSET / sizeof(unsigned long);
++ trampoline_pgtable = trampoline_32bit;
+
+ /*
+ * Move the top level page table out of trampoline memory,
--- /dev/null
+From 8b63cba746f86a754d66e302c43209cc9b9b6e39 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:04 +0200
+Subject: x86/decompressor: Store boot_params pointer in callee save register
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 8b63cba746f86a754d66e302c43209cc9b9b6e39 upstream.
+
+Instead of pushing and popping %RSI several times to preserve the struct
+boot_params pointer across the execution of the startup code, move it
+into a callee save register before the first call into C, and copy it
+back when needed.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-8-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 42 ++++++++++++++-----------------------
+ 1 file changed, 16 insertions(+), 26 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -408,10 +408,14 @@ SYM_CODE_START(startup_64)
+ lretq
+
+ .Lon_kernel_cs:
++ /*
++ * RSI holds a pointer to a boot_params structure provided by the
++ * loader, and this needs to be preserved across C function calls. So
++ * move it into a callee saved register.
++ */
++ movq %rsi, %r15
+
+- pushq %rsi
+ call load_stage1_idt
+- popq %rsi
+
+ #ifdef CONFIG_AMD_MEM_ENCRYPT
+ /*
+@@ -422,12 +426,10 @@ SYM_CODE_START(startup_64)
+ * CPUID instructions being issued, so go ahead and do that now via
+ * sev_enable(), which will also handle the rest of the SEV-related
+ * detection/setup to ensure that has been done in advance of any dependent
+- * code.
++ * code. Pass the boot_params pointer as the first argument.
+ */
+- pushq %rsi
+- movq %rsi, %rdi /* real mode address */
++ movq %r15, %rdi
+ call sev_enable
+- popq %rsi
+ #endif
+
+ /*
+@@ -440,13 +442,10 @@ SYM_CODE_START(startup_64)
+ * - Non zero RDX means trampoline needs to enable 5-level
+ * paging.
+ *
+- * RSI holds real mode data and needs to be preserved across
+- * this function call.
++ * Pass the boot_params pointer as the first argument.
+ */
+- pushq %rsi
+- movq %rsi, %rdi /* real mode address */
++ movq %r15, %rdi
+ call paging_prepare
+- popq %rsi
+
+ /* Save the trampoline address in RCX */
+ movq %rax, %rcx
+@@ -459,9 +458,9 @@ SYM_CODE_START(startup_64)
+ * because the architecture does not guarantee that GPRs will retain
+ * their full 64-bit values across a 32-bit mode switch.
+ */
++ pushq %r15
+ pushq %rbp
+ pushq %rbx
+- pushq %rsi
+
+ /*
+ * Push the 64-bit address of trampoline_return() onto the new stack.
+@@ -478,9 +477,9 @@ SYM_CODE_START(startup_64)
+ lretq
+ trampoline_return:
+ /* Restore live 64-bit registers */
+- popq %rsi
+ popq %rbx
+ popq %rbp
++ popq %r15
+
+ /* Restore the stack, the 32-bit trampoline uses its own stack */
+ leaq rva(boot_stack_end)(%rbx), %rsp
+@@ -490,14 +489,9 @@ trampoline_return:
+ *
+ * RDI is address of the page table to use instead of page table
+ * in trampoline memory (if required).
+- *
+- * RSI holds real mode data and needs to be preserved across
+- * this function call.
+ */
+- pushq %rsi
+ leaq rva(top_pgtable)(%rbx), %rdi
+ call cleanup_trampoline
+- popq %rsi
+
+ /* Zero EFLAGS */
+ pushq $0
+@@ -507,7 +501,6 @@ trampoline_return:
+ * Copy the compressed kernel to the end of our buffer
+ * where decompression in place becomes safe.
+ */
+- pushq %rsi
+ leaq (_bss-8)(%rip), %rsi
+ leaq rva(_bss-8)(%rbx), %rdi
+ movl $(_bss - startup_32), %ecx
+@@ -515,7 +508,6 @@ trampoline_return:
+ std
+ rep movsq
+ cld
+- popq %rsi
+
+ /*
+ * The GDT may get overwritten either during the copy we just did or
+@@ -562,30 +554,28 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ shrq $3, %rcx
+ rep stosq
+
+- pushq %rsi
+ call load_stage2_idt
+
+ /* Pass boot_params to initialize_identity_maps() */
+- movq (%rsp), %rdi
++ movq %r15, %rdi
+ call initialize_identity_maps
+- popq %rsi
+
+ /*
+ * Do the extraction, and jump to the new kernel..
+ */
+- pushq %rsi /* Save the real mode argument */
+- movq %rsi, %rdi /* real mode address */
++ /* pass struct boot_params pointer */
++ movq %r15, %rdi
+ leaq boot_heap(%rip), %rsi /* malloc area for uncompression */
+ leaq input_data(%rip), %rdx /* input_data */
+ movl input_len(%rip), %ecx /* input_len */
+ movq %rbp, %r8 /* output target address */
+ movl output_len(%rip), %r9d /* decompressed length, end of relocs */
+ call extract_kernel /* returns kernel entry point in %rax */
+- popq %rsi
+
+ /*
+ * Jump to the decompressed kernel.
+ */
++ movq %r15, %rsi
+ jmp *%rax
+ SYM_FUNC_END(.Lrelocated)
+
--- /dev/null
+From 918a7a04e71745e99a0efc6753e587439b794b29 Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:07 +0200
+Subject: x86/decompressor: Use standard calling convention for trampoline
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit 918a7a04e71745e99a0efc6753e587439b794b29 upstream.
+
+Update the trampoline code so its arguments are passed via RDI and RSI,
+which matches the ordinary SysV calling convention for x86_64. This will
+allow this code to be called directly from C.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
+Link: https://lore.kernel.org/r/20230807162720.545787-11-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/boot/compressed/head_64.S | 27 +++++++++++++--------------
+ arch/x86/boot/compressed/pgtable.h | 2 +-
+ 2 files changed, 14 insertions(+), 15 deletions(-)
+
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -447,9 +447,9 @@ SYM_CODE_START(startup_64)
+ movq %r15, %rdi
+ call paging_prepare
+
+- /* Save the trampoline address in RCX */
+- movq %rax, %rcx
+-
++ /* Pass the trampoline address and boolean flag as args #1 and #2 */
++ movq %rax, %rdi
++ movq %rdx, %rsi
+ leaq TRAMPOLINE_32BIT_CODE_OFFSET(%rax), %rax
+ call *%rax
+
+@@ -549,11 +549,14 @@ SYM_FUNC_START_LOCAL_NOALIGN(.Lrelocated
+ SYM_FUNC_END(.Lrelocated)
+
+ /*
+- * This is the 32-bit trampoline that will be copied over to low memory.
++ * This is the 32-bit trampoline that will be copied over to low memory. It
++ * will be called using the ordinary 64-bit calling convention from code
++ * running in 64-bit mode.
+ *
+ * Return address is at the top of the stack (might be above 4G).
+- * ECX contains the base address of the trampoline memory.
+- * Non zero RDX means trampoline needs to enable 5-level paging.
++ * The first argument (EDI) contains the 32-bit addressable base of the
++ * trampoline memory. A non-zero second argument (ESI) means that the
++ * trampoline needs to enable 5-level paging.
+ */
+ SYM_CODE_START(trampoline_32bit_src)
+ /*
+@@ -600,7 +603,7 @@ SYM_CODE_START(trampoline_32bit_src)
+ movl %eax, %cr0
+
+ /* Check what paging mode we want to be in after the trampoline */
+- testl %edx, %edx
++ testl %esi, %esi
+ jz 1f
+
+ /* We want 5-level paging: don't touch CR3 if it already points to 5-level page tables */
+@@ -615,21 +618,17 @@ SYM_CODE_START(trampoline_32bit_src)
+ jz 3f
+ 2:
+ /* Point CR3 to the trampoline's new top level page table */
+- leal TRAMPOLINE_32BIT_PGTABLE_OFFSET(%ecx), %eax
++ leal TRAMPOLINE_32BIT_PGTABLE_OFFSET(%edi), %eax
+ movl %eax, %cr3
+ 3:
+ /* Set EFER.LME=1 as a precaution in case hypervsior pulls the rug */
+- pushl %ecx
+- pushl %edx
+ movl $MSR_EFER, %ecx
+ rdmsr
+ btsl $_EFER_LME, %eax
+ /* Avoid writing EFER if no change was made (for TDX guest) */
+ jc 1f
+ wrmsr
+-1: popl %edx
+- popl %ecx
+-
++1:
+ #ifdef CONFIG_X86_MCE
+ /*
+ * Preserve CR4.MCE if the kernel will enable #MC support.
+@@ -646,7 +645,7 @@ SYM_CODE_START(trampoline_32bit_src)
+
+ /* Enable PAE and LA57 (if required) paging modes */
+ orl $X86_CR4_PAE, %eax
+- testl %edx, %edx
++ testl %esi, %esi
+ jz 1f
+ orl $X86_CR4_LA57, %eax
+ 1:
+--- a/arch/x86/boot/compressed/pgtable.h
++++ b/arch/x86/boot/compressed/pgtable.h
+@@ -14,7 +14,7 @@
+
+ extern unsigned long *trampoline_32bit;
+
+-extern void trampoline_32bit_src(void *return_ptr);
++extern void trampoline_32bit_src(void *trampoline, bool enable_5lvl);
+
+ #endif /* __ASSEMBLER__ */
+ #endif /* BOOT_COMPRESSED_PAGETABLE_H */
--- /dev/null
+From cc3fdda2876e58a7e83e558ab51853cf106afb6a Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Tue, 22 Nov 2022 17:10:17 +0100
+Subject: x86/efi: Make the deprecated EFI handover protocol optional
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit cc3fdda2876e58a7e83e558ab51853cf106afb6a upstream.
+
+The EFI handover protocol permits a bootloader to invoke the kernel as a
+EFI PE/COFF application, while passing a bootparams struct as a third
+argument to the entrypoint function call.
+
+This has no basis in the UEFI specification, and there are better ways
+to pass additional data to a UEFI application (UEFI configuration
+tables, UEFI variables, UEFI protocols) than going around the
+StartImage() boot service and jumping to a fixed offset in the loaded
+image, just to call a different function that takes a third parameter.
+
+The reason for handling struct bootparams in the bootloader was that the
+EFI stub could only load initrd images from the EFI system partition,
+and so passing it via struct bootparams was needed for loaders like
+GRUB, which pass the initrd in memory, and may load it from anywhere,
+including from the network. Another motivation was EFI mixed mode, which
+could not use the initrd loader in the EFI stub at all due to 32/64 bit
+incompatibilities (which will be fixed shortly [0]), and could not
+invoke the ordinary PE/COFF entry point either, for the same reasons.
+
+Given that loaders such as GRUB already carried the bootparams handling
+in order to implement non-EFI boot, retaining that code and just passing
+bootparams to the EFI stub was a reasonable choice (although defining an
+alternate entrypoint could have been avoided.) However, the GRUB side
+changes never made it upstream, and are only shipped by some of the
+distros in their downstream versions.
+
+In the meantime, EFI support has been added to other Linux architecture
+ports, as well as to U-boot and systemd, including arch-agnostic methods
+for passing initrd images in memory [1], and for doing mixed mode boot
+[2], none of them requiring anything like the EFI handover protocol. So
+given that only out-of-tree distro GRUB relies on this, let's permit it
+to be omitted from the build, in preparation for retiring it completely
+at a later date. (Note that systemd-boot does have an implementation as
+well, but only uses it as a fallback for booting images that do not
+implement the LoadFile2 based initrd loading method, i.e., v5.8 or older)
+
+[0] https://lore.kernel.org/all/20220927085842.2860715-1-ardb@kernel.org/
+[1] ec93fc371f01 ("efi/libstub: Add support for loading the initrd from a device path")
+[2] 97aa276579b2 ("efi/x86: Add true mixed mode entry point into .compat section")
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov <bp@suse.de>
+Link: https://lore.kernel.org/r/20221122161017.2426828-18-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ arch/x86/Kconfig | 17 +++++++++++++++++
+ arch/x86/boot/compressed/head_64.S | 4 +++-
+ arch/x86/boot/header.S | 2 +-
+ arch/x86/boot/tools/build.c | 2 ++
+ 4 files changed, 23 insertions(+), 2 deletions(-)
+
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@ -1982,6 +1982,23 @@ config EFI_STUB
+
+ See Documentation/admin-guide/efi-stub.rst for more information.
+
++config EFI_HANDOVER_PROTOCOL
++ bool "EFI handover protocol (DEPRECATED)"
++ depends on EFI_STUB
++ default y
++ help
++ Select this in order to include support for the deprecated EFI
++ handover protocol, which defines alternative entry points into the
++ EFI stub. This is a practice that has no basis in the UEFI
++ specification, and requires a priori knowledge on the part of the
++ bootloader about Linux/x86 specific ways of passing the command line
++ and initrd, and where in memory those assets may be loaded.
++
++ If in doubt, say Y. Even though the corresponding support is not
++ present in upstream GRUB or other bootloaders, most distros build
++ GRUB with numerous downstream patches applied, and may rely on the
++ handover protocol as as result.
++
+ config EFI_MIXED
+ bool "EFI mixed-mode support"
+ depends on EFI_STUB && X86_64
+--- a/arch/x86/boot/compressed/head_64.S
++++ b/arch/x86/boot/compressed/head_64.S
+@@ -286,7 +286,7 @@ SYM_FUNC_START(startup_32)
+ lret
+ SYM_FUNC_END(startup_32)
+
+-#ifdef CONFIG_EFI_MIXED
++#if IS_ENABLED(CONFIG_EFI_MIXED) && IS_ENABLED(CONFIG_EFI_HANDOVER_PROTOCOL)
+ .org 0x190
+ SYM_FUNC_START(efi32_stub_entry)
+ add $0x4, %esp /* Discard return address */
+@@ -535,7 +535,9 @@ trampoline_return:
+ SYM_CODE_END(startup_64)
+
+ #ifdef CONFIG_EFI_STUB
++#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+ .org 0x390
++#endif
+ SYM_FUNC_START(efi64_stub_entry)
+ and $~0xf, %rsp /* realign the stack */
+ movq %rdx, %rbx /* save boot_params pointer */
+--- a/arch/x86/boot/header.S
++++ b/arch/x86/boot/header.S
+@@ -406,7 +406,7 @@ xloadflags:
+ # define XLF1 0
+ #endif
+
+-#ifdef CONFIG_EFI_STUB
++#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+ # ifdef CONFIG_EFI_MIXED
+ # define XLF23 (XLF_EFI_HANDOVER_32|XLF_EFI_HANDOVER_64)
+ # else
+--- a/arch/x86/boot/tools/build.c
++++ b/arch/x86/boot/tools/build.c
+@@ -290,6 +290,7 @@ static void efi_stub_entry_update(void)
+ {
+ unsigned long addr = efi32_stub_entry;
+
++#ifdef CONFIG_EFI_HANDOVER_PROTOCOL
+ #ifdef CONFIG_X86_64
+ /* Yes, this is really how we defined it :( */
+ addr = efi64_stub_entry - 0x200;
+@@ -299,6 +300,7 @@ static void efi_stub_entry_update(void)
+ if (efi32_stub_entry != addr)
+ die("32-bit and 64-bit EFI entry points do not match\n");
+ #endif
++#endif
+ put_unaligned_le32(addr, &buf[0x264]);
+ }
+
--- /dev/null
+From d2d7a54f69b67cd0a30e0ebb5307cb2de625baac Mon Sep 17 00:00:00 2001
+From: Ard Biesheuvel <ardb@kernel.org>
+Date: Mon, 7 Aug 2023 18:27:00 +0200
+Subject: x86/efistub: Branch straight to kernel entry point from C code
+
+From: Ard Biesheuvel <ardb@kernel.org>
+
+commit d2d7a54f69b67cd0a30e0ebb5307cb2de625baac upstream.
+
+Instead of returning to the calling code in assembler that does nothing
+more than perform an indirect call with the boot_params pointer in
+register ESI/RSI, perform the jump directly from the EFI stub C code.
+This will allow the asm entrypoint code to be dropped entirely in
+subsequent patches.
+
+Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
+Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
+Link: https://lore.kernel.org/r/20230807162720.545787-4-ardb@kernel.org
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+---
+ drivers/firmware/efi/libstub/x86-stub.c | 22 +++++++++++++++++-----
+ 1 file changed, 17 insertions(+), 5 deletions(-)
+
+--- a/drivers/firmware/efi/libstub/x86-stub.c
++++ b/drivers/firmware/efi/libstub/x86-stub.c
+@@ -279,7 +279,7 @@ adjust_memory_range_protection(unsigned
+ #define TRAMPOLINE_PLACEMENT_BASE ((128 - 8)*1024)
+ #define TRAMPOLINE_PLACEMENT_SIZE (640*1024 - (128 - 8)*1024)
+
+-void startup_32(struct boot_params *boot_params);
++extern const u8 startup_32[], startup_64[];
+
+ static void
+ setup_memory_protection(unsigned long image_base, unsigned long image_size)
+@@ -760,10 +760,19 @@ static efi_status_t exit_boot(struct boo
+ return EFI_SUCCESS;
+ }
+
++static void __noreturn enter_kernel(unsigned long kernel_addr,
++ struct boot_params *boot_params)
++{
++ /* enter decompressed kernel with boot_params pointer in RSI/ESI */
++ asm("jmp *%0"::"r"(kernel_addr), "S"(boot_params));
++
++ unreachable();
++}
++
+ /*
+- * On success, we return the address of startup_32, which has potentially been
+- * relocated by efi_relocate_kernel.
+- * On failure, we exit to the firmware via efi_exit instead of returning.
++ * On success, this routine will jump to the relocated image directly and never
++ * return. On failure, it will exit to the firmware via efi_exit() instead of
++ * returning.
+ */
+ asmlinkage unsigned long efi_main(efi_handle_t handle,
+ efi_system_table_t *sys_table_arg,
+@@ -905,7 +914,10 @@ asmlinkage unsigned long efi_main(efi_ha
+ goto fail;
+ }
+
+- return bzimage_addr;
++ if (IS_ENABLED(CONFIG_X86_64))
++ bzimage_addr += startup_64 - startup_32;
++
++ enter_kernel(bzimage_addr, boot_params);
+ fail:
+ efi_err("efi_main() failed!\n");
+
projects / thirdparty / kernel / stable-queue.git / commitdiff
