From: Greg Kroah-Hartman Date: Mon, 8 Apr 2024 12:37:36 +0000 (+0200) Subject: 6.6-stable patches X-Git-Tag: v5.15.154~17 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=acd093817546e56a8778e6cdb8d32f8286611be3;p=thirdparty%2Fkernel%2Fstable-queue.git 6.6-stable patches added patches: x86-efistub-remap-kernel-text-read-only-before-dropping-nx-attribute.patch x86-sev-move-early-startup-code-into-.head.text-section.patch x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch --- diff --git a/queue-6.6/series b/queue-6.6/series index 3074034295c..5c8b4d3b963 100644 --- a/queue-6.6/series +++ b/queue-6.6/series @@ -247,4 +247,6 @@ bpf-put-uprobe-link-s-path-and-task-in-release-callback.patch bpf-support-deferring-bpf_link-dealloc-to-after-rcu-grace-period.patch x86-head-64-move-the-__head-definition-to-asm-init.h.patch efi-libstub-add-generic-support-for-parsing-mem_encrypt.patch -x86-boot-move-mem_encrypt-parsing-to-the-decompressor.patch +x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch +x86-sev-move-early-startup-code-into-.head.text-section.patch +x86-efistub-remap-kernel-text-read-only-before-dropping-nx-attribute.patch diff --git a/queue-6.6/x86-boot-move-mem_encrypt-parsing-to-the-decompressor.patch b/queue-6.6/x86-boot-move-mem_encrypt-parsing-to-the-decompressor.patch deleted file mode 100644 index 1e4bca312c7..00000000000 --- a/queue-6.6/x86-boot-move-mem_encrypt-parsing-to-the-decompressor.patch +++ /dev/null @@ -1,195 +0,0 @@ -From cd0d9d92c8bb46e77de62efd7df13069ddd61e7d Mon Sep 17 00:00:00 2001 -From: Ard Biesheuvel -Date: Tue, 27 Feb 2024 16:19:14 +0100 -Subject: x86/boot: Move mem_encrypt= parsing to the decompressor - -From: Ard Biesheuvel - -commit cd0d9d92c8bb46e77de62efd7df13069ddd61e7d upstream. - -The early SME/SEV code parses the command line very early, in order to -decide whether or not memory encryption should be enabled, which needs -to occur even before the initial page tables are created. - -This is problematic for a number of reasons: -- this early code runs from the 1:1 mapping provided by the decompressor - or firmware, which uses a different translation than the one assumed by - the linker, and so the code needs to be built in a special way; -- parsing external input while the entire kernel image is still mapped - writable is a bad idea in general, and really does not belong in - security minded code; -- the current code ignores the built-in command line entirely (although - this appears to be the case for the entire decompressor) - -Given that the decompressor/EFI stub is an intrinsic part of the x86 -bootable kernel image, move the command line parsing there and out of -the core kernel. This removes the need to build lib/cmdline.o in a -special way, or to use RIP-relative LEA instructions in inline asm -blocks. - -This involves a new xloadflag in the setup header to indicate -that mem_encrypt=on appeared on the kernel command line. - -Signed-off-by: Ard Biesheuvel -Signed-off-by: Borislav Petkov (AMD) -Tested-by: Tom Lendacky -Link: https://lore.kernel.org/r/20240227151907.387873-17-ardb+git@google.com -Signed-off-by: Ard Biesheuvel -Signed-off-by: Greg Kroah-Hartman ---- - arch/x86/boot/compressed/misc.c | 15 +++++++++++++++ - arch/x86/include/uapi/asm/bootparam.h | 1 + - arch/x86/lib/Makefile | 13 ------------- - arch/x86/mm/mem_encrypt_identity.c | 32 +++----------------------------- - drivers/firmware/efi/libstub/x86-stub.c | 3 +++ - 5 files changed, 22 insertions(+), 42 deletions(-) - ---- a/arch/x86/boot/compressed/misc.c -+++ b/arch/x86/boot/compressed/misc.c -@@ -358,6 +358,19 @@ unsigned long decompress_kernel(unsigned - } - - /* -+ * Set the memory encryption xloadflag based on the mem_encrypt= command line -+ * parameter, if provided. -+ */ -+static void parse_mem_encrypt(struct setup_header *hdr) -+{ -+ int on = cmdline_find_option_bool("mem_encrypt=on"); -+ int off = cmdline_find_option_bool("mem_encrypt=off"); -+ -+ if (on > off) -+ hdr->xloadflags |= XLF_MEM_ENCRYPTION; -+} -+ -+/* - * The compressed kernel image (ZO), has been moved so that its position - * is against the end of the buffer used to hold the uncompressed kernel - * image (VO) and the execution environment (.bss, .brk), which makes sure -@@ -387,6 +400,8 @@ asmlinkage __visible void *extract_kerne - /* Clear flags intended for solely in-kernel use. */ - boot_params->hdr.loadflags &= ~KASLR_FLAG; - -+ parse_mem_encrypt(&boot_params_ptr->hdr); -+ - sanitize_boot_params(boot_params); - - if (boot_params->screen_info.orig_video_mode == 7) { ---- a/arch/x86/include/uapi/asm/bootparam.h -+++ b/arch/x86/include/uapi/asm/bootparam.h -@@ -38,6 +38,7 @@ - #define XLF_EFI_KEXEC (1<<4) - #define XLF_5LEVEL (1<<5) - #define XLF_5LEVEL_ENABLED (1<<6) -+#define XLF_MEM_ENCRYPTION (1<<7) - - #ifndef __ASSEMBLY__ - ---- a/arch/x86/lib/Makefile -+++ b/arch/x86/lib/Makefile -@@ -14,19 +14,6 @@ ifdef CONFIG_KCSAN - CFLAGS_REMOVE_delay.o = $(CC_FLAGS_FTRACE) - endif - --# Early boot use of cmdline; don't instrument it --ifdef CONFIG_AMD_MEM_ENCRYPT --KCOV_INSTRUMENT_cmdline.o := n --KASAN_SANITIZE_cmdline.o := n --KCSAN_SANITIZE_cmdline.o := n -- --ifdef CONFIG_FUNCTION_TRACER --CFLAGS_REMOVE_cmdline.o = -pg --endif -- --CFLAGS_cmdline.o := -fno-stack-protector -fno-jump-tables --endif -- - inat_tables_script = $(srctree)/arch/x86/tools/gen-insn-attr-x86.awk - inat_tables_maps = $(srctree)/arch/x86/lib/x86-opcode-map.txt - quiet_cmd_inat_tables = GEN $@ ---- a/arch/x86/mm/mem_encrypt_identity.c -+++ b/arch/x86/mm/mem_encrypt_identity.c -@@ -43,7 +43,6 @@ - - #include - #include --#include - #include - #include - -@@ -95,9 +94,6 @@ struct sme_populate_pgd_data { - */ - static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch"); - --static char sme_cmdline_arg[] __initdata = "mem_encrypt"; --static char sme_cmdline_on[] __initdata = "on"; -- - static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd) - { - unsigned long pgd_start, pgd_end, pgd_size; -@@ -504,11 +500,9 @@ void __init sme_encrypt_kernel(struct bo - - void __init sme_enable(struct boot_params *bp) - { -- const char *cmdline_ptr, *cmdline_arg, *cmdline_on; - unsigned int eax, ebx, ecx, edx; - unsigned long feature_mask; - unsigned long me_mask; -- char buffer[16]; - bool snp; - u64 msr; - -@@ -551,6 +545,9 @@ void __init sme_enable(struct boot_param - - /* Check if memory encryption is enabled */ - if (feature_mask == AMD_SME_BIT) { -+ if (!(bp->hdr.xloadflags & XLF_MEM_ENCRYPTION)) -+ return; -+ - /* - * No SME if Hypervisor bit is set. This check is here to - * prevent a guest from trying to enable SME. For running as a -@@ -570,31 +567,8 @@ void __init sme_enable(struct boot_param - msr = __rdmsr(MSR_AMD64_SYSCFG); - if (!(msr & MSR_AMD64_SYSCFG_MEM_ENCRYPT)) - return; -- } else { -- /* SEV state cannot be controlled by a command line option */ -- goto out; - } - -- /* -- * Fixups have not been applied to phys_base yet and we're running -- * identity mapped, so we must obtain the address to the SME command -- * line argument data using rip-relative addressing. -- */ -- asm ("lea sme_cmdline_arg(%%rip), %0" -- : "=r" (cmdline_arg) -- : "p" (sme_cmdline_arg)); -- asm ("lea sme_cmdline_on(%%rip), %0" -- : "=r" (cmdline_on) -- : "p" (sme_cmdline_on)); -- -- cmdline_ptr = (const char *)((u64)bp->hdr.cmd_line_ptr | -- ((u64)bp->ext_cmd_line_ptr << 32)); -- -- if (cmdline_find_option(cmdline_ptr, cmdline_arg, buffer, sizeof(buffer)) < 0 || -- strncmp(buffer, cmdline_on, sizeof(buffer))) -- return; -- --out: - RIP_REL_REF(sme_me_mask) = me_mask; - physical_mask &= ~me_mask; - cc_vendor = CC_VENDOR_AMD; ---- a/drivers/firmware/efi/libstub/x86-stub.c -+++ b/drivers/firmware/efi/libstub/x86-stub.c -@@ -888,6 +888,9 @@ void __noreturn efi_stub_entry(efi_handl - } - } - -+ if (efi_mem_encrypt > 0) -+ hdr->xloadflags |= XLF_MEM_ENCRYPTION; -+ - status = efi_decompress_kernel(&kernel_entry); - if (status != EFI_SUCCESS) { - efi_err("Failed to decompress kernel\n"); diff --git a/queue-6.6/x86-efistub-remap-kernel-text-read-only-before-dropping-nx-attribute.patch b/queue-6.6/x86-efistub-remap-kernel-text-read-only-before-dropping-nx-attribute.patch new file mode 100644 index 00000000000..f8e236164d6 --- /dev/null +++ b/queue-6.6/x86-efistub-remap-kernel-text-read-only-before-dropping-nx-attribute.patch @@ -0,0 +1,93 @@ +From 9c55461040a9264b7e44444c53d26480b438eda6 Mon Sep 17 00:00:00 2001 +From: Ard Biesheuvel +Date: Thu, 25 Jan 2024 14:32:07 +0100 +Subject: x86/efistub: Remap kernel text read-only before dropping NX attribute + +From: Ard Biesheuvel + +commit 9c55461040a9264b7e44444c53d26480b438eda6 upstream. + +Currently, the EFI stub invokes the EFI memory attributes protocol to +strip any NX restrictions from the entire loaded kernel, resulting in +all code and data being mapped read-write-execute. + +The point of the EFI memory attributes protocol is to remove the need +for all memory allocations to be mapped with both write and execute +permissions by default, and make it the OS loader's responsibility to +transition data mappings to code mappings where appropriate. + +Even though the UEFI specification does not appear to leave room for +denying memory attribute changes based on security policy, let's be +cautious and avoid relying on the ability to create read-write-execute +mappings. This is trivially achievable, given that the amount of kernel +code executing via the firmware's 1:1 mapping is rather small and +limited to the .head.text region. So let's drop the NX restrictions only +on that subregion, but not before remapping it as read-only first. + +Signed-off-by: Ard Biesheuvel +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/boot/compressed/Makefile | 2 +- + arch/x86/boot/compressed/misc.c | 1 + + arch/x86/include/asm/boot.h | 1 + + drivers/firmware/efi/libstub/x86-stub.c | 11 ++++++++++- + 4 files changed, 13 insertions(+), 2 deletions(-) + +--- a/arch/x86/boot/compressed/Makefile ++++ b/arch/x86/boot/compressed/Makefile +@@ -84,7 +84,7 @@ LDFLAGS_vmlinux += -T + hostprogs := mkpiggy + HOST_EXTRACFLAGS += -I$(srctree)/tools/include + +-sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' ++sed-voffset := -e 's/^\([0-9a-fA-F]*\) [ABCDGRSTVW] \(_text\|__start_rodata\|__bss_start\|_end\)$$/\#define VO_\2 _AC(0x\1,UL)/p' + + quiet_cmd_voffset = VOFFSET $@ + cmd_voffset = $(NM) $< | sed -n $(sed-voffset) > $@ +--- a/arch/x86/boot/compressed/misc.c ++++ b/arch/x86/boot/compressed/misc.c +@@ -330,6 +330,7 @@ static size_t parse_elf(void *output) + return ehdr.e_entry - LOAD_PHYSICAL_ADDR; + } + ++const unsigned long kernel_text_size = VO___start_rodata - VO__text; + const unsigned long kernel_total_size = VO__end - VO__text; + + static u8 boot_heap[BOOT_HEAP_SIZE] __aligned(4); +--- a/arch/x86/include/asm/boot.h ++++ b/arch/x86/include/asm/boot.h +@@ -81,6 +81,7 @@ + + #ifndef __ASSEMBLY__ + extern unsigned int output_len; ++extern const unsigned long kernel_text_size; + extern const unsigned long kernel_total_size; + + unsigned long decompress_kernel(unsigned char *outbuf, unsigned long virt_addr, +--- a/drivers/firmware/efi/libstub/x86-stub.c ++++ b/drivers/firmware/efi/libstub/x86-stub.c +@@ -238,6 +238,15 @@ efi_status_t efi_adjust_memory_range_pro + rounded_end = roundup(start + size, EFI_PAGE_SIZE); + + if (memattr != NULL) { ++ status = efi_call_proto(memattr, set_memory_attributes, ++ rounded_start, ++ rounded_end - rounded_start, ++ EFI_MEMORY_RO); ++ if (status != EFI_SUCCESS) { ++ efi_warn("Failed to set EFI_MEMORY_RO attribute\n"); ++ return status; ++ } ++ + status = efi_call_proto(memattr, clear_memory_attributes, + rounded_start, + rounded_end - rounded_start, +@@ -816,7 +825,7 @@ static efi_status_t efi_decompress_kerne + + *kernel_entry = addr + entry; + +- return efi_adjust_memory_range_protection(addr, kernel_total_size); ++ return efi_adjust_memory_range_protection(addr, kernel_text_size); + } + + static void __noreturn enter_kernel(unsigned long kernel_addr, diff --git a/queue-6.6/x86-sev-move-early-startup-code-into-.head.text-section.patch b/queue-6.6/x86-sev-move-early-startup-code-into-.head.text-section.patch new file mode 100644 index 00000000000..fbfdf8a06f1 --- /dev/null +++ b/queue-6.6/x86-sev-move-early-startup-code-into-.head.text-section.patch @@ -0,0 +1,194 @@ +From 428080c9b19bfda37c478cd626dbd3851db1aff9 Mon Sep 17 00:00:00 2001 +From: Ard Biesheuvel +Date: Tue, 27 Feb 2024 16:19:16 +0100 +Subject: x86/sev: Move early startup code into .head.text section + +From: Ard Biesheuvel + +commit 428080c9b19bfda37c478cd626dbd3851db1aff9 upstream. + +In preparation for implementing rigorous build time checks to enforce +that only code that can support it will be called from the early 1:1 +mapping of memory, move SEV init code that is called in this manner to +the .head.text section. + +Signed-off-by: Ard Biesheuvel +Signed-off-by: Borislav Petkov (AMD) +Tested-by: Tom Lendacky +Link: https://lore.kernel.org/r/20240227151907.387873-19-ardb+git@google.com +Signed-off-by: Ard Biesheuvel +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/boot/compressed/sev.c | 3 +++ + arch/x86/include/asm/sev.h | 10 +++++----- + arch/x86/kernel/sev-shared.c | 23 ++++++++++------------- + arch/x86/kernel/sev.c | 14 ++++++++------ + 4 files changed, 26 insertions(+), 24 deletions(-) + +--- a/arch/x86/boot/compressed/sev.c ++++ b/arch/x86/boot/compressed/sev.c +@@ -116,6 +116,9 @@ static bool fault_in_kernel_space(unsign + #undef __init + #define __init + ++#undef __head ++#define __head ++ + #define __BOOT_COMPRESSED + + /* Basic instruction decoding support needed */ +--- a/arch/x86/include/asm/sev.h ++++ b/arch/x86/include/asm/sev.h +@@ -199,15 +199,15 @@ static inline int pvalidate(unsigned lon + struct snp_guest_request_ioctl; + + void setup_ghcb(void); +-void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, +- unsigned long npages); +-void __init early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, +- unsigned long npages); ++void early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, ++ unsigned long npages); ++void early_snp_set_memory_shared(unsigned long vaddr, unsigned long paddr, ++ unsigned long npages); + void snp_set_memory_shared(unsigned long vaddr, unsigned long npages); + void snp_set_memory_private(unsigned long vaddr, unsigned long npages); + void snp_set_wakeup_secondary_cpu(void); + bool snp_init(struct boot_params *bp); +-void __init __noreturn snp_abort(void); ++void __noreturn snp_abort(void); + void snp_dmi_setup(void); + int snp_issue_guest_request(u64 exit_code, struct snp_req_data *input, struct snp_guest_request_ioctl *rio); + void snp_accept_memory(phys_addr_t start, phys_addr_t end); +--- a/arch/x86/kernel/sev-shared.c ++++ b/arch/x86/kernel/sev-shared.c +@@ -89,7 +89,8 @@ static bool __init sev_es_check_cpu_feat + return true; + } + +-static void __noreturn sev_es_terminate(unsigned int set, unsigned int reason) ++static void __head __noreturn ++sev_es_terminate(unsigned int set, unsigned int reason) + { + u64 val = GHCB_MSR_TERM_REQ; + +@@ -326,13 +327,7 @@ static int sev_cpuid_hv(struct ghcb *ghc + */ + static const struct snp_cpuid_table *snp_cpuid_get_table(void) + { +- void *ptr; +- +- asm ("lea cpuid_table_copy(%%rip), %0" +- : "=r" (ptr) +- : "p" (&cpuid_table_copy)); +- +- return ptr; ++ return &RIP_REL_REF(cpuid_table_copy); + } + + /* +@@ -391,7 +386,7 @@ static u32 snp_cpuid_calc_xsave_size(u64 + return xsave_size; + } + +-static bool ++static bool __head + snp_cpuid_get_validated_func(struct cpuid_leaf *leaf) + { + const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); +@@ -528,7 +523,8 @@ static int snp_cpuid_postprocess(struct + * Returns -EOPNOTSUPP if feature not enabled. Any other non-zero return value + * should be treated as fatal by caller. + */ +-static int snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) ++static int __head ++snp_cpuid(struct ghcb *ghcb, struct es_em_ctxt *ctxt, struct cpuid_leaf *leaf) + { + const struct snp_cpuid_table *cpuid_table = snp_cpuid_get_table(); + +@@ -570,7 +566,7 @@ static int snp_cpuid(struct ghcb *ghcb, + * page yet, so it only supports the MSR based communication with the + * hypervisor and only the CPUID exit-code. + */ +-void __init do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) ++void __head do_vc_no_ghcb(struct pt_regs *regs, unsigned long exit_code) + { + unsigned int subfn = lower_bits(regs->cx, 32); + unsigned int fn = lower_bits(regs->ax, 32); +@@ -1016,7 +1012,8 @@ struct cc_setup_data { + * Search for a Confidential Computing blob passed in as a setup_data entry + * via the Linux Boot Protocol. + */ +-static struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) ++static __head ++struct cc_blob_sev_info *find_cc_blob_setup_data(struct boot_params *bp) + { + struct cc_setup_data *sd = NULL; + struct setup_data *hdr; +@@ -1043,7 +1040,7 @@ static struct cc_blob_sev_info *find_cc_ + * mapping needs to be updated in sync with all the changes to virtual memory + * layout and related mapping facilities throughout the boot process. + */ +-static void __init setup_cpuid_table(const struct cc_blob_sev_info *cc_info) ++static void __head setup_cpuid_table(const struct cc_blob_sev_info *cc_info) + { + const struct snp_cpuid_table *cpuid_table_fw, *cpuid_table; + int i; +--- a/arch/x86/kernel/sev.c ++++ b/arch/x86/kernel/sev.c +@@ -26,6 +26,7 @@ + #include + #include + ++#include + #include + #include + #include +@@ -683,8 +684,9 @@ static u64 __init get_jump_table_addr(vo + return ret; + } + +-static void early_set_pages_state(unsigned long vaddr, unsigned long paddr, +- unsigned long npages, enum psc_op op) ++static void __head ++early_set_pages_state(unsigned long vaddr, unsigned long paddr, ++ unsigned long npages, enum psc_op op) + { + unsigned long paddr_end; + u64 val; +@@ -740,7 +742,7 @@ e_term: + sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC); + } + +-void __init early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, ++void __head early_snp_set_memory_private(unsigned long vaddr, unsigned long paddr, + unsigned long npages) + { + /* +@@ -2045,7 +2047,7 @@ fail: + * + * Scan for the blob in that order. + */ +-static __init struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) ++static __head struct cc_blob_sev_info *find_cc_blob(struct boot_params *bp) + { + struct cc_blob_sev_info *cc_info; + +@@ -2071,7 +2073,7 @@ found_cc_info: + return cc_info; + } + +-bool __init snp_init(struct boot_params *bp) ++bool __head snp_init(struct boot_params *bp) + { + struct cc_blob_sev_info *cc_info; + +@@ -2093,7 +2095,7 @@ bool __init snp_init(struct boot_params + return true; + } + +-void __init __noreturn snp_abort(void) ++void __head __noreturn snp_abort(void) + { + sev_es_terminate(SEV_TERM_SET_GEN, GHCB_SNP_UNSUPPORTED); + } diff --git a/queue-6.6/x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch b/queue-6.6/x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch new file mode 100644 index 00000000000..883c6243983 --- /dev/null +++ b/queue-6.6/x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch @@ -0,0 +1,213 @@ +From 48204aba801f1b512b3abed10b8e1a63e03f3dd1 Mon Sep 17 00:00:00 2001 +From: Ard Biesheuvel +Date: Tue, 27 Feb 2024 16:19:15 +0100 +Subject: x86/sme: Move early SME kernel encryption handling into .head.text + +From: Ard Biesheuvel + +commit 48204aba801f1b512b3abed10b8e1a63e03f3dd1 upstream. + +The .head.text section is the initial primary entrypoint of the core +kernel, and is entered with the CPU executing from a 1:1 mapping of +memory. Such code must never access global variables using absolute +references, as these are based on the kernel virtual mapping which is +not active yet at this point. + +Given that the SME startup code is also called from this early execution +context, move it into .head.text as well. This will allow more thorough +build time checks in the future to ensure that early startup code only +uses RIP-relative references to global variables. + +Also replace some occurrences of __pa_symbol() [which relies on the +compiler generating an absolute reference, which is not guaranteed] and +an open coded RIP-relative access with RIP_REL_REF(). + +Signed-off-by: Ard Biesheuvel +Signed-off-by: Borislav Petkov (AMD) +Tested-by: Tom Lendacky +Link: https://lore.kernel.org/r/20240227151907.387873-18-ardb+git@google.com +Signed-off-by: Ard Biesheuvel +Signed-off-by: Greg Kroah-Hartman +--- + arch/x86/include/asm/mem_encrypt.h | 8 +++---- + arch/x86/mm/mem_encrypt_identity.c | 42 ++++++++++++++----------------------- + 2 files changed, 21 insertions(+), 29 deletions(-) + +--- a/arch/x86/include/asm/mem_encrypt.h ++++ b/arch/x86/include/asm/mem_encrypt.h +@@ -46,8 +46,8 @@ void __init sme_unmap_bootdata(char *rea + void __init sme_early_init(void); + void __init sev_setup_arch(void); + +-void __init sme_encrypt_kernel(struct boot_params *bp); +-void __init sme_enable(struct boot_params *bp); ++void sme_encrypt_kernel(struct boot_params *bp); ++void sme_enable(struct boot_params *bp); + + int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size); + int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size); +@@ -81,8 +81,8 @@ static inline void __init sme_unmap_boot + static inline void __init sme_early_init(void) { } + static inline void __init sev_setup_arch(void) { } + +-static inline void __init sme_encrypt_kernel(struct boot_params *bp) { } +-static inline void __init sme_enable(struct boot_params *bp) { } ++static inline void sme_encrypt_kernel(struct boot_params *bp) { } ++static inline void sme_enable(struct boot_params *bp) { } + + static inline void sev_es_init_vc_handling(void) { } + +--- a/arch/x86/mm/mem_encrypt_identity.c ++++ b/arch/x86/mm/mem_encrypt_identity.c +@@ -41,6 +41,7 @@ + #include + #include + ++#include + #include + #include + #include +@@ -98,7 +99,7 @@ static char sme_workarea[2 * PMD_SIZE] _ + static char sme_cmdline_arg[] __initdata = "mem_encrypt"; + static char sme_cmdline_on[] __initdata = "on"; + +-static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd) ++static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd) + { + unsigned long pgd_start, pgd_end, pgd_size; + pgd_t *pgd_p; +@@ -113,7 +114,7 @@ static void __init sme_clear_pgd(struct + memset(pgd_p, 0, pgd_size); + } + +-static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) ++static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd) + { + pgd_t *pgd; + p4d_t *p4d; +@@ -150,7 +151,7 @@ static pud_t __init *sme_prepare_pgd(str + return pud; + } + +-static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd) ++static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd) + { + pud_t *pud; + pmd_t *pmd; +@@ -166,7 +167,7 @@ static void __init sme_populate_pgd_larg + set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags)); + } + +-static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd) ++static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd) + { + pud_t *pud; + pmd_t *pmd; +@@ -192,7 +193,7 @@ static void __init sme_populate_pgd(stru + set_pte(pte, __pte(ppd->paddr | ppd->pte_flags)); + } + +-static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) ++static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd) + { + while (ppd->vaddr < ppd->vaddr_end) { + sme_populate_pgd_large(ppd); +@@ -202,7 +203,7 @@ static void __init __sme_map_range_pmd(s + } + } + +-static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd) ++static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd) + { + while (ppd->vaddr < ppd->vaddr_end) { + sme_populate_pgd(ppd); +@@ -212,7 +213,7 @@ static void __init __sme_map_range_pte(s + } + } + +-static void __init __sme_map_range(struct sme_populate_pgd_data *ppd, ++static void __head __sme_map_range(struct sme_populate_pgd_data *ppd, + pmdval_t pmd_flags, pteval_t pte_flags) + { + unsigned long vaddr_end; +@@ -236,22 +237,22 @@ static void __init __sme_map_range(struc + __sme_map_range_pte(ppd); + } + +-static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd) ++static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd) + { + __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC); + } + +-static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd) ++static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd) + { + __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC); + } + +-static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd) ++static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd) + { + __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP); + } + +-static unsigned long __init sme_pgtable_calc(unsigned long len) ++static unsigned long __head sme_pgtable_calc(unsigned long len) + { + unsigned long entries = 0, tables = 0; + +@@ -288,7 +289,7 @@ static unsigned long __init sme_pgtable_ + return entries + tables; + } + +-void __init sme_encrypt_kernel(struct boot_params *bp) ++void __head sme_encrypt_kernel(struct boot_params *bp) + { + unsigned long workarea_start, workarea_end, workarea_len; + unsigned long execute_start, execute_end, execute_len; +@@ -323,9 +324,8 @@ void __init sme_encrypt_kernel(struct bo + * memory from being cached. + */ + +- /* Physical addresses gives us the identity mapped virtual addresses */ +- kernel_start = __pa_symbol(_text); +- kernel_end = ALIGN(__pa_symbol(_end), PMD_SIZE); ++ kernel_start = (unsigned long)RIP_REL_REF(_text); ++ kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE); + kernel_len = kernel_end - kernel_start; + + initrd_start = 0; +@@ -343,14 +343,6 @@ void __init sme_encrypt_kernel(struct bo + #endif + + /* +- * We're running identity mapped, so we must obtain the address to the +- * SME encryption workarea using rip-relative addressing. +- */ +- asm ("lea sme_workarea(%%rip), %0" +- : "=r" (workarea_start) +- : "p" (sme_workarea)); +- +- /* + * Calculate required number of workarea bytes needed: + * executable encryption area size: + * stack page (PAGE_SIZE) +@@ -359,7 +351,7 @@ void __init sme_encrypt_kernel(struct bo + * pagetable structures for the encryption of the kernel + * pagetable structures for workarea (in case not currently mapped) + */ +- execute_start = workarea_start; ++ execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea); + execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE; + execute_len = execute_end - execute_start; + +@@ -502,7 +494,7 @@ void __init sme_encrypt_kernel(struct bo + native_write_cr3(__native_read_cr3()); + } + +-void __init sme_enable(struct boot_params *bp) ++void __head sme_enable(struct boot_params *bp) + { + const char *cmdline_ptr, *cmdline_arg, *cmdline_on; + unsigned int eax, ebx, ecx, edx;