queue-6.6/x86-sme-move-early-sme-kernel-encryption-handling-into-.head.text.patch

   1 From 48204aba801f1b512b3abed10b8e1a63e03f3dd1 Mon Sep 17 00:00:00 2001
   2 From: Ard Biesheuvel <ardb@kernel.org>
   3 Date: Tue, 27 Feb 2024 16:19:15 +0100
   4 Subject: x86/sme: Move early SME kernel encryption handling into .head.text
   5
   6 From: Ard Biesheuvel <ardb@kernel.org>
   7
   8 commit 48204aba801f1b512b3abed10b8e1a63e03f3dd1 upstream.
   9
  10 The .head.text section is the initial primary entrypoint of the core
  11 kernel, and is entered with the CPU executing from a 1:1 mapping of
  12 memory. Such code must never access global variables using absolute
  13 references, as these are based on the kernel virtual mapping which is
  14 not active yet at this point.
  15
  16 Given that the SME startup code is also called from this early execution
  17 context, move it into .head.text as well. This will allow more thorough
  18 build time checks in the future to ensure that early startup code only
  19 uses RIP-relative references to global variables.
  20
  21 Also replace some occurrences of __pa_symbol() [which relies on the
  22 compiler generating an absolute reference, which is not guaranteed] and
  23 an open coded RIP-relative access with RIP_REL_REF().
  24
  25 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  26 Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
  27 Tested-by: Tom Lendacky <thomas.lendacky@amd.com>
  28 Link: https://lore.kernel.org/r/20240227151907.387873-18-ardb+git@google.com
  29 Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
  30 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  31 ---
  32  arch/x86/include/asm/mem_encrypt.h |    8 +++----
  33  arch/x86/mm/mem_encrypt_identity.c |   42 ++++++++++++++-----------------------
  34  2 files changed, 21 insertions(+), 29 deletions(-)
  35
  36 --- a/arch/x86/include/asm/mem_encrypt.h
  37 +++ b/arch/x86/include/asm/mem_encrypt.h
  38 @@ -46,8 +46,8 @@ void __init sme_unmap_bootdata(char *rea
  39  void __init sme_early_init(void);
  40  void __init sev_setup_arch(void);
  41
  42 -void __init sme_encrypt_kernel(struct boot_params *bp);
  43 -void __init sme_enable(struct boot_params *bp);
  44 +void sme_encrypt_kernel(struct boot_params *bp);
  45 +void sme_enable(struct boot_params *bp);
  46
  47  int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
  48  int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
  49 @@ -81,8 +81,8 @@ static inline void __init sme_unmap_boot
  50  static inline void __init sme_early_init(void) { }
  51  static inline void __init sev_setup_arch(void) { }
  52
  53 -static inline void __init sme_encrypt_kernel(struct boot_params *bp) { }
  54 -static inline void __init sme_enable(struct boot_params *bp) { }
  55 +static inline void sme_encrypt_kernel(struct boot_params *bp) { }
  56 +static inline void sme_enable(struct boot_params *bp) { }
  57
  58  static inline void sev_es_init_vc_handling(void) { }
  59
  60 --- a/arch/x86/mm/mem_encrypt_identity.c
  61 +++ b/arch/x86/mm/mem_encrypt_identity.c
  62 @@ -41,6 +41,7 @@
  63  #include <linux/mem_encrypt.h>
  64  #include <linux/cc_platform.h>
  65
  66 +#include <asm/init.h>
  67  #include <asm/setup.h>
  68  #include <asm/sections.h>
  69  #include <asm/cmdline.h>
  70 @@ -98,7 +99,7 @@ static char sme_workarea[2 * PMD_SIZE] _
  71  static char sme_cmdline_arg[] __initdata = "mem_encrypt";
  72  static char sme_cmdline_on[]  __initdata = "on";
  73
  74 -static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
  75 +static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
  76  {
  77         unsigned long pgd_start, pgd_end, pgd_size;
  78         pgd_t *pgd_p;
  79 @@ -113,7 +114,7 @@ static void __init sme_clear_pgd(struct
  80         memset(pgd_p, 0, pgd_size);
  81  }
  82
  83 -static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
  84 +static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
  85  {
  86         pgd_t *pgd;
  87         p4d_t *p4d;
  88 @@ -150,7 +151,7 @@ static pud_t __init *sme_prepare_pgd(str
  89         return pud;
  90  }
  91
  92 -static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
  93 +static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
  94  {
  95         pud_t *pud;
  96         pmd_t *pmd;
  97 @@ -166,7 +167,7 @@ static void __init sme_populate_pgd_larg
  98         set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
  99  }
 100
 101 -static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 102 +static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
 103  {
 104         pud_t *pud;
 105         pmd_t *pmd;
 106 @@ -192,7 +193,7 @@ static void __init sme_populate_pgd(stru
 107                 set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
 108  }
 109
 110 -static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 111 +static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
 112  {
 113         while (ppd->vaddr < ppd->vaddr_end) {
 114                 sme_populate_pgd_large(ppd);
 115 @@ -202,7 +203,7 @@ static void __init __sme_map_range_pmd(s
 116         }
 117  }
 118
 119 -static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 120 +static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
 121  {
 122         while (ppd->vaddr < ppd->vaddr_end) {
 123                 sme_populate_pgd(ppd);
 124 @@ -212,7 +213,7 @@ static void __init __sme_map_range_pte(s
 125         }
 126  }
 127
 128 -static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
 129 +static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
 130                                    pmdval_t pmd_flags, pteval_t pte_flags)
 131  {
 132         unsigned long vaddr_end;
 133 @@ -236,22 +237,22 @@ static void __init __sme_map_range(struc
 134         __sme_map_range_pte(ppd);
 135  }
 136
 137 -static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
 138 +static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
 139  {
 140         __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
 141  }
 142
 143 -static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
 144 +static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
 145  {
 146         __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
 147  }
 148
 149 -static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
 150 +static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
 151  {
 152         __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
 153  }
 154
 155 -static unsigned long __init sme_pgtable_calc(unsigned long len)
 156 +static unsigned long __head sme_pgtable_calc(unsigned long len)
 157  {
 158         unsigned long entries = 0, tables = 0;
 159
 160 @@ -288,7 +289,7 @@ static unsigned long __init sme_pgtable_
 161         return entries + tables;
 162  }
 163
 164 -void __init sme_encrypt_kernel(struct boot_params *bp)
 165 +void __head sme_encrypt_kernel(struct boot_params *bp)
 166  {
 167         unsigned long workarea_start, workarea_end, workarea_len;
 168         unsigned long execute_start, execute_end, execute_len;
 169 @@ -323,9 +324,8 @@ void __init sme_encrypt_kernel(struct bo
 170          *     memory from being cached.
 171          */
 172
 173 -       /* Physical addresses gives us the identity mapped virtual addresses */
 174 -       kernel_start = __pa_symbol(_text);
 175 -       kernel_end = ALIGN(__pa_symbol(_end), PMD_SIZE);
 176 +       kernel_start = (unsigned long)RIP_REL_REF(_text);
 177 +       kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
 178         kernel_len = kernel_end - kernel_start;
 179
 180         initrd_start = 0;
 181 @@ -343,14 +343,6 @@ void __init sme_encrypt_kernel(struct bo
 182  #endif
 183
 184         /*
 185 -        * We're running identity mapped, so we must obtain the address to the
 186 -        * SME encryption workarea using rip-relative addressing.
 187 -        */
 188 -       asm ("lea sme_workarea(%%rip), %0"
 189 -            : "=r" (workarea_start)
 190 -            : "p" (sme_workarea));
 191 -
 192 -       /*
 193          * Calculate required number of workarea bytes needed:
 194          *   executable encryption area size:
 195          *     stack page (PAGE_SIZE)
 196 @@ -359,7 +351,7 @@ void __init sme_encrypt_kernel(struct bo
 197          *   pagetable structures for the encryption of the kernel
 198          *   pagetable structures for workarea (in case not currently mapped)
 199          */
 200 -       execute_start = workarea_start;
 201 +       execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
 202         execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
 203         execute_len = execute_end - execute_start;
 204
 205 @@ -502,7 +494,7 @@ void __init sme_encrypt_kernel(struct bo
 206         native_write_cr3(__native_read_cr3());
 207  }
 208
 209 -void __init sme_enable(struct boot_params *bp)
 210 +void __head sme_enable(struct boot_params *bp)
 211  {
 212         const char *cmdline_ptr, *cmdline_arg, *cmdline_on;
 213         unsigned int eax, ebx, ecx, edx;