x86-tsc-fix-erroneous-tsc-rate-on-skylake-xeon.patch
pipe-avoid-round_pipe_size-nr_pages-overflow-on-32-bit.patch
x86-apic-vector-fix-off-by-one-in-error-path.patch
+x86-mm-clean-up-register-saving-in-the-__enc_copy-assembly-code.patch
+x86-mm-use-a-struct-to-reduce-parameters-for-sme-pgd-mapping.patch
+x86-mm-centralize-pmd-flags-in-sme_encrypt_kernel.patch
+x86-mm-prepare-sme_encrypt_kernel-for-page-aligned-encryption.patch
--- /dev/null
+From 2b5d00b6c2cdd94f6d6a494a6f6c0c0fc7b8e711 Mon Sep 17 00:00:00 2001
+From: Tom Lendacky <thomas.lendacky@amd.com>
+Date: Wed, 10 Jan 2018 13:26:16 -0600
+Subject: x86/mm: Centralize PMD flags in sme_encrypt_kernel()
+
+From: Tom Lendacky <thomas.lendacky@amd.com>
+
+commit 2b5d00b6c2cdd94f6d6a494a6f6c0c0fc7b8e711 upstream.
+
+In preparation for encrypting more than just the kernel during early
+boot processing, centralize the use of the PMD flag settings based
+on the type of mapping desired. When 4KB aligned encryption is added,
+this will allow either PTE flags or large page PMD flags to be used
+without requiring the caller to adjust.
+
+Tested-by: Gabriel Craciunescu <nix.or.die@gmail.com>
+Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
+Reviewed-by: Borislav Petkov <bp@suse.de>
+Cc: Borislav Petkov <bp@alien8.de>
+Cc: Brijesh Singh <brijesh.singh@amd.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: http://lkml.kernel.org/r/20180110192615.6026.14767.stgit@tlendack-t1.amdoffice.net
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ arch/x86/mm/mem_encrypt.c | 137 ++++++++++++++++++++++++++--------------------
+ 1 file changed, 79 insertions(+), 58 deletions(-)
+
+--- a/arch/x86/mm/mem_encrypt.c
++++ b/arch/x86/mm/mem_encrypt.c
+@@ -217,31 +217,39 @@ struct sme_populate_pgd_data {
+ void *pgtable_area;
+ pgd_t *pgd;
+
+- pmdval_t pmd_val;
++ pmdval_t pmd_flags;
++ unsigned long paddr;
++
+ unsigned long vaddr;
++ unsigned long vaddr_end;
+ };
+
+-static void __init sme_clear_pgd(pgd_t *pgd_base, unsigned long start,
+- unsigned long end)
++static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
+ {
+ unsigned long pgd_start, pgd_end, pgd_size;
+ pgd_t *pgd_p;
+
+- pgd_start = start & PGDIR_MASK;
+- pgd_end = end & PGDIR_MASK;
++ pgd_start = ppd->vaddr & PGDIR_MASK;
++ pgd_end = ppd->vaddr_end & PGDIR_MASK;
+
+- pgd_size = (((pgd_end - pgd_start) / PGDIR_SIZE) + 1);
+- pgd_size *= sizeof(pgd_t);
++ pgd_size = (((pgd_end - pgd_start) / PGDIR_SIZE) + 1) * sizeof(pgd_t);
+
+- pgd_p = pgd_base + pgd_index(start);
++ pgd_p = ppd->pgd + pgd_index(ppd->vaddr);
+
+ memset(pgd_p, 0, pgd_size);
+ }
+
+-#define PGD_FLAGS _KERNPG_TABLE_NOENC
+-#define P4D_FLAGS _KERNPG_TABLE_NOENC
+-#define PUD_FLAGS _KERNPG_TABLE_NOENC
+-#define PMD_FLAGS (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL)
++#define PGD_FLAGS _KERNPG_TABLE_NOENC
++#define P4D_FLAGS _KERNPG_TABLE_NOENC
++#define PUD_FLAGS _KERNPG_TABLE_NOENC
++
++#define PMD_FLAGS_LARGE (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL)
++
++#define PMD_FLAGS_DEC PMD_FLAGS_LARGE
++#define PMD_FLAGS_DEC_WP ((PMD_FLAGS_DEC & ~_PAGE_CACHE_MASK) | \
++ (_PAGE_PAT | _PAGE_PWT))
++
++#define PMD_FLAGS_ENC (PMD_FLAGS_LARGE | _PAGE_ENC)
+
+ static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
+ {
+@@ -310,7 +318,35 @@ static void __init sme_populate_pgd_larg
+
+ pmd_p += pmd_index(ppd->vaddr);
+ if (!native_pmd_val(*pmd_p) || !(native_pmd_val(*pmd_p) & _PAGE_PSE))
+- native_set_pmd(pmd_p, native_make_pmd(ppd->pmd_val));
++ native_set_pmd(pmd_p, native_make_pmd(ppd->paddr | ppd->pmd_flags));
++}
++
++static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
++ pmdval_t pmd_flags)
++{
++ ppd->pmd_flags = pmd_flags;
++
++ while (ppd->vaddr < ppd->vaddr_end) {
++ sme_populate_pgd_large(ppd);
++
++ ppd->vaddr += PMD_PAGE_SIZE;
++ ppd->paddr += PMD_PAGE_SIZE;
++ }
++}
++
++static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
++{
++ __sme_map_range(ppd, PMD_FLAGS_ENC);
++}
++
++static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
++{
++ __sme_map_range(ppd, PMD_FLAGS_DEC);
++}
++
++static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
++{
++ __sme_map_range(ppd, PMD_FLAGS_DEC_WP);
+ }
+
+ static unsigned long __init sme_pgtable_calc(unsigned long len)
+@@ -370,7 +406,6 @@ void __init sme_encrypt_kernel(void)
+ unsigned long kernel_start, kernel_end, kernel_len;
+ struct sme_populate_pgd_data ppd;
+ unsigned long pgtable_area_len;
+- unsigned long paddr, pmd_flags;
+ unsigned long decrypted_base;
+
+ if (!sme_active())
+@@ -442,14 +477,10 @@ void __init sme_encrypt_kernel(void)
+ * addressing the workarea.
+ */
+ ppd.pgd = (pgd_t *)native_read_cr3_pa();
+- paddr = workarea_start;
+- while (paddr < workarea_end) {
+- ppd.pmd_val = paddr + PMD_FLAGS;
+- ppd.vaddr = paddr;
+- sme_populate_pgd_large(&ppd);
+-
+- paddr += PMD_PAGE_SIZE;
+- }
++ ppd.paddr = workarea_start;
++ ppd.vaddr = workarea_start;
++ ppd.vaddr_end = workarea_end;
++ sme_map_range_decrypted(&ppd);
+
+ /* Flush the TLB - no globals so cr3 is enough */
+ native_write_cr3(__native_read_cr3());
+@@ -464,17 +495,6 @@ void __init sme_encrypt_kernel(void)
+ memset(ppd.pgd, 0, sizeof(pgd_t) * PTRS_PER_PGD);
+ ppd.pgtable_area += sizeof(pgd_t) * PTRS_PER_PGD;
+
+- /* Add encrypted kernel (identity) mappings */
+- pmd_flags = PMD_FLAGS | _PAGE_ENC;
+- paddr = kernel_start;
+- while (paddr < kernel_end) {
+- ppd.pmd_val = paddr + pmd_flags;
+- ppd.vaddr = paddr;
+- sme_populate_pgd_large(&ppd);
+-
+- paddr += PMD_PAGE_SIZE;
+- }
+-
+ /*
+ * A different PGD index/entry must be used to get different
+ * pagetable entries for the decrypted mapping. Choose the next
+@@ -484,29 +504,28 @@ void __init sme_encrypt_kernel(void)
+ decrypted_base = (pgd_index(workarea_end) + 1) & (PTRS_PER_PGD - 1);
+ decrypted_base <<= PGDIR_SHIFT;
+
+- /* Add decrypted, write-protected kernel (non-identity) mappings */
+- pmd_flags = (PMD_FLAGS & ~_PAGE_CACHE_MASK) | (_PAGE_PAT | _PAGE_PWT);
+- paddr = kernel_start;
+- while (paddr < kernel_end) {
+- ppd.pmd_val = paddr + pmd_flags;
+- ppd.vaddr = paddr + decrypted_base;
+- sme_populate_pgd_large(&ppd);
++ /* Add encrypted kernel (identity) mappings */
++ ppd.paddr = kernel_start;
++ ppd.vaddr = kernel_start;
++ ppd.vaddr_end = kernel_end;
++ sme_map_range_encrypted(&ppd);
+
+- paddr += PMD_PAGE_SIZE;
+- }
++ /* Add decrypted, write-protected kernel (non-identity) mappings */
++ ppd.paddr = kernel_start;
++ ppd.vaddr = kernel_start + decrypted_base;
++ ppd.vaddr_end = kernel_end + decrypted_base;
++ sme_map_range_decrypted_wp(&ppd);
+
+ /* Add decrypted workarea mappings to both kernel mappings */
+- paddr = workarea_start;
+- while (paddr < workarea_end) {
+- ppd.pmd_val = paddr + PMD_FLAGS;
+- ppd.vaddr = paddr;
+- sme_populate_pgd_large(&ppd);
+-
+- ppd.vaddr = paddr + decrypted_base;
+- sme_populate_pgd_large(&ppd);
+-
+- paddr += PMD_PAGE_SIZE;
+- }
++ ppd.paddr = workarea_start;
++ ppd.vaddr = workarea_start;
++ ppd.vaddr_end = workarea_end;
++ sme_map_range_decrypted(&ppd);
++
++ ppd.paddr = workarea_start;
++ ppd.vaddr = workarea_start + decrypted_base;
++ ppd.vaddr_end = workarea_end + decrypted_base;
++ sme_map_range_decrypted(&ppd);
+
+ /* Perform the encryption */
+ sme_encrypt_execute(kernel_start, kernel_start + decrypted_base,
+@@ -517,11 +536,13 @@ void __init sme_encrypt_kernel(void)
+ * the decrypted areas - all that is needed for this is to remove
+ * the PGD entry/entries.
+ */
+- sme_clear_pgd(ppd.pgd, kernel_start + decrypted_base,
+- kernel_end + decrypted_base);
+-
+- sme_clear_pgd(ppd.pgd, workarea_start + decrypted_base,
+- workarea_end + decrypted_base);
++ ppd.vaddr = kernel_start + decrypted_base;
++ ppd.vaddr_end = kernel_end + decrypted_base;
++ sme_clear_pgd(&ppd);
++
++ ppd.vaddr = workarea_start + decrypted_base;
++ ppd.vaddr_end = workarea_end + decrypted_base;
++ sme_clear_pgd(&ppd);
+
+ /* Flush the TLB - no globals so cr3 is enough */
+ native_write_cr3(__native_read_cr3());
--- /dev/null
+From 1303880179e67c59e801429b7e5d0f6b21137d99 Mon Sep 17 00:00:00 2001
+From: Tom Lendacky <thomas.lendacky@amd.com>
+Date: Wed, 10 Jan 2018 13:25:56 -0600
+Subject: x86/mm: Clean up register saving in the __enc_copy() assembly code
+
+From: Tom Lendacky <thomas.lendacky@amd.com>
+
+commit 1303880179e67c59e801429b7e5d0f6b21137d99 upstream.
+
+Clean up the use of PUSH and POP and when registers are saved in the
+__enc_copy() assembly function in order to improve the readability of the code.
+
+Move parameter register saving into general purpose registers earlier
+in the code and move all the pushes to the beginning of the function
+with corresponding pops at the end.
+
+We do this to prepare fixes.
+
+Tested-by: Gabriel Craciunescu <nix.or.die@gmail.com>
+Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
+Reviewed-by: Borislav Petkov <bp@suse.de>
+Cc: Borislav Petkov <bp@alien8.de>
+Cc: Brijesh Singh <brijesh.singh@amd.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: http://lkml.kernel.org/r/20180110192556.6026.74187.stgit@tlendack-t1.amdoffice.net
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ arch/x86/mm/mem_encrypt_boot.S | 20 ++++++++++----------
+ 1 file changed, 10 insertions(+), 10 deletions(-)
+
+--- a/arch/x86/mm/mem_encrypt_boot.S
++++ b/arch/x86/mm/mem_encrypt_boot.S
+@@ -103,20 +103,19 @@ ENTRY(__enc_copy)
+ orq $X86_CR4_PGE, %rdx
+ mov %rdx, %cr4
+
++ push %r15
++
++ movq %rcx, %r9 /* Save kernel length */
++ movq %rdi, %r10 /* Save encrypted kernel address */
++ movq %rsi, %r11 /* Save decrypted kernel address */
++
+ /* Set the PAT register PA5 entry to write-protect */
+- push %rcx
+ movl $MSR_IA32_CR_PAT, %ecx
+ rdmsr
+- push %rdx /* Save original PAT value */
++ mov %rdx, %r15 /* Save original PAT value */
+ andl $0xffff00ff, %edx /* Clear PA5 */
+ orl $0x00000500, %edx /* Set PA5 to WP */
+ wrmsr
+- pop %rdx /* RDX contains original PAT value */
+- pop %rcx
+-
+- movq %rcx, %r9 /* Save kernel length */
+- movq %rdi, %r10 /* Save encrypted kernel address */
+- movq %rsi, %r11 /* Save decrypted kernel address */
+
+ wbinvd /* Invalidate any cache entries */
+
+@@ -138,12 +137,13 @@ ENTRY(__enc_copy)
+ jnz 1b /* Kernel length not zero? */
+
+ /* Restore PAT register */
+- push %rdx /* Save original PAT value */
+ movl $MSR_IA32_CR_PAT, %ecx
+ rdmsr
+- pop %rdx /* Restore original PAT value */
++ mov %r15, %rdx /* Restore original PAT value */
+ wrmsr
+
++ pop %r15
++
+ ret
+ .L__enc_copy_end:
+ ENDPROC(__enc_copy)
--- /dev/null
+From cc5f01e28d6c60f274fd1e33b245f679f79f543c Mon Sep 17 00:00:00 2001
+From: Tom Lendacky <thomas.lendacky@amd.com>
+Date: Wed, 10 Jan 2018 13:26:26 -0600
+Subject: x86/mm: Prepare sme_encrypt_kernel() for PAGE aligned encryption
+
+From: Tom Lendacky <thomas.lendacky@amd.com>
+
+commit cc5f01e28d6c60f274fd1e33b245f679f79f543c upstream.
+
+In preparation for encrypting more than just the kernel, the encryption
+support in sme_encrypt_kernel() needs to support 4KB page aligned
+encryption instead of just 2MB large page aligned encryption.
+
+Update the routines that populate the PGD to support non-2MB aligned
+addresses. This is done by creating PTE page tables for the start
+and end portion of the address range that fall outside of the 2MB
+alignment. This results in, at most, two extra pages to hold the
+PTE entries for each mapping of a range.
+
+Tested-by: Gabriel Craciunescu <nix.or.die@gmail.com>
+Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
+Reviewed-by: Borislav Petkov <bp@suse.de>
+Cc: Borislav Petkov <bp@alien8.de>
+Cc: Brijesh Singh <brijesh.singh@amd.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: http://lkml.kernel.org/r/20180110192626.6026.75387.stgit@tlendack-t1.amdoffice.net
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ arch/x86/mm/mem_encrypt.c | 123 +++++++++++++++++++++++++++++++++++------
+ arch/x86/mm/mem_encrypt_boot.S | 20 ++++--
+ 2 files changed, 121 insertions(+), 22 deletions(-)
+
+--- a/arch/x86/mm/mem_encrypt.c
++++ b/arch/x86/mm/mem_encrypt.c
+@@ -218,6 +218,7 @@ struct sme_populate_pgd_data {
+ pgd_t *pgd;
+
+ pmdval_t pmd_flags;
++ pteval_t pte_flags;
+ unsigned long paddr;
+
+ unsigned long vaddr;
+@@ -242,6 +243,7 @@ static void __init sme_clear_pgd(struct
+ #define PGD_FLAGS _KERNPG_TABLE_NOENC
+ #define P4D_FLAGS _KERNPG_TABLE_NOENC
+ #define PUD_FLAGS _KERNPG_TABLE_NOENC
++#define PMD_FLAGS _KERNPG_TABLE_NOENC
+
+ #define PMD_FLAGS_LARGE (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL)
+
+@@ -251,7 +253,15 @@ static void __init sme_clear_pgd(struct
+
+ #define PMD_FLAGS_ENC (PMD_FLAGS_LARGE | _PAGE_ENC)
+
+-static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
++#define PTE_FLAGS (__PAGE_KERNEL_EXEC & ~_PAGE_GLOBAL)
++
++#define PTE_FLAGS_DEC PTE_FLAGS
++#define PTE_FLAGS_DEC_WP ((PTE_FLAGS_DEC & ~_PAGE_CACHE_MASK) | \
++ (_PAGE_PAT | _PAGE_PWT))
++
++#define PTE_FLAGS_ENC (PTE_FLAGS | _PAGE_ENC)
++
++static pmd_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
+ {
+ pgd_t *pgd_p;
+ p4d_t *p4d_p;
+@@ -302,7 +312,7 @@ static void __init sme_populate_pgd_larg
+ pud_p += pud_index(ppd->vaddr);
+ if (native_pud_val(*pud_p)) {
+ if (native_pud_val(*pud_p) & _PAGE_PSE)
+- return;
++ return NULL;
+
+ pmd_p = (pmd_t *)(native_pud_val(*pud_p) & ~PTE_FLAGS_MASK);
+ } else {
+@@ -316,16 +326,55 @@ static void __init sme_populate_pgd_larg
+ native_set_pud(pud_p, pud);
+ }
+
++ return pmd_p;
++}
++
++static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
++{
++ pmd_t *pmd_p;
++
++ pmd_p = sme_prepare_pgd(ppd);
++ if (!pmd_p)
++ return;
++
+ pmd_p += pmd_index(ppd->vaddr);
+ if (!native_pmd_val(*pmd_p) || !(native_pmd_val(*pmd_p) & _PAGE_PSE))
+ native_set_pmd(pmd_p, native_make_pmd(ppd->paddr | ppd->pmd_flags));
+ }
+
+-static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
+- pmdval_t pmd_flags)
++static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
+ {
+- ppd->pmd_flags = pmd_flags;
++ pmd_t *pmd_p;
++ pte_t *pte_p;
+
++ pmd_p = sme_prepare_pgd(ppd);
++ if (!pmd_p)
++ return;
++
++ pmd_p += pmd_index(ppd->vaddr);
++ if (native_pmd_val(*pmd_p)) {
++ if (native_pmd_val(*pmd_p) & _PAGE_PSE)
++ return;
++
++ pte_p = (pte_t *)(native_pmd_val(*pmd_p) & ~PTE_FLAGS_MASK);
++ } else {
++ pmd_t pmd;
++
++ pte_p = ppd->pgtable_area;
++ memset(pte_p, 0, sizeof(*pte_p) * PTRS_PER_PTE);
++ ppd->pgtable_area += sizeof(*pte_p) * PTRS_PER_PTE;
++
++ pmd = native_make_pmd((pteval_t)pte_p + PMD_FLAGS);
++ native_set_pmd(pmd_p, pmd);
++ }
++
++ pte_p += pte_index(ppd->vaddr);
++ if (!native_pte_val(*pte_p))
++ native_set_pte(pte_p, native_make_pte(ppd->paddr | ppd->pte_flags));
++}
++
++static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
++{
+ while (ppd->vaddr < ppd->vaddr_end) {
+ sme_populate_pgd_large(ppd);
+
+@@ -334,33 +383,71 @@ static void __init __sme_map_range(struc
+ }
+ }
+
++static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
++{
++ while (ppd->vaddr < ppd->vaddr_end) {
++ sme_populate_pgd(ppd);
++
++ ppd->vaddr += PAGE_SIZE;
++ ppd->paddr += PAGE_SIZE;
++ }
++}
++
++static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
++ pmdval_t pmd_flags, pteval_t pte_flags)
++{
++ unsigned long vaddr_end;
++
++ ppd->pmd_flags = pmd_flags;
++ ppd->pte_flags = pte_flags;
++
++ /* Save original end value since we modify the struct value */
++ vaddr_end = ppd->vaddr_end;
++
++ /* If start is not 2MB aligned, create PTE entries */
++ ppd->vaddr_end = ALIGN(ppd->vaddr, PMD_PAGE_SIZE);
++ __sme_map_range_pte(ppd);
++
++ /* Create PMD entries */
++ ppd->vaddr_end = vaddr_end & PMD_PAGE_MASK;
++ __sme_map_range_pmd(ppd);
++
++ /* If end is not 2MB aligned, create PTE entries */
++ ppd->vaddr_end = vaddr_end;
++ __sme_map_range_pte(ppd);
++}
++
+ static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
+ {
+- __sme_map_range(ppd, PMD_FLAGS_ENC);
++ __sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
+ }
+
+ static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
+ {
+- __sme_map_range(ppd, PMD_FLAGS_DEC);
++ __sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
+ }
+
+ static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
+ {
+- __sme_map_range(ppd, PMD_FLAGS_DEC_WP);
++ __sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
+ }
+
+ static unsigned long __init sme_pgtable_calc(unsigned long len)
+ {
+- unsigned long p4d_size, pud_size, pmd_size;
++ unsigned long p4d_size, pud_size, pmd_size, pte_size;
+ unsigned long total;
+
+ /*
+ * Perform a relatively simplistic calculation of the pagetable
+- * entries that are needed. That mappings will be covered by 2MB
+- * PMD entries so we can conservatively calculate the required
++ * entries that are needed. Those mappings will be covered mostly
++ * by 2MB PMD entries so we can conservatively calculate the required
+ * number of P4D, PUD and PMD structures needed to perform the
+- * mappings. Incrementing the count for each covers the case where
+- * the addresses cross entries.
++ * mappings. For mappings that are not 2MB aligned, PTE mappings
++ * would be needed for the start and end portion of the address range
++ * that fall outside of the 2MB alignment. This results in, at most,
++ * two extra pages to hold PTE entries for each range that is mapped.
++ * Incrementing the count for each covers the case where the addresses
++ * cross entries.
+ */
+ if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+ p4d_size = (ALIGN(len, PGDIR_SIZE) / PGDIR_SIZE) + 1;
+@@ -374,8 +461,9 @@ static unsigned long __init sme_pgtable_
+ }
+ pmd_size = (ALIGN(len, PUD_SIZE) / PUD_SIZE) + 1;
+ pmd_size *= sizeof(pmd_t) * PTRS_PER_PMD;
++ pte_size = 2 * sizeof(pte_t) * PTRS_PER_PTE;
+
+- total = p4d_size + pud_size + pmd_size;
++ total = p4d_size + pud_size + pmd_size + pte_size;
+
+ /*
+ * Now calculate the added pagetable structures needed to populate
+@@ -458,10 +546,13 @@ void __init sme_encrypt_kernel(void)
+
+ /*
+ * The total workarea includes the executable encryption area and
+- * the pagetable area.
++ * the pagetable area. The start of the workarea is already 2MB
++ * aligned, align the end of the workarea on a 2MB boundary so that
++ * we don't try to create/allocate PTE entries from the workarea
++ * before it is mapped.
+ */
+ workarea_len = execute_len + pgtable_area_len;
+- workarea_end = workarea_start + workarea_len;
++ workarea_end = ALIGN(workarea_start + workarea_len, PMD_PAGE_SIZE);
+
+ /*
+ * Set the address to the start of where newly created pagetable
+--- a/arch/x86/mm/mem_encrypt_boot.S
++++ b/arch/x86/mm/mem_encrypt_boot.S
+@@ -104,6 +104,7 @@ ENTRY(__enc_copy)
+ mov %rdx, %cr4
+
+ push %r15
++ push %r12
+
+ movq %rcx, %r9 /* Save kernel length */
+ movq %rdi, %r10 /* Save encrypted kernel address */
+@@ -119,21 +120,27 @@ ENTRY(__enc_copy)
+
+ wbinvd /* Invalidate any cache entries */
+
+- /* Copy/encrypt 2MB at a time */
++ /* Copy/encrypt up to 2MB at a time */
++ movq $PMD_PAGE_SIZE, %r12
+ 1:
++ cmpq %r12, %r9
++ jnb 2f
++ movq %r9, %r12
++
++2:
+ movq %r11, %rsi /* Source - decrypted kernel */
+ movq %r8, %rdi /* Dest - intermediate copy buffer */
+- movq $PMD_PAGE_SIZE, %rcx /* 2MB length */
++ movq %r12, %rcx
+ rep movsb
+
+ movq %r8, %rsi /* Source - intermediate copy buffer */
+ movq %r10, %rdi /* Dest - encrypted kernel */
+- movq $PMD_PAGE_SIZE, %rcx /* 2MB length */
++ movq %r12, %rcx
+ rep movsb
+
+- addq $PMD_PAGE_SIZE, %r11
+- addq $PMD_PAGE_SIZE, %r10
+- subq $PMD_PAGE_SIZE, %r9 /* Kernel length decrement */
++ addq %r12, %r11
++ addq %r12, %r10
++ subq %r12, %r9 /* Kernel length decrement */
+ jnz 1b /* Kernel length not zero? */
+
+ /* Restore PAT register */
+@@ -142,6 +149,7 @@ ENTRY(__enc_copy)
+ mov %r15, %rdx /* Restore original PAT value */
+ wrmsr
+
++ pop %r12
+ pop %r15
+
+ ret
--- /dev/null
+From bacf6b499e11760aef73a3bb5ce4e5eea74a3fd4 Mon Sep 17 00:00:00 2001
+From: Tom Lendacky <thomas.lendacky@amd.com>
+Date: Wed, 10 Jan 2018 13:26:05 -0600
+Subject: x86/mm: Use a struct to reduce parameters for SME PGD mapping
+
+From: Tom Lendacky <thomas.lendacky@amd.com>
+
+commit bacf6b499e11760aef73a3bb5ce4e5eea74a3fd4 upstream.
+
+In preparation for follow-on patches, combine the PGD mapping parameters
+into a struct to reduce the number of function arguments and allow for
+direct updating of the next pagetable mapping area pointer.
+
+Tested-by: Gabriel Craciunescu <nix.or.die@gmail.com>
+Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
+Reviewed-by: Borislav Petkov <bp@suse.de>
+Cc: Borislav Petkov <bp@alien8.de>
+Cc: Brijesh Singh <brijesh.singh@amd.com>
+Cc: Linus Torvalds <torvalds@linux-foundation.org>
+Cc: Peter Zijlstra <peterz@infradead.org>
+Cc: Thomas Gleixner <tglx@linutronix.de>
+Link: http://lkml.kernel.org/r/20180110192605.6026.96206.stgit@tlendack-t1.amdoffice.net
+Signed-off-by: Ingo Molnar <mingo@kernel.org>
+Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
+
+---
+ arch/x86/mm/mem_encrypt.c | 92 +++++++++++++++++++++++-----------------------
+ 1 file changed, 47 insertions(+), 45 deletions(-)
+
+--- a/arch/x86/mm/mem_encrypt.c
++++ b/arch/x86/mm/mem_encrypt.c
+@@ -213,6 +213,14 @@ void swiotlb_set_mem_attributes(void *va
+ set_memory_decrypted((unsigned long)vaddr, size >> PAGE_SHIFT);
+ }
+
++struct sme_populate_pgd_data {
++ void *pgtable_area;
++ pgd_t *pgd;
++
++ pmdval_t pmd_val;
++ unsigned long vaddr;
++};
++
+ static void __init sme_clear_pgd(pgd_t *pgd_base, unsigned long start,
+ unsigned long end)
+ {
+@@ -235,15 +243,14 @@ static void __init sme_clear_pgd(pgd_t *
+ #define PUD_FLAGS _KERNPG_TABLE_NOENC
+ #define PMD_FLAGS (__PAGE_KERNEL_LARGE_EXEC & ~_PAGE_GLOBAL)
+
+-static void __init *sme_populate_pgd(pgd_t *pgd_base, void *pgtable_area,
+- unsigned long vaddr, pmdval_t pmd_val)
++static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
+ {
+ pgd_t *pgd_p;
+ p4d_t *p4d_p;
+ pud_t *pud_p;
+ pmd_t *pmd_p;
+
+- pgd_p = pgd_base + pgd_index(vaddr);
++ pgd_p = ppd->pgd + pgd_index(ppd->vaddr);
+ if (native_pgd_val(*pgd_p)) {
+ if (IS_ENABLED(CONFIG_X86_5LEVEL))
+ p4d_p = (p4d_t *)(native_pgd_val(*pgd_p) & ~PTE_FLAGS_MASK);
+@@ -253,15 +260,15 @@ static void __init *sme_populate_pgd(pgd
+ pgd_t pgd;
+
+ if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+- p4d_p = pgtable_area;
++ p4d_p = ppd->pgtable_area;
+ memset(p4d_p, 0, sizeof(*p4d_p) * PTRS_PER_P4D);
+- pgtable_area += sizeof(*p4d_p) * PTRS_PER_P4D;
++ ppd->pgtable_area += sizeof(*p4d_p) * PTRS_PER_P4D;
+
+ pgd = native_make_pgd((pgdval_t)p4d_p + PGD_FLAGS);
+ } else {
+- pud_p = pgtable_area;
++ pud_p = ppd->pgtable_area;
+ memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
+- pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD;
++ ppd->pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD;
+
+ pgd = native_make_pgd((pgdval_t)pud_p + PGD_FLAGS);
+ }
+@@ -269,44 +276,41 @@ static void __init *sme_populate_pgd(pgd
+ }
+
+ if (IS_ENABLED(CONFIG_X86_5LEVEL)) {
+- p4d_p += p4d_index(vaddr);
++ p4d_p += p4d_index(ppd->vaddr);
+ if (native_p4d_val(*p4d_p)) {
+ pud_p = (pud_t *)(native_p4d_val(*p4d_p) & ~PTE_FLAGS_MASK);
+ } else {
+ p4d_t p4d;
+
+- pud_p = pgtable_area;
++ pud_p = ppd->pgtable_area;
+ memset(pud_p, 0, sizeof(*pud_p) * PTRS_PER_PUD);
+- pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD;
++ ppd->pgtable_area += sizeof(*pud_p) * PTRS_PER_PUD;
+
+ p4d = native_make_p4d((pudval_t)pud_p + P4D_FLAGS);
+ native_set_p4d(p4d_p, p4d);
+ }
+ }
+
+- pud_p += pud_index(vaddr);
++ pud_p += pud_index(ppd->vaddr);
+ if (native_pud_val(*pud_p)) {
+ if (native_pud_val(*pud_p) & _PAGE_PSE)
+- goto out;
++ return;
+
+ pmd_p = (pmd_t *)(native_pud_val(*pud_p) & ~PTE_FLAGS_MASK);
+ } else {
+ pud_t pud;
+
+- pmd_p = pgtable_area;
++ pmd_p = ppd->pgtable_area;
+ memset(pmd_p, 0, sizeof(*pmd_p) * PTRS_PER_PMD);
+- pgtable_area += sizeof(*pmd_p) * PTRS_PER_PMD;
++ ppd->pgtable_area += sizeof(*pmd_p) * PTRS_PER_PMD;
+
+ pud = native_make_pud((pmdval_t)pmd_p + PUD_FLAGS);
+ native_set_pud(pud_p, pud);
+ }
+
+- pmd_p += pmd_index(vaddr);
++ pmd_p += pmd_index(ppd->vaddr);
+ if (!native_pmd_val(*pmd_p) || !(native_pmd_val(*pmd_p) & _PAGE_PSE))
+- native_set_pmd(pmd_p, native_make_pmd(pmd_val));
+-
+-out:
+- return pgtable_area;
++ native_set_pmd(pmd_p, native_make_pmd(ppd->pmd_val));
+ }
+
+ static unsigned long __init sme_pgtable_calc(unsigned long len)
+@@ -364,11 +368,10 @@ void __init sme_encrypt_kernel(void)
+ unsigned long workarea_start, workarea_end, workarea_len;
+ unsigned long execute_start, execute_end, execute_len;
+ unsigned long kernel_start, kernel_end, kernel_len;
++ struct sme_populate_pgd_data ppd;
+ unsigned long pgtable_area_len;
+ unsigned long paddr, pmd_flags;
+ unsigned long decrypted_base;
+- void *pgtable_area;
+- pgd_t *pgd;
+
+ if (!sme_active())
+ return;
+@@ -432,18 +435,18 @@ void __init sme_encrypt_kernel(void)
+ * pagetables and when the new encrypted and decrypted kernel
+ * mappings are populated.
+ */
+- pgtable_area = (void *)execute_end;
++ ppd.pgtable_area = (void *)execute_end;
+
+ /*
+ * Make sure the current pagetable structure has entries for
+ * addressing the workarea.
+ */
+- pgd = (pgd_t *)native_read_cr3_pa();
++ ppd.pgd = (pgd_t *)native_read_cr3_pa();
+ paddr = workarea_start;
+ while (paddr < workarea_end) {
+- pgtable_area = sme_populate_pgd(pgd, pgtable_area,
+- paddr,
+- paddr + PMD_FLAGS);
++ ppd.pmd_val = paddr + PMD_FLAGS;
++ ppd.vaddr = paddr;
++ sme_populate_pgd_large(&ppd);
+
+ paddr += PMD_PAGE_SIZE;
+ }
+@@ -457,17 +460,17 @@ void __init sme_encrypt_kernel(void)
+ * populated with new PUDs and PMDs as the encrypted and decrypted
+ * kernel mappings are created.
+ */
+- pgd = pgtable_area;
+- memset(pgd, 0, sizeof(*pgd) * PTRS_PER_PGD);
+- pgtable_area += sizeof(*pgd) * PTRS_PER_PGD;
++ ppd.pgd = ppd.pgtable_area;
++ memset(ppd.pgd, 0, sizeof(pgd_t) * PTRS_PER_PGD);
++ ppd.pgtable_area += sizeof(pgd_t) * PTRS_PER_PGD;
+
+ /* Add encrypted kernel (identity) mappings */
+ pmd_flags = PMD_FLAGS | _PAGE_ENC;
+ paddr = kernel_start;
+ while (paddr < kernel_end) {
+- pgtable_area = sme_populate_pgd(pgd, pgtable_area,
+- paddr,
+- paddr + pmd_flags);
++ ppd.pmd_val = paddr + pmd_flags;
++ ppd.vaddr = paddr;
++ sme_populate_pgd_large(&ppd);
+
+ paddr += PMD_PAGE_SIZE;
+ }
+@@ -485,9 +488,9 @@ void __init sme_encrypt_kernel(void)
+ pmd_flags = (PMD_FLAGS & ~_PAGE_CACHE_MASK) | (_PAGE_PAT | _PAGE_PWT);
+ paddr = kernel_start;
+ while (paddr < kernel_end) {
+- pgtable_area = sme_populate_pgd(pgd, pgtable_area,
+- paddr + decrypted_base,
+- paddr + pmd_flags);
++ ppd.pmd_val = paddr + pmd_flags;
++ ppd.vaddr = paddr + decrypted_base;
++ sme_populate_pgd_large(&ppd);
+
+ paddr += PMD_PAGE_SIZE;
+ }
+@@ -495,30 +498,29 @@ void __init sme_encrypt_kernel(void)
+ /* Add decrypted workarea mappings to both kernel mappings */
+ paddr = workarea_start;
+ while (paddr < workarea_end) {
+- pgtable_area = sme_populate_pgd(pgd, pgtable_area,
+- paddr,
+- paddr + PMD_FLAGS);
+-
+- pgtable_area = sme_populate_pgd(pgd, pgtable_area,
+- paddr + decrypted_base,
+- paddr + PMD_FLAGS);
++ ppd.pmd_val = paddr + PMD_FLAGS;
++ ppd.vaddr = paddr;
++ sme_populate_pgd_large(&ppd);
++
++ ppd.vaddr = paddr + decrypted_base;
++ sme_populate_pgd_large(&ppd);
+
+ paddr += PMD_PAGE_SIZE;
+ }
+
+ /* Perform the encryption */
+ sme_encrypt_execute(kernel_start, kernel_start + decrypted_base,
+- kernel_len, workarea_start, (unsigned long)pgd);
++ kernel_len, workarea_start, (unsigned long)ppd.pgd);
+
+ /*
+ * At this point we are running encrypted. Remove the mappings for
+ * the decrypted areas - all that is needed for this is to remove
+ * the PGD entry/entries.
+ */
+- sme_clear_pgd(pgd, kernel_start + decrypted_base,
++ sme_clear_pgd(ppd.pgd, kernel_start + decrypted_base,
+ kernel_end + decrypted_base);
+
+- sme_clear_pgd(pgd, workarea_start + decrypted_base,
++ sme_clear_pgd(ppd.pgd, workarea_start + decrypted_base,
+ workarea_end + decrypted_base);
+
+ /* Flush the TLB - no globals so cr3 is enough */
projects / thirdparty / kernel / stable-queue.git / commitdiff
