x86/mm: Refactor __set_clr_pte_enc()

Refactor __set_clr_pte_enc() and add two new helper functions to
set/clear PTE C-bit from early SEV/SNP initialization code and later
during shutdown/kexec especially when all CPUs are stopped and
interrupts are disabled and set_memory_xx() interfaces can't be used.

Co-developed-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Ashish Kalra <ashish.kalra@amd.com>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/5df4aa450447f28294d1c5a890e27b63ed4ded36.1722520012.git.ashish.kalra@amd.com

diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 2e49c4a9e7fe7aac27ef310db4d5f1b8f9a6c9e5..5f598937f090d3afc36f720f1390c4e99a048cd0 100644 (file)
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -322,6 +322,22 @@ struct svsm_attest_call {
        u8 rsvd[4];
 };
 
+/* PTE descriptor used for the prepare_pte_enc() operations. */
+struct pte_enc_desc {
+       pte_t *kpte;
+       int pte_level;
+       bool encrypt;
+       /* pfn of the kpte above */
+       unsigned long pfn;
+       /* physical address of @pfn */
+       unsigned long pa;
+       /* virtual address of @pfn */
+       void *va;
+       /* memory covered by the pte */
+       unsigned long size;
+       pgprot_t new_pgprot;
+};
+
 /*
  * SVSM protocol structure
  */
@@ -437,6 +453,8 @@ u64 snp_get_unsupported_features(u64 status);
 u64 sev_get_status(void);
 void sev_show_status(void);
 void snp_update_svsm_ca(void);
+int prepare_pte_enc(struct pte_enc_desc *d);
+void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot);
 
 #else  /* !CONFIG_AMD_MEM_ENCRYPT */
 
@@ -474,6 +492,8 @@ static inline u64 snp_get_unsupported_features(u64 status) { return 0; }
 static inline u64 sev_get_status(void) { return 0; }
 static inline void sev_show_status(void) { }
 static inline void snp_update_svsm_ca(void) { }
+static inline int prepare_pte_enc(struct pte_enc_desc *d) { return 0; }
+static inline void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot) { }
 
 #endif /* CONFIG_AMD_MEM_ENCRYPT */
 

diff --git a/arch/x86/mm/mem_encrypt_amd.c b/arch/x86/mm/mem_encrypt_amd.c
index 86a476a426c26dd5e49ea3c102ab4a98636bf170..f4be81db72eecce7e06608a6478bd98688975ba3 100644 (file)
--- a/arch/x86/mm/mem_encrypt_amd.c
+++ b/arch/x86/mm/mem_encrypt_amd.c
@@ -311,59 +311,82 @@ static int amd_enc_status_change_finish(unsigned long vaddr, int npages, bool en
        return 0;
 }
 
-static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
+int prepare_pte_enc(struct pte_enc_desc *d)
 {
-       pgprot_t old_prot, new_prot;
-       unsigned long pfn, pa, size;
-       pte_t new_pte;
+       pgprot_t old_prot;
 
-       pfn = pg_level_to_pfn(level, kpte, &old_prot);
-       if (!pfn)
-               return;
+       d->pfn = pg_level_to_pfn(d->pte_level, d->kpte, &old_prot);
+       if (!d->pfn)
+               return 1;
 
-       new_prot = old_prot;
-       if (enc)
-               pgprot_val(new_prot) |= _PAGE_ENC;
+       d->new_pgprot = old_prot;
+       if (d->encrypt)
+               pgprot_val(d->new_pgprot) |= _PAGE_ENC;
        else
-               pgprot_val(new_prot) &= ~_PAGE_ENC;
+               pgprot_val(d->new_pgprot) &= ~_PAGE_ENC;
 
        /* If prot is same then do nothing. */
-       if (pgprot_val(old_prot) == pgprot_val(new_prot))
-               return;
+       if (pgprot_val(old_prot) == pgprot_val(d->new_pgprot))
+               return 1;
 
-       pa = pfn << PAGE_SHIFT;
-       size = page_level_size(level);
+       d->pa = d->pfn << PAGE_SHIFT;
+       d->size = page_level_size(d->pte_level);
 
        /*
-        * We are going to perform in-place en-/decryption and change the
-        * physical page attribute from C=1 to C=0 or vice versa. Flush the
-        * caches to ensure that data gets accessed with the correct C-bit.
+        * In-place en-/decryption and physical page attribute change
+        * from C=1 to C=0 or vice versa will be performed. Flush the
+        * caches to ensure that data gets accessed with the correct
+        * C-bit.
         */
-       clflush_cache_range(__va(pa), size);
+       if (d->va)
+               clflush_cache_range(d->va, d->size);
+       else
+               clflush_cache_range(__va(d->pa), d->size);
+
+       return 0;
+}
+
+void set_pte_enc_mask(pte_t *kpte, unsigned long pfn, pgprot_t new_prot)
+{
+       pte_t new_pte;
+
+       /* Change the page encryption mask. */
+       new_pte = pfn_pte(pfn, new_prot);
+       set_pte_atomic(kpte, new_pte);
+}
+
+static void __init __set_clr_pte_enc(pte_t *kpte, int level, bool enc)
+{
+       struct pte_enc_desc d = {
+               .kpte        = kpte,
+               .pte_level   = level,
+               .encrypt     = enc
+       };
+
+       if (prepare_pte_enc(&d))
+               return;
 
        /* Encrypt/decrypt the contents in-place */
        if (enc) {
-               sme_early_encrypt(pa, size);
+               sme_early_encrypt(d.pa, d.size);
        } else {
-               sme_early_decrypt(pa, size);
+               sme_early_decrypt(d.pa, d.size);
 
                /*
                 * ON SNP, the page state in the RMP table must happen
                 * before the page table updates.
                 */
-               early_snp_set_memory_shared((unsigned long)__va(pa), pa, 1);
+               early_snp_set_memory_shared((unsigned long)__va(d.pa), d.pa, 1);
        }
 
-       /* Change the page encryption mask. */
-       new_pte = pfn_pte(pfn, new_prot);
-       set_pte_atomic(kpte, new_pte);
+       set_pte_enc_mask(kpte, d.pfn, d.new_pgprot);
 
        /*
         * If page is set encrypted in the page table, then update the RMP table to
         * add this page as private.
         */
        if (enc)
-               early_snp_set_memory_private((unsigned long)__va(pa), pa, 1);
+               early_snp_set_memory_private((unsigned long)__va(d.pa), d.pa, 1);
 }
 
 static int __init early_set_memory_enc_dec(unsigned long vaddr,