x86/coco: Add API to handle encryption mask

AMD SME/SEV uses a bit in the page table entries to indicate that the
page is encrypted and not accessible to the VMM.

TDX uses a similar approach, but the polarity of the mask is opposite to
AMD: if the bit is set the page is accessible to VMM.

Provide vendor-neutral API to deal with the mask: cc_mkenc() and
cc_mkdec() modify given address to make it encrypted/decrypted. It can
be applied to phys_addr_t, pgprotval_t or page table entry value.

pgprot_encrypted() and pgprot_decrypted() reimplemented using new
helpers.

The implementation will be extended to cover TDX.

pgprot_decrypted() is used by drivers (i915, virtio_gpu, vfio).
cc_mkdec() called by pgprot_decrypted(). Export cc_mkdec().

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
Link: https://lore.kernel.org/r/20220222185740.26228-5-kirill.shutemov@linux.intel.com

diff --git a/arch/x86/coco/core.c b/arch/x86/coco/core.c
index 476dcd198af5ff42191da48897ca05ce32c4c8c3..fc1365dd927e8001971bcb86a2a8a41bb89ef33e 100644 (file)
--- a/arch/x86/coco/core.c
+++ b/arch/x86/coco/core.c
@@ -14,6 +14,7 @@
 #include <asm/processor.h>
 
 static enum cc_vendor vendor __ro_after_init;
+static u64 cc_mask __ro_after_init;
 
 static bool intel_cc_platform_has(enum cc_attr attr)
 {
@@ -84,7 +85,33 @@ bool cc_platform_has(enum cc_attr attr)
 }
 EXPORT_SYMBOL_GPL(cc_platform_has);
 
+u64 cc_mkenc(u64 val)
+{
+       switch (vendor) {
+       case CC_VENDOR_AMD:
+               return val | cc_mask;
+       default:
+               return val;
+       }
+}
+
+u64 cc_mkdec(u64 val)
+{
+       switch (vendor) {
+       case CC_VENDOR_AMD:
+               return val & ~cc_mask;
+       default:
+               return val;
+       }
+}
+EXPORT_SYMBOL_GPL(cc_mkdec);
+
 __init void cc_set_vendor(enum cc_vendor v)
 {
        vendor = v;
 }
+
+__init void cc_set_mask(u64 mask)
+{
+       cc_mask = mask;
+}

diff --git a/arch/x86/include/asm/coco.h b/arch/x86/include/asm/coco.h
index e49f9ddb6ae62149a5924812d144b7659bec6302..3d98c3a60d34f97fc6403a7248b9c06c03b63f8d 100644 (file)
--- a/arch/x86/include/asm/coco.h
+++ b/arch/x86/include/asm/coco.h
@@ -2,6 +2,8 @@
 #ifndef _ASM_X86_COCO_H
 #define _ASM_X86_COCO_H
 
+#include <asm/types.h>
+
 enum cc_vendor {
        CC_VENDOR_NONE,
        CC_VENDOR_AMD,
@@ -10,5 +12,21 @@ enum cc_vendor {
 };
 
 void cc_set_vendor(enum cc_vendor v);
+void cc_set_mask(u64 mask);
+
+#ifdef CONFIG_ARCH_HAS_CC_PLATFORM
+u64 cc_mkenc(u64 val);
+u64 cc_mkdec(u64 val);
+#else
+static inline u64 cc_mkenc(u64 val)
+{
+       return val;
+}
+
+static inline u64 cc_mkdec(u64 val)
+{
+       return val;
+}
+#endif
 
 #endif /* _ASM_X86_COCO_H */

diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 8a9432fb3802b3f60e12331b23e43b78f48cddc8..62ab07e24aef93093d0bfa7793c78c672a2759eb 100644 (file)
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -15,17 +15,12 @@
                     cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))     \
         : (prot))
 
-/*
- * Macros to add or remove encryption attribute
- */
-#define pgprot_encrypted(prot) __pgprot(__sme_set(pgprot_val(prot)))
-#define pgprot_decrypted(prot) __pgprot(__sme_clr(pgprot_val(prot)))
-
 #ifndef __ASSEMBLY__
 #include <linux/spinlock.h>
 #include <asm/x86_init.h>
 #include <asm/pkru.h>
 #include <asm/fpu/api.h>
+#include <asm/coco.h>
 #include <asm-generic/pgtable_uffd.h>
 #include <linux/page_table_check.h>
 
@@ -38,6 +33,12 @@ void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
 void ptdump_walk_pgd_level_checkwx(void);
 void ptdump_walk_user_pgd_level_checkwx(void);
 
+/*
+ * Macros to add or remove encryption attribute
+ */
+#define pgprot_encrypted(prot) __pgprot(cc_mkenc(pgprot_val(prot)))
+#define pgprot_decrypted(prot) __pgprot(cc_mkdec(pgprot_val(prot)))
+
 #ifdef CONFIG_DEBUG_WX
 #define debug_checkwx()                ptdump_walk_pgd_level_checkwx()
 #define debug_checkwx_user()   ptdump_walk_user_pgd_level_checkwx()

diff --git a/arch/x86/mm/mem_encrypt_identity.c b/arch/x86/mm/mem_encrypt_identity.c
index 06314ae3998e5cc9ebf5df5a9aff4271fb68e6d0..b43bc24d2bb6415e4fe5eb7a4bec1914d5767fb1 100644 (file)
--- a/arch/x86/mm/mem_encrypt_identity.c
+++ b/arch/x86/mm/mem_encrypt_identity.c
@@ -604,5 +604,6 @@ out:
        if (sme_me_mask) {
                physical_mask &= ~sme_me_mask;
                cc_set_vendor(CC_VENDOR_AMD);
+               cc_set_mask(sme_me_mask);
        }
 }

diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index b4072115c8ef685b307a2b4a5fe4fe0c6331caec..1441db69cea5ded57895cfeb98e475903c13b276 100644 (file)
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -1989,6 +1989,7 @@ int set_memory_global(unsigned long addr, int numpages)
  */
 static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc)
 {
+       pgprot_t empty = __pgprot(0);
        struct cpa_data cpa;
        int ret;
 
@@ -1999,8 +2000,8 @@ static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc)
        memset(&cpa, 0, sizeof(cpa));
        cpa.vaddr = &addr;
        cpa.numpages = numpages;
-       cpa.mask_set = enc ? __pgprot(_PAGE_ENC) : __pgprot(0);
-       cpa.mask_clr = enc ? __pgprot(0) : __pgprot(_PAGE_ENC);
+       cpa.mask_set = enc ? pgprot_encrypted(empty) : pgprot_decrypted(empty);
+       cpa.mask_clr = enc ? pgprot_decrypted(empty) : pgprot_encrypted(empty);
        cpa.pgd = init_mm.pgd;
 
        /* Must avoid aliasing mappings in the highmem code */