x86/sev: Evict cache lines during SNP memory validation

Commit 7b306dfa326f70114312b320d083b21fa9481e1e upstream.

An SNP cache coherency vulnerability requires a cache line eviction
mitigation when validating memory after a page state change to private.
The specific mitigation is to touch the first and last byte of each 4K
page that is being validated. There is no need to perform the mitigation
when performing a page state change to shared and rescinding validation.

CPUID bit Fn8000001F_EBX[31] defines the COHERENCY_SFW_NO CPUID bit that,
when set, indicates that the software mitigation for this vulnerability is
not needed.

Implement the mitigation and invoke it when validating memory (making it
private) and the COHERENCY_SFW_NO bit is not set, indicating the SNP guest
is vulnerable.

Co-developed-by: Michael Roth <michael.roth@amd.com>
Signed-off-by: Michael Roth <michael.roth@amd.com>
Signed-off-by: Tom Lendacky <thomas.lendacky@amd.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/boot/cpuflags.c b/arch/x86/boot/cpuflags.c
index d75237ba7ce94415de457a38d41c38df69c0c8a7..5660d3229d29c2de1fd87c1486b9f6de235667bd 100644 (file)
--- a/arch/x86/boot/cpuflags.c
+++ b/arch/x86/boot/cpuflags.c
@@ -115,5 +115,18 @@ void get_cpuflags(void)
                        cpuid(0x80000001, &ignored, &ignored, &cpu.flags[6],
                              &cpu.flags[1]);
                }
+
+               if (max_amd_level >= 0x8000001f) {
+                       u32 ebx;
+
+                       /*
+                        * The X86_FEATURE_COHERENCY_SFW_NO feature bit is in
+                        * the virtualization flags entry (word 8) and set by
+                        * scattered.c, so the bit needs to be explicitly set.
+                        */
+                       cpuid(0x8000001f, &ignored, &ebx, &ignored, &ignored);
+                       if (ebx & BIT(31))
+                               set_bit(X86_FEATURE_COHERENCY_SFW_NO, cpu.flags);
+               }
        }
 }

diff --git a/arch/x86/coco/sev/shared.c b/arch/x86/coco/sev/shared.c
index 71de5319408910b251930eb1fb3598b3fb605d93..f5936da235c71352d0a70912ec2b66bc45fd4827 100644 (file)
--- a/arch/x86/coco/sev/shared.c
+++ b/arch/x86/coco/sev/shared.c
@@ -1243,6 +1243,24 @@ static void svsm_pval_terminate(struct svsm_pvalidate_call *pc, int ret, u64 svs
        __pval_terminate(pfn, action, page_size, ret, svsm_ret);
 }
 
+static inline void sev_evict_cache(void *va, int npages)
+{
+       volatile u8 val __always_unused;
+       u8 *bytes = va;
+       int page_idx;
+
+       /*
+        * For SEV guests, a read from the first/last cache-lines of a 4K page
+        * using the guest key is sufficient to cause a flush of all cache-lines
+        * associated with that 4K page without incurring all the overhead of a
+        * full CLFLUSH sequence.
+        */
+       for (page_idx = 0; page_idx < npages; page_idx++) {
+               val = bytes[page_idx * PAGE_SIZE];
+               val = bytes[page_idx * PAGE_SIZE + PAGE_SIZE - 1];
+       }
+}
+
 static void svsm_pval_4k_page(unsigned long paddr, bool validate)
 {
        struct svsm_pvalidate_call *pc;
@@ -1295,6 +1313,13 @@ static void pvalidate_4k_page(unsigned long vaddr, unsigned long paddr, bool val
                if (ret)
                        __pval_terminate(PHYS_PFN(paddr), validate, RMP_PG_SIZE_4K, ret, 0);
        }
+
+       /*
+        * If validating memory (making it private) and affected by the
+        * cache-coherency vulnerability, perform the cache eviction mitigation.
+        */
+       if (validate && !has_cpuflag(X86_FEATURE_COHERENCY_SFW_NO))
+               sev_evict_cache((void *)vaddr, 1);
 }
 
 static void pval_pages(struct snp_psc_desc *desc)
@@ -1479,10 +1504,31 @@ static void svsm_pval_pages(struct snp_psc_desc *desc)
 
 static void pvalidate_pages(struct snp_psc_desc *desc)
 {
+       struct psc_entry *e;
+       unsigned int i;
+
        if (snp_vmpl)
                svsm_pval_pages(desc);
        else
                pval_pages(desc);
+
+       /*
+        * If not affected by the cache-coherency vulnerability there is no need
+        * to perform the cache eviction mitigation.
+        */
+       if (cpu_feature_enabled(X86_FEATURE_COHERENCY_SFW_NO))
+               return;
+
+       for (i = 0; i <= desc->hdr.end_entry; i++) {
+               e = &desc->entries[i];
+
+               /*
+                * If validating memory (making it private) perform the cache
+                * eviction mitigation.
+                */
+               if (e->operation == SNP_PAGE_STATE_PRIVATE)
+                       sev_evict_cache(pfn_to_kaddr(e->gfn), e->pagesize ? 512 : 1);
+       }
 }
 
 static int vmgexit_psc(struct ghcb *ghcb, struct snp_psc_desc *desc)

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index ef5749a0d8c24d116f314d12d57209b52e223769..98e72c1391f24054246b657b05364c7b0bc2e812 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -227,6 +227,7 @@
 #define X86_FEATURE_FLEXPRIORITY       ( 8*32+ 1) /* "flexpriority" Intel FlexPriority */
 #define X86_FEATURE_EPT                        ( 8*32+ 2) /* "ept" Intel Extended Page Table */
 #define X86_FEATURE_VPID               ( 8*32+ 3) /* "vpid" Intel Virtual Processor ID */
+#define X86_FEATURE_COHERENCY_SFW_NO   ( 8*32+ 4) /* SNP cache coherency software work around not needed */
 
 #define X86_FEATURE_VMMCALL            ( 8*32+15) /* "vmmcall" Prefer VMMCALL to VMCALL */
 #define X86_FEATURE_XENPV              ( 8*32+16) /* Xen paravirtual guest */

diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index bc4993aa41edf2033fb1a7eb50c528e1a6e5a81b..c463363ae1d49ef15ff8127561148226ae3d4121 100644 (file)
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -47,6 +47,7 @@ static const struct cpuid_bit cpuid_bits[] = {
        { X86_FEATURE_PROC_FEEDBACK,    CPUID_EDX, 11, 0x80000007, 0 },
        { X86_FEATURE_FAST_CPPC,        CPUID_EDX, 15, 0x80000007, 0 },
        { X86_FEATURE_MBA,              CPUID_EBX,  6, 0x80000008, 0 },
+       { X86_FEATURE_COHERENCY_SFW_NO, CPUID_EBX, 31, 0x8000001f, 0 },
        { X86_FEATURE_SMBA,             CPUID_EBX,  2, 0x80000020, 0 },
        { X86_FEATURE_BMEC,             CPUID_EBX,  3, 0x80000020, 0 },
        { X86_FEATURE_TSA_SQ_NO,        CPUID_ECX,  1, 0x80000021, 0 },