x86/vmscape: Add conditional IBPB mitigation

commit 2f8f173413f1cbf52660d04df92d0069c4306d25 upstream.

VMSCAPE is a vulnerability that exploits insufficient branch predictor
isolation between a guest and a userspace hypervisor (like QEMU). Existing
mitigations already protect kernel/KVM from a malicious guest. Userspace
can additionally be protected by flushing the branch predictors after a
VMexit.

Since it is the userspace that consumes the poisoned branch predictors,
conditionally issue an IBPB after a VMexit and before returning to
userspace. Workloads that frequently switch between hypervisor and
userspace will incur the most overhead from the new IBPB.

This new IBPB is not integrated with the existing IBPB sites. For
instance, a task can use the existing speculation control prctl() to
get an IBPB at context switch time. With this implementation, the
IBPB is doubled up: one at context switch and another before running
userspace.

The intent is to integrate and optimize these cases post-embargo.

[ dhansen: elaborate on suboptimal IBPB solution ]

Suggested-by: Dave Hansen <dave.hansen@linux.intel.com>
Signed-off-by: Pawan Gupta <pawan.kumar.gupta@linux.intel.com>
Signed-off-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Dave Hansen <dave.hansen@linux.intel.com>
Reviewed-by: Borislav Petkov (AMD) <bp@alien8.de>
Acked-by: Sean Christopherson <seanjc@google.com>
[Amit: Use CPUID word 22 instead of 21 for the 5.10 backport]
Signed-off-by: Amit Shah <amit.shah@amd.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index b19bda7d1547a927ed698b489c2d88ee8053c015..79017db18b02b6c4be4b5037c1196e3b10a860cb 100644 (file)
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -415,6 +415,7 @@
 #define X86_FEATURE_TSA_SQ_NO          (22*32+11) /* "" AMD CPU not vulnerable to TSA-SQ */
 #define X86_FEATURE_TSA_L1_NO          (22*32+12) /* "" AMD CPU not vulnerable to TSA-L1 */
 #define X86_FEATURE_CLEAR_CPU_BUF_VM   (22*32+13) /* "" Clear CPU buffers using VERW before VMRUN */
+#define X86_FEATURE_IBPB_EXIT_TO_USER  (22*32+14) /* Use IBPB on exit-to-userspace, see VMSCAPE bug */
 
 /*
  * BUG word(s)

diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index 264ab414e9f632d3d37f8b336df2aa34762dc5fb..b513cb76380a4b2e22b137144a4194b7b441f310 100644 (file)
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -72,6 +72,13 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
         */
        current_thread_info()->status &= ~(TS_COMPAT | TS_I386_REGS_POKED);
 #endif
+
+       /* Avoid unnecessary reads of 'x86_ibpb_exit_to_user' */
+       if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER) &&
+           this_cpu_read(x86_ibpb_exit_to_user)) {
+               indirect_branch_prediction_barrier();
+               this_cpu_write(x86_ibpb_exit_to_user, false);
+       }
 }
 #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
 

diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h
index ce5e6e70d2a48f334e681743dd986e531037d880..c49f42c8e5e1885c19ad76e41e444187d8e9701c 100644 (file)
--- a/arch/x86/include/asm/nospec-branch.h
+++ b/arch/x86/include/asm/nospec-branch.h
@@ -365,6 +365,8 @@ void alternative_msr_write(unsigned int msr, u64 val, unsigned int feature)
 
 extern u64 x86_pred_cmd;
 
+DECLARE_PER_CPU(bool, x86_ibpb_exit_to_user);
+
 static inline void indirect_branch_prediction_barrier(void)
 {
        alternative_msr_write(MSR_IA32_PRED_CMD, x86_pred_cmd, X86_FEATURE_USE_IBPB);

diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index 6ff9fd836d874a2ff8090488ddf078983f9bebb1..423e7c67e20be45bee626e1372464ade75e2e973 100644 (file)
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -58,6 +58,14 @@ EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
 DEFINE_PER_CPU(u64, x86_spec_ctrl_current);
 EXPORT_SYMBOL_GPL(x86_spec_ctrl_current);
 
+/*
+ * Set when the CPU has run a potentially malicious guest. An IBPB will
+ * be needed to before running userspace. That IBPB will flush the branch
+ * predictor content.
+ */
+DEFINE_PER_CPU(bool, x86_ibpb_exit_to_user);
+EXPORT_PER_CPU_SYMBOL_GPL(x86_ibpb_exit_to_user);
+
 u64 x86_pred_cmd __ro_after_init = PRED_CMD_IBPB;
 EXPORT_SYMBOL_GPL(x86_pred_cmd);
 

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 8952f3567b69acad3d624560122e4f63fcfdae29..4150dfe421c1fb3a558f95a00c472d8a1cb20aeb 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -9329,6 +9329,15 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 
        kvm_x86_ops.handle_exit_irqoff(vcpu);
 
+       /*
+        * Mark this CPU as needing a branch predictor flush before running
+        * userspace. Must be done before enabling preemption to ensure it gets
+        * set for the CPU that actually ran the guest, and not the CPU that it
+        * may migrate to.
+        */
+       if (cpu_feature_enabled(X86_FEATURE_IBPB_EXIT_TO_USER))
+               this_cpu_write(x86_ibpb_exit_to_user, true);
+
        /*
         * Consume any pending interrupts, including the possible source of
         * VM-Exit on SVM and any ticks that occur between VM-Exit and now.