KVM: VMX: Move core VMXON enablement to kernel

Move the innermost VMXON+VMXOFF logic out of KVM and into to core x86 so
that TDX can (eventually) force VMXON without having to rely on KVM being
loaded, e.g. to do SEAMCALLs during initialization.

Opportunistically update the comment regarding emergency disabling via NMI
to clarify that virt_rebooting will be set by _another_ emergency callback,
i.e. that virt_rebooting doesn't need to be set before VMCLEAR, only
before _this_ invocation does VMXOFF.

Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Tested-by: Chao Gao <chao.gao@intel.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Tested-by: Sagi Shahar <sagis@google.com>
Link: https://patch.msgid.link/20260214012702.2368778-7-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c
index 44524a387c58b6d1d8305dd35e5ac14a56ee0951..b5726b50e77dbeb632b9a0ee9dc9b0a3ccb5203e 100644 (file)
--- a/arch/x86/events/intel/pt.c
+++ b/arch/x86/events/intel/pt.c
@@ -1591,7 +1591,6 @@ void intel_pt_handle_vmx(int on)
 
        local_irq_restore(flags);
 }
-EXPORT_SYMBOL_FOR_KVM(intel_pt_handle_vmx);
 
 /*
  * PMU callbacks

diff --git a/arch/x86/include/asm/virt.h b/arch/x86/include/asm/virt.h
index 0da6db4f5b0ce9f71279510f9c8a4f6dfb431b98..cca0210a5c16412aabf933e912078bd3c79bc488 100644 (file)
--- a/arch/x86/include/asm/virt.h
+++ b/arch/x86/include/asm/virt.h
@@ -2,8 +2,6 @@
 #ifndef _ASM_X86_VIRT_H
 #define _ASM_X86_VIRT_H
 
-#include <linux/percpu-defs.h>
-
 #include <asm/reboot.h>
 
 #if IS_ENABLED(CONFIG_KVM_X86)
@@ -12,7 +10,9 @@ extern bool virt_rebooting;
 void __init x86_virt_init(void);
 
 #if IS_ENABLED(CONFIG_KVM_INTEL)
-DECLARE_PER_CPU(struct vmcs *, root_vmcs);
+int x86_vmx_enable_virtualization_cpu(void);
+int x86_vmx_disable_virtualization_cpu(void);
+void x86_vmx_emergency_disable_virtualization_cpu(void);
 #endif
 
 #else

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index e767835a4f3a75c459d20f81f92a48f9048c710a..36238cc694fd1ea5403a9576506733605a7591d5 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -786,41 +786,16 @@ static int vmx_set_guest_uret_msr(struct vcpu_vmx *vmx,
        return ret;
 }
 
-/*
- * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
- *
- * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
- * atomically track post-VMXON state, e.g. this may be called in NMI context.
- * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
- * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
- * magically in RM, VM86, compat mode, or at CPL>0.
- */
-static int kvm_cpu_vmxoff(void)
-{
-       asm goto("1: vmxoff\n\t"
-                         _ASM_EXTABLE(1b, %l[fault])
-                         ::: "cc", "memory" : fault);
-
-       cr4_clear_bits(X86_CR4_VMXE);
-       return 0;
-
-fault:
-       cr4_clear_bits(X86_CR4_VMXE);
-       return -EIO;
-}
-
 void vmx_emergency_disable_virtualization_cpu(void)
 {
        int cpu = raw_smp_processor_id();
        struct loaded_vmcs *v;
 
-       virt_rebooting = true;
-
        /*
         * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be
-        * set in task context.  If this races with VMX is disabled by an NMI,
-        * VMCLEAR and VMXOFF may #UD, but KVM will eat those faults due to
-        * virt_rebooting set.
+        * set in task context.  If this races with _another_ emergency call
+        * from NMI context, VMCLEAR may #UD, but KVM will eat those faults due
+        * to virt_rebooting being set by the interrupting NMI callback.
         */
        if (!(__read_cr4() & X86_CR4_VMXE))
                return;
@@ -832,7 +807,7 @@ void vmx_emergency_disable_virtualization_cpu(void)
                        vmcs_clear(v->shadow_vmcs);
        }
 
-       kvm_cpu_vmxoff();
+       x86_vmx_emergency_disable_virtualization_cpu();
 }
 
 static void __loaded_vmcs_clear(void *arg)
@@ -2988,34 +2963,9 @@ int vmx_check_processor_compat(void)
        return 0;
 }
 
-static int kvm_cpu_vmxon(u64 vmxon_pointer)
-{
-       u64 msr;
-
-       cr4_set_bits(X86_CR4_VMXE);
-
-       asm goto("1: vmxon %[vmxon_pointer]\n\t"
-                         _ASM_EXTABLE(1b, %l[fault])
-                         : : [vmxon_pointer] "m"(vmxon_pointer)
-                         : : fault);
-       return 0;
-
-fault:
-       WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
-                 rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
-       cr4_clear_bits(X86_CR4_VMXE);
-
-       return -EFAULT;
-}
-
 int vmx_enable_virtualization_cpu(void)
 {
        int cpu = raw_smp_processor_id();
-       u64 phys_addr = __pa(per_cpu(root_vmcs, cpu));
-       int r;
-
-       if (cr4_read_shadow() & X86_CR4_VMXE)
-               return -EBUSY;
 
        /*
         * This can happen if we hot-added a CPU but failed to allocate
@@ -3024,15 +2974,7 @@ int vmx_enable_virtualization_cpu(void)
        if (kvm_is_using_evmcs() && !hv_get_vp_assist_page(cpu))
                return -EFAULT;
 
-       intel_pt_handle_vmx(1);
-
-       r = kvm_cpu_vmxon(phys_addr);
-       if (r) {
-               intel_pt_handle_vmx(0);
-               return r;
-       }
-
-       return 0;
+       return x86_vmx_enable_virtualization_cpu();
 }
 
 static void vmclear_local_loaded_vmcss(void)
@@ -3049,12 +2991,9 @@ void vmx_disable_virtualization_cpu(void)
 {
        vmclear_local_loaded_vmcss();
 
-       if (kvm_cpu_vmxoff())
-               kvm_spurious_fault();
+       x86_vmx_disable_virtualization_cpu();
 
        hv_reset_evmcs();
-
-       intel_pt_handle_vmx(0);
 }
 
 struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags)

diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
index 40495872fdfbbe46a9014c9eca7282fc73dd700b..dc426c2bc24adfd3b238e0b78687cad004e796db 100644 (file)
--- a/arch/x86/virt/hw.c
+++ b/arch/x86/virt/hw.c
@@ -15,8 +15,89 @@ __visible bool virt_rebooting;
 EXPORT_SYMBOL_FOR_KVM(virt_rebooting);
 
 #if IS_ENABLED(CONFIG_KVM_INTEL)
-DEFINE_PER_CPU(struct vmcs *, root_vmcs);
-EXPORT_PER_CPU_SYMBOL(root_vmcs);
+static DEFINE_PER_CPU(struct vmcs *, root_vmcs);
+
+static int x86_virt_cpu_vmxon(void)
+{
+       u64 vmxon_pointer = __pa(per_cpu(root_vmcs, raw_smp_processor_id()));
+       u64 msr;
+
+       cr4_set_bits(X86_CR4_VMXE);
+
+       asm goto("1: vmxon %[vmxon_pointer]\n\t"
+                         _ASM_EXTABLE(1b, %l[fault])
+                         : : [vmxon_pointer] "m"(vmxon_pointer)
+                         : : fault);
+       return 0;
+
+fault:
+       WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
+                 rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
+       cr4_clear_bits(X86_CR4_VMXE);
+
+       return -EFAULT;
+}
+
+int x86_vmx_enable_virtualization_cpu(void)
+{
+       int r;
+
+       if (cr4_read_shadow() & X86_CR4_VMXE)
+               return -EBUSY;
+
+       intel_pt_handle_vmx(1);
+
+       r = x86_virt_cpu_vmxon();
+       if (r) {
+               intel_pt_handle_vmx(0);
+               return r;
+       }
+
+       return 0;
+}
+EXPORT_SYMBOL_FOR_KVM(x86_vmx_enable_virtualization_cpu);
+
+/*
+ * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
+ *
+ * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
+ * atomically track post-VMXON state, e.g. this may be called in NMI context.
+ * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
+ * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
+ * magically in RM, VM86, compat mode, or at CPL>0.
+ */
+int x86_vmx_disable_virtualization_cpu(void)
+{
+       int r = -EIO;
+
+       asm goto("1: vmxoff\n\t"
+                _ASM_EXTABLE(1b, %l[fault])
+                ::: "cc", "memory" : fault);
+       r = 0;
+
+fault:
+       cr4_clear_bits(X86_CR4_VMXE);
+       intel_pt_handle_vmx(0);
+       return r;
+}
+EXPORT_SYMBOL_FOR_KVM(x86_vmx_disable_virtualization_cpu);
+
+void x86_vmx_emergency_disable_virtualization_cpu(void)
+{
+       virt_rebooting = true;
+
+       /*
+        * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be
+        * set in task context.  If this races with _another_ emergency call
+        * from NMI context, VMXOFF may #UD, but kernel will eat those faults
+        * due to virt_rebooting being set by the interrupting NMI callback.
+        */
+       if (!(__read_cr4() & X86_CR4_VMXE))
+               return;
+
+       x86_vmx_disable_virtualization_cpu();
+}
+EXPORT_SYMBOL_FOR_KVM(x86_vmx_emergency_disable_virtualization_cpu);
 
 static __init void x86_vmx_exit(void)
 {