KVM: x86: Unify cross-vCPU IBPB

Both SVM and VMX have similar implementation for executing an IBPB
between running different vCPUs on the same CPU to create separate
prediction domains for different vCPUs.

For VMX, when the currently loaded VMCS is changed in
vmx_vcpu_load_vmcs(), an IBPB is executed if there is no 'buddy', which
is the case on vCPU load. The intention is to execute an IBPB when
switching vCPUs, but not when switching the VMCS within the same vCPU.
Executing an IBPB on nested transitions within the same vCPU is handled
separately and conditionally in nested_vmx_vmexit().

For SVM, the current VMCB is tracked on vCPU load and an IBPB is
executed when it is changed. The intention is also to execute an IBPB
when switching vCPUs, although it is possible that in some cases an IBBP
is executed when switching VMCBs for the same vCPU. Executing an IBPB on
nested transitions should be handled separately, and is proposed at [1].

Unify the logic by tracking the last loaded vCPU and execuintg the IBPB
on vCPU change in kvm_arch_vcpu_load() instead. When a vCPU is
destroyed, make sure all references to it are removed from any CPU. This
is similar to how SVM clears the current_vmcb tracking on vCPU
destruction. Remove the current VMCB tracking in SVM as it is no longer
required, as well as the 'buddy' parameter to vmx_vcpu_load_vmcs().

[1] https://lore.kernel.org/lkml/20250221163352.3818347-4-yosry.ahmed@linux.dev

Link: https://lore.kernel.org/all/20250320013759.3965869-1-yosry.ahmed@linux.dev
Signed-off-by: Yosry Ahmed <yosry.ahmed@linux.dev>
[sean: tweak comment to stay at/under 80 columns]
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index e6802e33c54dfd798cea8b1b9abfcf3aba9c6bf8..7d5f5ea9794f3497fdf3c1ac443df24ae38e0de4 100644 (file)
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -1492,25 +1492,10 @@ out:
        return err;
 }
 
-static void svm_clear_current_vmcb(struct vmcb *vmcb)
-{
-       int i;
-
-       for_each_possible_cpu(i)
-               cmpxchg(per_cpu_ptr(&svm_data.current_vmcb, i), vmcb, NULL);
-}
-
 static void svm_vcpu_free(struct kvm_vcpu *vcpu)
 {
        struct vcpu_svm *svm = to_svm(vcpu);
 
-       /*
-        * The vmcb page can be recycled, causing a false negative in
-        * svm_vcpu_load(). So, ensure that no logical CPU has this
-        * vmcb page recorded as its current vmcb.
-        */
-       svm_clear_current_vmcb(svm->vmcb);
-
        svm_leave_nested(vcpu);
        svm_free_nested(svm);
 
@@ -1562,19 +1547,9 @@ static void svm_prepare_host_switch(struct kvm_vcpu *vcpu)
 
 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
-       struct vcpu_svm *svm = to_svm(vcpu);
-       struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, cpu);
-
        if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
                shrink_ple_window(vcpu);
 
-       if (sd->current_vmcb != svm->vmcb) {
-               sd->current_vmcb = svm->vmcb;
-
-               if (!cpu_feature_enabled(X86_FEATURE_IBPB_ON_VMEXIT) &&
-                   static_branch_likely(&switch_vcpu_ibpb))
-                       indirect_branch_prediction_barrier();
-       }
        if (kvm_vcpu_apicv_active(vcpu))
                avic_vcpu_load(vcpu, cpu);
 }

diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index d4490eaed55dd42130552e517d899c5f16d090cb..c4584cee8f71cf215f6b2a74619da96601426201 100644 (file)
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -338,8 +338,6 @@ struct svm_cpu_data {
        struct vmcb *save_area;
        unsigned long save_area_pa;
 
-       struct vmcb *current_vmcb;
-
        /* index = sev_asid, value = vmcb pointer */
        struct vmcb **sev_vmcbs;
 };

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index ebeb183270cfb834b8f5cce13b513779a0435393..4b3fe437a5a12c05d076f620972d2db7afa72493 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -301,7 +301,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
        cpu = get_cpu();
        prev = vmx->loaded_vmcs;
        vmx->loaded_vmcs = vmcs;
-       vmx_vcpu_load_vmcs(vcpu, cpu, prev);
+       vmx_vcpu_load_vmcs(vcpu, cpu);
        vmx_sync_vmcs_host_state(vmx, prev);
        put_cpu();
 
@@ -4520,12 +4520,12 @@ static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
 
        cpu = get_cpu();
        vmx->loaded_vmcs = &vmx->nested.vmcs02;
-       vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->vmcs01);
+       vmx_vcpu_load_vmcs(vcpu, cpu);
 
        sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
 
        vmx->loaded_vmcs = &vmx->vmcs01;
-       vmx_vcpu_load_vmcs(vcpu, cpu, &vmx->nested.vmcs02);
+       vmx_vcpu_load_vmcs(vcpu, cpu);
        put_cpu();
 }
 

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index ba08a7edc712aadc32420a04fabf7d61f19fc277..914d69c0bd9d0cadd31330ab6edcf689ed05637f 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -1445,8 +1445,7 @@ static void shrink_ple_window(struct kvm_vcpu *vcpu)
        }
 }
 
-void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
-                       struct loaded_vmcs *buddy)
+void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
        bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
@@ -1473,17 +1472,6 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
        if (prev != vmx->loaded_vmcs->vmcs) {
                per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
                vmcs_load(vmx->loaded_vmcs->vmcs);
-
-               /*
-                * No indirect branch prediction barrier needed when switching
-                * the active VMCS within a vCPU, unless IBRS is advertised to
-                * the vCPU.  To minimize the number of IBPBs executed, KVM
-                * performs IBPB on nested VM-Exit (a single nested transition
-                * may switch the active VMCS multiple times).
-                */
-               if (static_branch_likely(&switch_vcpu_ibpb) &&
-                   (!buddy || WARN_ON_ONCE(buddy->vmcs != prev)))
-                       indirect_branch_prediction_barrier();
        }
 
        if (!already_loaded) {
@@ -1522,7 +1510,7 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
        if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm))
                shrink_ple_window(vcpu);
 
-       vmx_vcpu_load_vmcs(vcpu, cpu, NULL);
+       vmx_vcpu_load_vmcs(vcpu, cpu);
 
        vmx_vcpu_pi_load(vcpu, cpu);
 }

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index 6d1e40ecc024137198318b854349d5b0d5ad59b3..b5758c33c60f9d69ce0f8459ac4fcacb00267b97 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -354,8 +354,7 @@ static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
        return vt->exit_intr_info;
 }
 
-void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
-                       struct loaded_vmcs *buddy);
+void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu);
 int allocate_vpid(void);
 void free_vpid(int vpid);
 void vmx_set_constant_host_state(struct vcpu_vmx *vmx);

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index f35f4456e388aef2f7496c7347fc2af517597554..b45e9d6b526185c8345a88604134a98e06bee325 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4991,6 +4991,8 @@ static bool need_emulate_wbinvd(struct kvm_vcpu *vcpu)
        return kvm_arch_has_noncoherent_dma(vcpu->kvm);
 }
 
+static DEFINE_PER_CPU(struct kvm_vcpu *, last_vcpu);
+
 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
        struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
@@ -5013,6 +5015,19 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 
        kvm_x86_call(vcpu_load)(vcpu, cpu);
 
+       if (vcpu != per_cpu(last_vcpu, cpu)) {
+               /*
+                * Flush the branch predictor when switching vCPUs on the same
+                * physical CPU, as each vCPU needs its own branch prediction
+                * domain.  No IBPB is needed when switching between L1 and L2
+                * on the same vCPU unless IBRS is advertised to the vCPU; that
+                * is handled on the nested VM-Exit path.
+                */
+               if (static_branch_likely(&switch_vcpu_ibpb))
+                       indirect_branch_prediction_barrier();
+               per_cpu(last_vcpu, cpu) = vcpu;
+       }
+
        /* Save host pkru register if supported */
        vcpu->arch.host_pkru = read_pkru();
 
@@ -12424,13 +12439,16 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
-       int idx;
+       int idx, cpu;
 
        kvm_clear_async_pf_completion_queue(vcpu);
        kvm_mmu_unload(vcpu);
 
        kvmclock_reset(vcpu);
 
+       for_each_possible_cpu(cpu)
+               cmpxchg(per_cpu_ptr(&last_vcpu, cpu), vcpu, NULL);
+
        kvm_x86_call(vcpu_free)(vcpu);
 
        kmem_cache_free(x86_emulator_cache, vcpu->arch.emulate_ctxt);