KVM: x86/pmu: Reprogram Host/Guest-Only counters on nested transitions

Reprogram PMU counters on nested transitions for the mediated PMU, to
re-evaluate Host-Only and Guest-Only bits and enable/disable the PMU
counters accordingly. For example, if Host-Only is set and Guest-Only is
cleared, a counter should be disabled when entering guest mode and
enabled when exiting guest mode.

According to the APM, when EFER.SVME is cleared, setting Host-Only or
Guest-Only disables the counter, so also trigger counter reprogramming
when EFER.SVME is toggled.

Counters setting any of Host-Only and Guest-Only bits are already being
tracked in pmc_has_mode_specific_enables, use the bitmap to reprogram
these counters.

Reprogram the counters synchronously on nested VMRUN/#VMEXIT and
EFER.SVME toggling. This is necessary as these instructions are counted
based on the new CPU state (after the instruction is retired in
hardware).  Hence, the PMU needs to be updated before instruction
emulation is completed and kvm_pmu_instruction_retired() is called.

Defer reprogramming the counters when force leaving guest mode through
svm_leave_nested() to avoid potentially reading stale state (e.g.
incorrect EFER). All flows force leaving nested are non-architectural,
so accuracy is irrelevant.

Refactor a helper out of kvm_pmu_request_reprogram_counters() that
accepts a boolean allowing synchronous vs deferred reprogramming, and
use that from SVM code to support both scenarios.

Signed-off-by: Yosry Ahmed <yosry@kernel.org>
Link: https://patch.msgid.link/20260527234711.4175166-13-yosry@kernel.org
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 84c834ad2cd473cb03acad85f3476526b3c981a8..b92dd2e5833567a956d51a0e9d3b0440d6452434 100644 (file)
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -685,6 +685,7 @@ void kvm_pmu_handle_event(struct kvm_vcpu *vcpu)
        kvm_for_each_pmc(pmu, pmc, bit, bitmap)
                kvm_pmu_recalc_pmc_emulation(pmu, pmc);
 }
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_pmu_handle_event);
 
 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx)
 {

diff --git a/arch/x86/kvm/pmu.h b/arch/x86/kvm/pmu.h
index 34c3c6913ef6279677d541e7e154ebfa8951a53f..a5821d7c87f9354e9cea156a1c85e1feb282584b 100644 (file)
--- a/arch/x86/kvm/pmu.h
+++ b/arch/x86/kvm/pmu.h
@@ -216,6 +216,7 @@ extern struct x86_pmu_capability kvm_pmu_cap;
 void kvm_init_pmu_capability(struct kvm_pmu_ops *pmu_ops);
 
 void kvm_pmu_recalc_pmc_emulation(struct kvm_pmu *pmu, struct kvm_pmc *pmc);
+void kvm_pmu_handle_event(struct kvm_vcpu *vcpu);
 
 static inline void kvm_pmu_request_counter_reprogram(struct kvm_pmc *pmc)
 {
@@ -225,14 +226,24 @@ static inline void kvm_pmu_request_counter_reprogram(struct kvm_pmc *pmc)
        kvm_make_request(KVM_REQ_PMU, pmc->vcpu);
 }
 
-static inline void kvm_pmu_request_counters_reprogram(struct kvm_pmu *pmu,
-                                                     u64 counters)
+static inline void __kvm_pmu_reprogram_counters(struct kvm_pmu *pmu,
+                                               u64 counters,
+                                               bool defer)
 {
        if (!counters)
                return;
 
        atomic64_or(counters, &pmu->__reprogram_pmi);
-       kvm_make_request(KVM_REQ_PMU, pmu_to_vcpu(pmu));
+       if (defer)
+               kvm_make_request(KVM_REQ_PMU, pmu_to_vcpu(pmu));
+       else
+               kvm_pmu_handle_event(pmu_to_vcpu(pmu));
+}
+
+static inline void kvm_pmu_request_counters_reprogram(struct kvm_pmu *pmu,
+                                                     u64 counters)
+{
+       __kvm_pmu_reprogram_counters(pmu, counters, true);
 }
 
 /*
@@ -261,7 +272,6 @@ static inline bool kvm_pmu_is_fastpath_emulation_allowed(struct kvm_vcpu *vcpu)
 }
 
 void kvm_pmu_deliver_pmi(struct kvm_vcpu *vcpu);
-void kvm_pmu_handle_event(struct kvm_vcpu *vcpu);
 int kvm_pmu_rdpmc(struct kvm_vcpu *vcpu, unsigned pmc, u64 *data);
 int kvm_pmu_check_rdpmc_early(struct kvm_vcpu *vcpu, unsigned int idx);
 bool kvm_pmu_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr);

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 3a90b7f83d8e3412e54368ea9d78d7d33c1d6389..d1d7bb20ff35c919226f2c6af8f1f2a1381c89e1 100644 (file)
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -837,6 +837,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 
        /* Enter Guest-Mode */
        enter_guest_mode(vcpu);
+       svm_pmu_handle_nested_transition(svm);
 
        /*
         * Filled at exit: exit_code, exit_info_1, exit_info_2, exit_int_info,
@@ -1320,6 +1321,8 @@ void nested_svm_vmexit(struct vcpu_svm *svm)
 
        /* Exit Guest-Mode */
        leave_guest_mode(vcpu);
+       svm_pmu_handle_nested_transition(svm);
+
        svm->nested.vmcb12_gpa = 0;
 
        kvm_warn_on_nested_run_pending(vcpu);
@@ -1537,6 +1540,15 @@ void svm_leave_nested(struct kvm_vcpu *vcpu)
 
                leave_guest_mode(vcpu);
 
+               /*
+                * Force leaving nested is a non-architectural flow so precision
+                * isn't a priority.  Defer updating the PMU until the next vCPU
+                * run, potentially tolerating some imprecision to avoid poking
+                * into PMU state from arbitrary contexts (e.g. to avoid using
+                * stale state).
+                */
+               __svm_pmu_handle_nested_transition(svm, true);
+
                svm_switch_vmcb(svm, &svm->vmcb01);
 
                nested_svm_uninit_mmu_context(vcpu);

diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index ab1014986b91923824f7f71608d5c052b6a2ea6f..502433c5b189da8bb29a39c9ab5743834b59f14b 100644 (file)
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -265,6 +265,7 @@ int svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
                                set_exception_intercept(svm, GP_VECTOR);
                }
 
+               svm_pmu_handle_nested_transition(svm);
                kvm_make_request(KVM_REQ_RECALC_INTERCEPTS, vcpu);
        }
 

diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index e5d9984ef6320f48fb5fc3fd495e0a3e6a7d4cb7..87c6b105deef6c75b4a1a9ff6c3d935b56021776 100644 (file)
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -25,6 +25,7 @@
 #include "cpuid.h"
 #include "kvm_cache_regs.h"
 #include "x86.h"
+#include "pmu.h"
 
 /*
  * Helpers to convert to/from physical addresses for pages whose address is
@@ -897,6 +898,28 @@ void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm);
 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb);
 
+
+static inline void __svm_pmu_handle_nested_transition(struct vcpu_svm *svm,
+                                                     bool defer)
+{
+       struct kvm_pmu *pmu = vcpu_to_pmu(&svm->vcpu);
+       u64 counters = *(u64 *)pmu->pmc_has_mode_specific_enables;
+
+       __kvm_pmu_reprogram_counters(pmu, counters, defer);
+}
+
+static inline void svm_pmu_handle_nested_transition(struct vcpu_svm *svm)
+{
+       /*
+        * Do NOT defer reprogramming the counters by default.  Instructions
+        * causing a state change are counted based on the _new_ CPU state
+        * (e.g. a successful VMRUN is counted in guest mode). Hence, the
+        * counters should be reprogrammed with the new state _before_ the
+        * instruction is potentially counted upon emulation completion.
+        */
+       __svm_pmu_handle_nested_transition(svm, false);
+}
+
 extern struct kvm_x86_nested_ops svm_nested_ops;
 
 /* avic.c */