KVM: x86: Unify L1TF flushing under per-CPU variable

Currently the tracking of the need to flush L1D for L1TF is tracked by
two bits: one per-CPU and one per-vCPU.

The per-vCPU bit is always set when the vCPU shows up on a core, so
there is no interesting state that's truly per-vCPU. Indeed, this is a
requirement, since L1D is a part of the physical CPU.

So simplify this by combining the two bits.

The vCPU bit was being written from preemption-enabled regions.  To play
nice with those cases, wrap all calls from KVM and use a raw write so that
request a flush with preemption enabled doesn't trigger what would
effectively be DEBUG_PREEMPT false positives.  Preemption doesn't need to
be disabled, as kvm_arch_vcpu_load() will mark the new CPU as needing a
flush if the vCPU task is migrated, or if userspace runs the vCPU on a
different task.

Signed-off-by: Brendan Jackman <jackmanb@google.com>
[sean: put raw write in KVM instead of in a hardirq.h variant]
Link: https://patch.msgid.link/20251113233746.1703361-10-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 692ec922945f5b00e9a9679ba75c884cb448a3ac..103af57e1060fa2ae75d8273ebdbaee0dcec9c8f 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1055,9 +1055,6 @@ struct kvm_vcpu_arch {
        /* be preempted when it's in kernel-mode(cpl=0) */
        bool preempted_in_kernel;
 
-       /* Flush the L1 Data cache for L1TF mitigation on VMENTER */
-       bool l1tf_flush_l1d;
-
        /* Host CPU on which VM-entry was most recently attempted */
        int last_vmentry_cpu;
 

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 667d66cf76d5e52c22f9517914307244ae868eea..a81637a9801925167336b99faba8c2902eb64a35 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -4859,7 +4859,7 @@ int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
         */
        BUILD_BUG_ON(lower_32_bits(PFERR_SYNTHETIC_MASK));
 
-       vcpu->arch.l1tf_flush_l1d = true;
+       kvm_request_l1tf_flush_l1d();
        if (!flags) {
                trace_kvm_page_fault(vcpu, fault_address, error_code);
 

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 76271962cb7083b475de6d7d24bf9cb918050650..256b51fc8c108522761deaec03c2049d33af8c32 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -3880,7 +3880,7 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
                goto vmentry_failed;
 
        /* Hide L1D cache contents from the nested guest.  */
-       vmx->vcpu.arch.l1tf_flush_l1d = true;
+       kvm_request_l1tf_flush_l1d();
 
        /*
         * Must happen outside of nested_vmx_enter_non_root_mode() as it will

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 59d3f2671177cb35bea4e2f3facfd15bfd0f8e6a..634f591d253e7f917ac23c2cab98d046770bc369 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -395,26 +395,16 @@ static noinstr void vmx_l1d_flush(struct kvm_vcpu *vcpu)
         * 'always'
         */
        if (static_branch_likely(&vmx_l1d_flush_cond)) {
-               bool flush_l1d;
-
                /*
-                * Clear the per-vcpu flush bit, it gets set again if the vCPU
+                * Clear the per-cpu flush bit, it gets set again if the vCPU
                 * is reloaded, i.e. if the vCPU is scheduled out or if KVM
                 * exits to userspace, or if KVM reaches one of the unsafe
-                * VMEXIT handlers, e.g. if KVM calls into the emulator.
-                */
-               flush_l1d = vcpu->arch.l1tf_flush_l1d;
-               vcpu->arch.l1tf_flush_l1d = false;
-
-               /*
-                * Clear the per-cpu flush bit, it gets set again from
-                * the interrupt handlers.
+                * VMEXIT handlers, e.g. if KVM calls into the emulator,
+                * or from the interrupt handlers.
                 */
-               flush_l1d |= kvm_get_cpu_l1tf_flush_l1d();
-               kvm_clear_cpu_l1tf_flush_l1d();
-
-               if (!flush_l1d)
+               if (!kvm_get_cpu_l1tf_flush_l1d())
                        return;
+               kvm_clear_cpu_l1tf_flush_l1d();
        }
 
        vcpu->stat.l1d_flush++;

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index be737d9645b9faecaa32813e9ccb2fb174fb6aba..6af37204bd976119404fa129f2993db9d8b819d6 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5156,7 +5156,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
        struct kvm_pmu *pmu = vcpu_to_pmu(vcpu);
 
-       vcpu->arch.l1tf_flush_l1d = true;
+       kvm_request_l1tf_flush_l1d();
 
        if (vcpu->scheduled_out && pmu->version && pmu->event_count) {
                pmu->need_cleanup = true;
@@ -7966,7 +7966,7 @@ int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val,
                                unsigned int bytes, struct x86_exception *exception)
 {
        /* kvm_write_guest_virt_system can pull in tons of pages. */
-       vcpu->arch.l1tf_flush_l1d = true;
+       kvm_request_l1tf_flush_l1d();
 
        return kvm_write_guest_virt_helper(addr, val, bytes, vcpu,
                                           PFERR_WRITE_MASK, exception);
@@ -9374,7 +9374,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
                return handle_emulation_failure(vcpu, emulation_type);
        }
 
-       vcpu->arch.l1tf_flush_l1d = true;
+       kvm_request_l1tf_flush_l1d();
 
        if (!(emulation_type & EMULTYPE_NO_DECODE)) {
                kvm_clear_exception_queue(vcpu);

diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index f3dc77f006f9046da939f021165b81b01865e78e..cd67ccbb747f97f20456cfcd1b1c822904ccf96f 100644 (file)
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -420,6 +420,20 @@ static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk)
        return !(kvm->arch.disabled_quirks & quirk);
 }
 
+static __always_inline void kvm_request_l1tf_flush_l1d(void)
+{
+#if IS_ENABLED(CONFIG_CPU_MITIGATIONS) && IS_ENABLED(CONFIG_KVM_INTEL)
+       /*
+        * Use a raw write to set the per-CPU flag, as KVM will ensure a flush
+        * even if preemption is currently enabled..  If the current vCPU task
+        * is migrated to a different CPU (or userspace runs the vCPU on a
+        * different task) before the next VM-Entry, then kvm_arch_vcpu_load()
+        * will request a flush on the new CPU.
+        */
+       raw_cpu_write(irq_stat.kvm_cpu_l1tf_flush_l1d, 1);
+#endif
+}
+
 void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip);
 
 u64 get_kvmclock_ns(struct kvm *kvm);