KVM: x86: Tell ->inject_page_fault() whether or a fault came from hardware

When injecting a page fault (including nested TDP faults into L1), tell the
injection routine whether or not the fault originated in hardware, i.e. if
KVM is effectively forwarding a fault it intercept.  For nested TDP fault
injection, KVM needs to grab PAGE_WALK vs. GUEST_FINAL information from the
VMCB/VMCS, _if_ the fault originated in hardware.

Note, simply checking whether or not the original exit was due a #NPF or
EPT Violation isn't sufficient/correct, as the fault being synthesized for
L1 may or may not be the "same" fault that triggered a VM-Exit from L2.
E.g. if access to emulated MMIO in L2 hits a !PRESENT fault (EPT Violation
or #NPF), e.g. because MMIO caching is disabled or it's the first time the
GPA has been accessed by L2, then KVM will enter the emulator.  If
emulating the MMIO instruction then hits a nested TDP fault, e.g. because
L2 was accessing MMIO with a MOVSQ (memory-to-memory move), or because L1
has since unmapped the code stream, then the TDP fault synthesized to L1
will not be the same emulated fault the triggered the VM-Exit.

No functional change intended (nothing uses the new param, yet...).

Link: https://patch.msgid.link/20260522232701.3671446-3-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 2b986a733cd6210f5405c00c5681d9ab92645019..29fec7c59a6f47e79d69b4232f7f928c5c65b6f3 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -484,7 +484,8 @@ struct kvm_mmu {
        u64 (*get_pdptr)(struct kvm_vcpu *vcpu, int index);
        int (*page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
        void (*inject_page_fault)(struct kvm_vcpu *vcpu,
-                                 struct x86_exception *fault);
+                                 struct x86_exception *fault,
+                                 bool from_hardware);
        gpa_t (*gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
                            gpa_t gva_or_gpa, u64 access,
                            struct x86_exception *exception);
@@ -2304,9 +2305,18 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
 void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
                           bool has_error_code, u32 error_code);
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
-void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
-                                   struct x86_exception *fault);
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
+                          bool from_hardware);
+void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
+                                     struct x86_exception *fault,
+                                     bool from_hardware);
+
+static inline void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
+                                                 struct x86_exception *fault)
+{
+       __kvm_inject_emulated_page_fault(vcpu, fault, false);
+}
+
 bool kvm_require_cpl(struct kvm_vcpu *vcpu, int required_cpl);
 bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr);
 

diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 51f8b45223143b7c9ff62c3e61de4ba6b92b097b..cc9c7deb34bcc001ae34f551c3dd9c0df932e61f 100644 (file)
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -813,7 +813,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
         */
        if (!r) {
                if (!fault->prefetch)
-                       kvm_inject_emulated_page_fault(vcpu, &walker.fault);
+                       __kvm_inject_emulated_page_fault(vcpu, &walker.fault, true);
 
                return RET_PF_RETRY;
        }

diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index 3d1fd1776e195d96c2dfb98490d916f6f9b3343f..edb15f9c6403e6fd99bf6e7f136defb606e55fb2 100644 (file)
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -34,7 +34,8 @@
 #define CC KVM_NESTED_VMENTER_CONSISTENCY_CHECK
 
 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
-                                      struct x86_exception *fault)
+                                      struct x86_exception *fault,
+                                      bool from_hardware)
 {
        struct vcpu_svm *svm = to_svm(vcpu);
        struct vmcb *vmcb = svm->vmcb;

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 4690a4d23709d4b8f643fb62d52aaf987014d166..3bb7eaa7b2a5a4265cec560cef1b33922a828e01 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -411,7 +411,8 @@ static void nested_ept_invalidate_addr(struct kvm_vcpu *vcpu, gpa_t eptp,
 }
 
 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
-               struct x86_exception *fault)
+                                        struct x86_exception *fault,
+                                        bool from_hardware)
 {
        struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
        struct vcpu_vmx *vmx = to_vmx(vcpu);

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 9c52c11ba5f5e9a241096aef065b68286ed3c940..93958df302306f3166bd17329038d6b5bda7fe96 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -969,7 +969,8 @@ static int complete_emulated_insn_gp(struct kvm_vcpu *vcpu, int err)
                                       EMULTYPE_COMPLETE_USER_EXIT);
 }
 
-void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
+void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault,
+                          bool from_hardware)
 {
        ++vcpu->stat.pf_guest;
 
@@ -986,8 +987,9 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault)
                                        fault->address);
 }
 
-void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
-                                   struct x86_exception *fault)
+void __kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
+                                     struct x86_exception *fault,
+                                     bool from_hardware)
 {
        struct kvm_mmu *fault_mmu;
        WARN_ON_ONCE(fault->vector != PF_VECTOR);
@@ -1004,9 +1006,9 @@ void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
                kvm_mmu_invalidate_addr(vcpu, fault_mmu, fault->address,
                                        KVM_MMU_ROOT_CURRENT);
 
-       fault_mmu->inject_page_fault(vcpu, fault);
+       fault_mmu->inject_page_fault(vcpu, fault, from_hardware);
 }
-EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_inject_emulated_page_fault);
+EXPORT_SYMBOL_FOR_KVM_INTERNAL(__kvm_inject_emulated_page_fault);
 
 void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 {
@@ -14058,7 +14060,7 @@ bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
                fault.nested_page_fault = false;
                fault.address = work->arch.token;
                fault.async_page_fault = true;
-               kvm_inject_page_fault(vcpu, &fault);
+               kvm_inject_page_fault(vcpu, &fault, false);
                return true;
        } else {
                /*
@@ -14229,7 +14231,7 @@ void kvm_fixup_and_inject_pf_error(struct kvm_vcpu *vcpu, gva_t gva, u16 error_c
                fault.address = gva;
                fault.async_page_fault = false;
        }
-       vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault);
+       vcpu->arch.walk_mmu->inject_page_fault(vcpu, &fault, true);
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_fixup_and_inject_pf_error);