KVM: x86: Use a dedicated flow for queueing re-injected exceptions

Open code the filling of vcpu->arch.exception in kvm_requeue_exception()
instead of bouncing through kvm_multiple_exception(), as re-injection
doesn't actually share that much code with "normal" injection, e.g. the
VM-Exit interception check, payload delivery, and nested exception code
is all bypassed as those flows only apply during initial injection.

When FRED comes along, the special casing will only get worse, as FRED
explicitly tracks nested exceptions and essentially delivers the payload
on the stack frame, i.e. re-injection will need more inputs, and normal
injection will have yet more code that needs to be bypassed when KVM is
re-injecting an exception.

No functional change intended.

Signed-off-by: Xin Li (Intel) <xin@zytor.com>
Tested-by: Shan Kang <shan.kang@intel.com>
Link: https://lore.kernel.org/r/20241001050110.3643764-2-xin@zytor.com
Signed-off-by: Sean Christopherson <seanjc@google.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index fef8f07cdc826b53fd0cd2ba7f39e0db70b10cc0..97978ca62b7cc0600707c965146bea61110c5b65 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -2161,8 +2161,8 @@ int kvm_emulate_rdpmc(struct kvm_vcpu *vcpu);
 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr);
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr, unsigned long payload);
-void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr);
-void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code);
+void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
+                          bool has_error_code, u32 error_code);
 void kvm_inject_page_fault(struct kvm_vcpu *vcpu, struct x86_exception *fault);
 void kvm_inject_emulated_page_fault(struct kvm_vcpu *vcpu,
                                    struct x86_exception *fault);

diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 88dd61389a2d1eb812af0846a68bf2e299aa728a..6bd7e44380e011b8ee5f5903b181d2f9187a7848 100644 (file)
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4116,20 +4116,23 @@ static void svm_complete_interrupts(struct kvm_vcpu *vcpu)
                vcpu->arch.nmi_injected = true;
                svm->nmi_l1_to_l2 = nmi_l1_to_l2;
                break;
-       case SVM_EXITINTINFO_TYPE_EXEPT:
+       case SVM_EXITINTINFO_TYPE_EXEPT: {
+               u32 error_code = 0;
+
                /*
                 * Never re-inject a #VC exception.
                 */
                if (vector == X86_TRAP_VC)
                        break;
 
-               if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
-                       u32 err = svm->vmcb->control.exit_int_info_err;
-                       kvm_requeue_exception_e(vcpu, vector, err);
+               if (exitintinfo & SVM_EXITINTINFO_VALID_ERR)
+                       error_code = svm->vmcb->control.exit_int_info_err;
 
-               } else
-                       kvm_requeue_exception(vcpu, vector);
+               kvm_requeue_exception(vcpu, vector,
+                                     exitintinfo & SVM_EXITINTINFO_VALID_ERR,
+                                     error_code);
                break;
+       }
        case SVM_EXITINTINFO_TYPE_INTR:
                kvm_queue_interrupt(vcpu, vector, false);
                break;

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 1e5f73a7f8f02d82d3b2dfa4b3b3a0dbc79afca7..374f5a2ca38e6860d6d137a47a6cb891dc6ad10e 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -7155,13 +7155,17 @@ static void __vmx_complete_interrupts(struct kvm_vcpu *vcpu,
        case INTR_TYPE_SOFT_EXCEPTION:
                vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
                fallthrough;
-       case INTR_TYPE_HARD_EXCEPTION:
-               if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) {
-                       u32 err = vmcs_read32(error_code_field);
-                       kvm_requeue_exception_e(vcpu, vector, err);
-               } else
-                       kvm_requeue_exception(vcpu, vector);
+       case INTR_TYPE_HARD_EXCEPTION: {
+               u32 error_code = 0;
+
+               if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK)
+                       error_code = vmcs_read32(error_code_field);
+
+               kvm_requeue_exception(vcpu, vector,
+                                     idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK,
+                                     error_code);
                break;
+       }
        case INTR_TYPE_SOFT_INTR:
                vcpu->arch.event_exit_inst_len = vmcs_read32(instr_len_field);
                fallthrough;

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a13e98f90a0ed055f83b445f52a5d5d0f19845e2..2231a2cd8489237a0c2b2ae91ae72e4585585ff3 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -800,9 +800,9 @@ static void kvm_queue_exception_vmexit(struct kvm_vcpu *vcpu, unsigned int vecto
        ex->payload = payload;
 }
 
-static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
-               unsigned nr, bool has_error, u32 error_code,
-               bool has_payload, unsigned long payload, bool reinject)
+static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr,
+                                  bool has_error, u32 error_code,
+                                  bool has_payload, unsigned long payload)
 {
        u32 prev_nr;
        int class1, class2;
@@ -810,13 +810,10 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
        kvm_make_request(KVM_REQ_EVENT, vcpu);
 
        /*
-        * If the exception is destined for L2 and isn't being reinjected,
-        * morph it to a VM-Exit if L1 wants to intercept the exception.  A
-        * previously injected exception is not checked because it was checked
-        * when it was original queued, and re-checking is incorrect if _L1_
-        * injected the exception, in which case it's exempt from interception.
+        * If the exception is destined for L2, morph it to a VM-Exit if L1
+        * wants to intercept the exception.
         */
-       if (!reinject && is_guest_mode(vcpu) &&
+       if (is_guest_mode(vcpu) &&
            kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) {
                kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code,
                                           has_payload, payload);
@@ -825,28 +822,9 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
 
        if (!vcpu->arch.exception.pending && !vcpu->arch.exception.injected) {
        queue:
-               if (reinject) {
-                       /*
-                        * On VM-Entry, an exception can be pending if and only
-                        * if event injection was blocked by nested_run_pending.
-                        * In that case, however, vcpu_enter_guest() requests an
-                        * immediate exit, and the guest shouldn't proceed far
-                        * enough to need reinjection.
-                        */
-                       WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
-                       vcpu->arch.exception.injected = true;
-                       if (WARN_ON_ONCE(has_payload)) {
-                               /*
-                                * A reinjected event has already
-                                * delivered its payload.
-                                */
-                               has_payload = false;
-                               payload = 0;
-                       }
-               } else {
-                       vcpu->arch.exception.pending = true;
-                       vcpu->arch.exception.injected = false;
-               }
+               vcpu->arch.exception.pending = true;
+               vcpu->arch.exception.injected = false;
+
                vcpu->arch.exception.has_error_code = has_error;
                vcpu->arch.exception.vector = nr;
                vcpu->arch.exception.error_code = error_code;
@@ -887,29 +865,52 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu,
 
 void kvm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr)
 {
-       kvm_multiple_exception(vcpu, nr, false, 0, false, 0, false);
+       kvm_multiple_exception(vcpu, nr, false, 0, false, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception);
 
-void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned nr)
-{
-       kvm_multiple_exception(vcpu, nr, false, 0, false, 0, true);
-}
-EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 
 void kvm_queue_exception_p(struct kvm_vcpu *vcpu, unsigned nr,
                           unsigned long payload)
 {
-       kvm_multiple_exception(vcpu, nr, false, 0, true, payload, false);
+       kvm_multiple_exception(vcpu, nr, false, 0, true, payload);
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception_p);
 
 static void kvm_queue_exception_e_p(struct kvm_vcpu *vcpu, unsigned nr,
                                    u32 error_code, unsigned long payload)
 {
-       kvm_multiple_exception(vcpu, nr, true, error_code,
-                              true, payload, false);
+       kvm_multiple_exception(vcpu, nr, true, error_code, true, payload);
+}
+
+void kvm_requeue_exception(struct kvm_vcpu *vcpu, unsigned int nr,
+                          bool has_error_code, u32 error_code)
+{
+
+       /*
+        * On VM-Entry, an exception can be pending if and only if event
+        * injection was blocked by nested_run_pending.  In that case, however,
+        * vcpu_enter_guest() requests an immediate exit, and the guest
+        * shouldn't proceed far enough to need reinjection.
+        */
+       WARN_ON_ONCE(kvm_is_exception_pending(vcpu));
+
+       /*
+        * Do not check for interception when injecting an event for L2, as the
+        * exception was checked for intercept when it was original queued, and
+        * re-checking is incorrect if _L1_ injected the exception, in which
+        * case it's exempt from interception.
+        */
+       kvm_make_request(KVM_REQ_EVENT, vcpu);
+
+       vcpu->arch.exception.injected = true;
+       vcpu->arch.exception.has_error_code = has_error_code;
+       vcpu->arch.exception.vector = nr;
+       vcpu->arch.exception.error_code = error_code;
+       vcpu->arch.exception.has_payload = false;
+       vcpu->arch.exception.payload = 0;
 }
+EXPORT_SYMBOL_GPL(kvm_requeue_exception);
 
 int kvm_complete_insn_gp(struct kvm_vcpu *vcpu, int err)
 {
@@ -980,16 +981,10 @@ void kvm_inject_nmi(struct kvm_vcpu *vcpu)
 
 void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
 {
-       kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, false);
+       kvm_multiple_exception(vcpu, nr, true, error_code, false, 0);
 }
 EXPORT_SYMBOL_GPL(kvm_queue_exception_e);
 
-void kvm_requeue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code)
-{
-       kvm_multiple_exception(vcpu, nr, true, error_code, false, 0, true);
-}
-EXPORT_SYMBOL_GPL(kvm_requeue_exception_e);
-
 /*
  * Checks if cpl <= required_cpl; if true, return true.  Otherwise queue
  * a #GP and return false.