KVM: s390: Rework guest entry logic

In __vcpu_run() and do_vsie_run(), we enter an RCU extended quiescent
state (EQS) by calling guest_enter_irqoff(), which lasts until
__vcpu_run() calls guest_exit_irqoff(). However, between the two we
enable interrupts and may handle interrupts during the EQS. As the IRQ
entry code will not wake RCU in this case, we may run the core IRQ code
and IRQ handler without RCU watching, leading to various potential
problems.

It is necessary to unmask (host) interrupts around entering the guest,
as entering the guest via SIE will not automatically unmask these. When
a host interrupt is taken from a guest, it is taken via its regular
host IRQ handler rather than being treated as a direct exit from SIE.
Due to this, we cannot simply mask interrupts around guest entry, and
must handle interrupts during this window, waking RCU as required.

Additionally, between guest_enter_irqoff() and guest_exit_irqoff(), we
use local_irq_enable() and local_irq_disable() to unmask interrupts,
violating the ordering requirements for RCU/lockdep/tracing around
entry/exit sequences. Further, since this occurs in an instrumentable
function, it's possible that instrumented code runs during this window,
with potential usage of RCU, etc.

To fix the RCU wakeup problem, an s390 implementation of
arch_in_rcu_eqs() is added which checks for PF_VCPU in current->flags.
PF_VCPU is set/cleared by guest_timing_{enter,exit}_irqoff(), which
surround the actual guest entry.

To fix the remaining issues, the lower-level guest entry logic is moved
into a shared noinstr helper function using the
guest_state_{enter,exit}_irqoff() helpers. These perform all the
lockdep/RCU/tracing manipulation necessary, but as sie64a() does not
enable/disable interrupts, we must do this explicitly with the
non-instrumented arch_local_irq_{enable,disable}() helpers:

        guest_state_enter_irqoff()

        arch_local_irq_enable();
        sie64a(...);
        arch_local_irq_disable();

        guest_state_exit_irqoff();

[ajd@linux.ibm.com: rebase, fix commit message]

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Frederic Weisbecker <frederic@kernel.org>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Janosch Frank <frankja@linux.ibm.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Paul E. McKenney <paulmck@kernel.org>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Claudio Imbrenda <imbrenda@linux.ibm.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Signed-off-by: Andrew Donnellan <ajd@linux.ibm.com>
Reviewed-by: Janosch Frank <frankja@linux.ibm.com>
Link: https://lore.kernel.org/r/20250708092742.104309-3-ajd@linux.ibm.com
Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
Message-ID: <20250708092742.104309-3-ajd@linux.ibm.com>

diff --git a/arch/s390/include/asm/entry-common.h b/arch/s390/include/asm/entry-common.h
index 35555c9446308d1ab76824536dd3da5deba9ef07..979af986a8febbf788229a2ea89911ed5457ee5b 100644 (file)
--- a/arch/s390/include/asm/entry-common.h
+++ b/arch/s390/include/asm/entry-common.h
@@ -59,4 +59,14 @@ static inline void arch_exit_to_user_mode_prepare(struct pt_regs *regs,
 
 #define arch_exit_to_user_mode_prepare arch_exit_to_user_mode_prepare
 
+static __always_inline bool arch_in_rcu_eqs(void)
+{
+       if (IS_ENABLED(CONFIG_KVM))
+               return current->flags & PF_VCPU;
+
+       return false;
+}
+
+#define arch_in_rcu_eqs arch_in_rcu_eqs
+
 #endif

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index cb89e54ada257eb4fdfe840ff37b2ea639c2d1cb..f870d09515cc12419eb0ef6495aabe90d005e22b 100644 (file)
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -716,6 +716,9 @@ extern char sie_exit;
 bool kvm_s390_pv_is_protected(struct kvm *kvm);
 bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu);
 
+extern int kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
+                                  u64 *gprs, unsigned long gasce);
+
 extern int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc);
 extern int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc);
 

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index d5ad10791c25fa0939d61b3c4187a108b1ded1b1..bfe9ba5c4f45197171bfc85d00ec7287d7a4e7b5 100644 (file)
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -5062,6 +5062,30 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
        return vcpu_post_run_handle_fault(vcpu);
 }
 
+int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
+                                   u64 *gprs, unsigned long gasce)
+{
+       int ret;
+
+       guest_state_enter_irqoff();
+
+       /*
+        * The guest_state_{enter,exit}_irqoff() functions inform lockdep and
+        * tracing that entry to the guest will enable host IRQs, and exit from
+        * the guest will disable host IRQs.
+        *
+        * We must not use lockdep/tracing/RCU in this critical section, so we
+        * use the low-level arch_local_irq_*() helpers to enable/disable IRQs.
+        */
+       arch_local_irq_enable();
+       ret = sie64a(scb, gprs, gasce);
+       arch_local_irq_disable();
+
+       guest_state_exit_irqoff();
+
+       return ret;
+}
+
 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
 static int __vcpu_run(struct kvm_vcpu *vcpu)
 {
@@ -5082,20 +5106,27 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                kvm_vcpu_srcu_read_unlock(vcpu);
                /*
                 * As PF_VCPU will be used in fault handler, between
-                * guest_enter and guest_exit should be no uaccess.
+                * guest_timing_enter_irqoff and guest_timing_exit_irqoff
+                * should be no uaccess.
                 */
-               local_irq_disable();
-               guest_enter_irqoff();
-               __disable_cpu_timer_accounting(vcpu);
-               local_irq_enable();
                if (kvm_s390_pv_cpu_is_protected(vcpu)) {
                        memcpy(sie_page->pv_grregs,
                               vcpu->run->s.regs.gprs,
                               sizeof(sie_page->pv_grregs));
                }
-               exit_reason = sie64a(vcpu->arch.sie_block,
-                                    vcpu->run->s.regs.gprs,
-                                    vcpu->arch.gmap->asce);
+
+               local_irq_disable();
+               guest_timing_enter_irqoff();
+               __disable_cpu_timer_accounting(vcpu);
+
+               exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
+                                                     vcpu->run->s.regs.gprs,
+                                                     vcpu->arch.gmap->asce);
+
+               __enable_cpu_timer_accounting(vcpu);
+               guest_timing_exit_irqoff();
+               local_irq_enable();
+
                if (kvm_s390_pv_cpu_is_protected(vcpu)) {
                        memcpy(vcpu->run->s.regs.gprs,
                               sie_page->pv_grregs,
@@ -5111,10 +5142,6 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
                                vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
                        }
                }
-               local_irq_disable();
-               __enable_cpu_timer_accounting(vcpu);
-               guest_exit_irqoff();
-               local_irq_enable();
                kvm_vcpu_srcu_read_lock(vcpu);
 
                rc = vcpu_post_run(vcpu, exit_reason);

diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 13a9661d2b28b9039a4a4626db22c4d501060c00..347268f89f2f186bea623a3adff7376cabc305b2 100644 (file)
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -1170,10 +1170,6 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
            vcpu->arch.sie_block->fpf & FPF_BPBC)
                set_thread_flag(TIF_ISOLATE_BP_GUEST);
 
-       local_irq_disable();
-       guest_enter_irqoff();
-       local_irq_enable();
-
        /*
         * Simulate a SIE entry of the VCPU (see sie64a), so VCPU blocking
         * and VCPU requests also hinder the vSIE from running and lead
@@ -1183,15 +1179,16 @@ static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
        vcpu->arch.sie_block->prog0c |= PROG_IN_SIE;
        current->thread.gmap_int_code = 0;
        barrier();
-       if (!kvm_s390_vcpu_sie_inhibited(vcpu))
-               rc = sie64a(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce);
+       if (!kvm_s390_vcpu_sie_inhibited(vcpu)) {
+               local_irq_disable();
+               guest_timing_enter_irqoff();
+               rc = kvm_s390_enter_exit_sie(scb_s, vcpu->run->s.regs.gprs, vsie_page->gmap->asce);
+               guest_timing_exit_irqoff();
+               local_irq_enable();
+       }
        barrier();
        vcpu->arch.sie_block->prog0c &= ~PROG_IN_SIE;
 
-       local_irq_disable();
-       guest_exit_irqoff();
-       local_irq_enable();
-
        /* restore guest state for bp isolation override */
        if (!guest_bp_isolation)
                clear_thread_flag(TIF_ISOLATE_BP_GUEST);