KVM: arm64: gic-v5: Implement PPI interrupt injection

This change introduces interrupt injection for PPIs for GICv5-based
guests.

The lifecycle of PPIs is largely managed by the hardware for a GICv5
system. The hypervisor injects pending state into the guest by using
the ICH_PPI_PENDRx_EL2 registers. These are used by the hardware to
pick a Highest Priority Pending Interrupt (HPPI) for the guest based
on the enable state of each individual interrupt. The enable state and
priority for each interrupt are provided by the guest itself (through
writes to the PPI registers).

When Direct Virtual Interrupt (DVI) is set for a particular PPI, the
hypervisor is even able to skip the injection of the pending state
altogether - it all happens in hardware.

The result of the above is that no AP lists are required for GICv5,
unlike for older GICs. Instead, for PPIs the ICH_PPI_* registers
fulfil the same purpose for all 128 PPIs. Hence, as long as the
ICH_PPI_* registers are populated prior to guest entry, and merged
back into the KVM shadow state on exit, the PPI state is preserved,
and interrupts can be injected.

When injecting the state of a PPI the state is merged into the
PPI-specific vgic_irq structure. The PPIs are made pending via the
ICH_PPI_PENDRx_EL2 registers, the value of which is generated from the
vgic_irq structures for each PPI exposed on guest entry. The
queue_irq_unlock() irq_op is required to kick the vCPU to ensure that
it seems the new state. The result is that no AP lists are used for
private interrupts on GICv5.

Prior to entering the guest, vgic_v5_flush_ppi_state() is called from
kvm_vgic_flush_hwstate(). This generates the pending state to inject
into the guest, and snapshots it (twice - an entry and an exit copy)
in order to track any changes. These changes can come from a guest
consuming an interrupt or from a guest making an Edge-triggered
interrupt pending.

When returning from running a guest, the guest's PPI state is merged
back into KVM's vgic_irq state in vgic_v5_merge_ppi_state() from
kvm_vgic_sync_hwstate(). The Enable and Active state is synced back for
all PPIs, and the pending state is synced back for Edge PPIs (Level is
driven directly by the devices generating said levels). The incoming
pending state from the guest is merged with KVM's shadow state to
avoid losing any incoming interrupts.

Signed-off-by: Sascha Bischoff <sascha.bischoff@arm.com>
Reviewed-by: Jonathan Cameron <jonathan.cameron@huawei.com>
Link: https://patch.msgid.link/20260319154937.3619520-21-sascha.bischoff@arm.com
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c
index 07f416fbc4bc853eaba0bd5013610afe80f0d5fd..014d6fc1d44ac636c0b62fae52a232d0b0c92420 100644 (file)
--- a/arch/arm64/kvm/vgic/vgic-v5.c
+++ b/arch/arm64/kvm/vgic/vgic-v5.c
@@ -122,6 +122,143 @@ int vgic_v5_finalize_ppi_state(struct kvm *kvm)
        return 0;
 }
 
+/*
+ * For GICv5, the PPIs are mostly directly managed by the hardware. We (the
+ * hypervisor) handle the pending, active, enable state save/restore, but don't
+ * need the PPIs to be queued on a per-VCPU AP list. Therefore, sanity check the
+ * state, unlock, and return.
+ */
+static bool vgic_v5_ppi_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
+                                        unsigned long flags)
+       __releases(&irq->irq_lock)
+{
+       struct kvm_vcpu *vcpu;
+
+       lockdep_assert_held(&irq->irq_lock);
+
+       if (WARN_ON_ONCE(!__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, irq->intid)))
+               goto out_unlock_fail;
+
+       vcpu = irq->target_vcpu;
+       if (WARN_ON_ONCE(!vcpu))
+               goto out_unlock_fail;
+
+       raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+
+       /* Directly kick the target VCPU to make sure it sees the IRQ */
+       kvm_make_request(KVM_REQ_IRQ_PENDING, vcpu);
+       kvm_vcpu_kick(vcpu);
+
+       return true;
+
+out_unlock_fail:
+       raw_spin_unlock_irqrestore(&irq->irq_lock, flags);
+
+       return false;
+}
+
+static struct irq_ops vgic_v5_ppi_irq_ops = {
+       .queue_irq_unlock = vgic_v5_ppi_queue_irq_unlock,
+};
+
+void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid)
+{
+       kvm_vgic_set_irq_ops(vcpu, vintid, &vgic_v5_ppi_irq_ops);
+}
+
+/*
+ * Detect any PPIs state changes, and propagate the state with KVM's
+ * shadow structures.
+ */
+void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu)
+{
+       struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;
+       DECLARE_BITMAP(changed_active, VGIC_V5_NR_PRIVATE_IRQS);
+       DECLARE_BITMAP(changed_pending, VGIC_V5_NR_PRIVATE_IRQS);
+       DECLARE_BITMAP(changed_bits, VGIC_V5_NR_PRIVATE_IRQS);
+       unsigned long *activer, *pendr_entry, *pendr;
+       int i;
+
+       activer = host_data_ptr(vgic_v5_ppi_state)->activer_exit;
+       pendr_entry = host_data_ptr(vgic_v5_ppi_state)->pendr_entry;
+       pendr = host_data_ptr(vgic_v5_ppi_state)->pendr_exit;
+
+       bitmap_xor(changed_active, cpu_if->vgic_ppi_activer, activer,
+                  VGIC_V5_NR_PRIVATE_IRQS);
+       bitmap_xor(changed_pending, pendr_entry, pendr,
+                  VGIC_V5_NR_PRIVATE_IRQS);
+       bitmap_or(changed_bits, changed_active, changed_pending,
+                 VGIC_V5_NR_PRIVATE_IRQS);
+
+       for_each_set_bit(i, changed_bits, VGIC_V5_NR_PRIVATE_IRQS) {
+               u32 intid = vgic_v5_make_ppi(i);
+               struct vgic_irq *irq;
+
+               irq = vgic_get_vcpu_irq(vcpu, intid);
+
+               scoped_guard(raw_spinlock_irqsave, &irq->irq_lock) {
+                       irq->active = test_bit(i, activer);
+
+                       /* This is an OR to avoid losing incoming edges! */
+                       if (irq->config == VGIC_CONFIG_EDGE)
+                               irq->pending_latch |= test_bit(i, pendr);
+               }
+
+               vgic_put_irq(vcpu->kvm, irq);
+       }
+
+       /*
+        * Re-inject the exit state as entry state next time!
+        *
+        * Note that the write of the Enable state is trapped, and hence there
+        * is nothing to explcitly sync back here as we already have the latest
+        * copy by definition.
+        */
+       bitmap_copy(cpu_if->vgic_ppi_activer, activer, VGIC_V5_NR_PRIVATE_IRQS);
+}
+
+void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu)
+{
+       DECLARE_BITMAP(pendr, VGIC_V5_NR_PRIVATE_IRQS);
+       int i;
+
+       /*
+        * Time to enter the guest - we first need to build the guest's
+        * ICC_PPI_PENDRx_EL1, however.
+        */
+       bitmap_zero(pendr, VGIC_V5_NR_PRIVATE_IRQS);
+       for_each_set_bit(i, vcpu->kvm->arch.vgic.gicv5_vm.vgic_ppi_mask,
+                        VGIC_V5_NR_PRIVATE_IRQS) {
+               u32 intid = vgic_v5_make_ppi(i);
+               struct vgic_irq *irq;
+
+               irq = vgic_get_vcpu_irq(vcpu, intid);
+
+               scoped_guard(raw_spinlock_irqsave, &irq->irq_lock)
+                       __assign_bit(i, pendr, irq_is_pending(irq));
+
+               vgic_put_irq(vcpu->kvm, irq);
+       }
+
+       /*
+        * Copy the shadow state to the pending reg that will be written to the
+        * ICH_PPI_PENDRx_EL2 regs. While the guest is running we track any
+        * incoming changes to the pending state in the vgic_irq structures. The
+        * incoming changes are merged with the outgoing changes on the return
+        * path.
+        */
+       bitmap_copy(host_data_ptr(vgic_v5_ppi_state)->pendr_entry, pendr,
+                   VGIC_V5_NR_PRIVATE_IRQS);
+
+       /*
+        * Make sure that we can correctly detect "edges" in the PPI
+        * state. There's a path where we never actually enter the guest, and
+        * failure to do this risks losing pending state
+        */
+       bitmap_copy(host_data_ptr(vgic_v5_ppi_state)->pendr_exit, pendr,
+                   VGIC_V5_NR_PRIVATE_IRQS);
+}
+
 void vgic_v5_load(struct kvm_vcpu *vcpu)
 {
        struct vgic_v5_cpu_if *cpu_if = &vcpu->arch.vgic_cpu.vgic_v5;

diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index c46c0e1db436e06e155af2ff09ded26098579cc2..485a9a3fab8dc47698c68e1165a6c8cb8671e717 100644 (file)
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -105,6 +105,18 @@ struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid)
        if (WARN_ON(!vcpu))
                return NULL;
 
+       if (vgic_is_v5(vcpu->kvm)) {
+               u32 int_num, hwirq_id;
+
+               if (!__irq_is_ppi(KVM_DEV_TYPE_ARM_VGIC_V5, intid))
+                       return NULL;
+
+               hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, intid);
+               int_num = array_index_nospec(hwirq_id, VGIC_V5_NR_PRIVATE_IRQS);
+
+               return &vcpu->arch.vgic_cpu.private_irqs[int_num];
+       }
+
        /* SGIs and PPIs */
        if (intid < VGIC_NR_PRIVATE_IRQS) {
                intid = array_index_nospec(intid, VGIC_NR_PRIVATE_IRQS);
@@ -830,8 +842,13 @@ retry:
                vgic_release_deleted_lpis(vcpu->kvm);
 }
 
-static inline void vgic_fold_lr_state(struct kvm_vcpu *vcpu)
+static void vgic_fold_state(struct kvm_vcpu *vcpu)
 {
+       if (vgic_is_v5(vcpu->kvm)) {
+               vgic_v5_fold_ppi_state(vcpu);
+               return;
+       }
+
        if (!*host_data_ptr(last_lr_irq))
                return;
 
@@ -1046,8 +1063,10 @@ void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu)
        if (can_access_vgic_from_kernel())
                vgic_save_state(vcpu);
 
-       vgic_fold_lr_state(vcpu);
-       vgic_prune_ap_list(vcpu);
+       vgic_fold_state(vcpu);
+
+       if (!vgic_is_v5(vcpu->kvm))
+               vgic_prune_ap_list(vcpu);
 }
 
 /* Sync interrupts that were deactivated through a DIR trap */
@@ -1080,6 +1099,17 @@ static inline void vgic_restore_state(struct kvm_vcpu *vcpu)
                __vgic_v3_restore_state(&vcpu->arch.vgic_cpu.vgic_v3);
 }
 
+static void vgic_flush_state(struct kvm_vcpu *vcpu)
+{
+       if (vgic_is_v5(vcpu->kvm)) {
+               vgic_v5_flush_ppi_state(vcpu);
+               return;
+       }
+
+       scoped_guard(raw_spinlock, &vcpu->arch.vgic_cpu.ap_list_lock)
+               vgic_flush_lr_state(vcpu);
+}
+
 /* Flush our emulation state into the GIC hardware before entering the guest. */
 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 {
@@ -1116,13 +1146,12 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
 
        DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
 
-       scoped_guard(raw_spinlock, &vcpu->arch.vgic_cpu.ap_list_lock)
-               vgic_flush_lr_state(vcpu);
+       vgic_flush_state(vcpu);
 
        if (can_access_vgic_from_kernel())
                vgic_restore_state(vcpu);
 
-       if (vgic_supports_direct_irqs(vcpu->kvm))
+       if (vgic_supports_direct_irqs(vcpu->kvm) && kvm_vgic_global_state.has_gicv4)
                vgic_v4_commit(vcpu);
 }
 

diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h
index cc487a69d038989c12ccfc089066d8d5d2dc576e..d90af676d5d065fcd1383dd39c0622944d7afd8b 100644 (file)
--- a/arch/arm64/kvm/vgic/vgic.h
+++ b/arch/arm64/kvm/vgic/vgic.h
@@ -364,6 +364,9 @@ void vgic_debug_init(struct kvm *kvm);
 void vgic_debug_destroy(struct kvm *kvm);
 
 int vgic_v5_probe(const struct gic_kvm_info *info);
+void vgic_v5_set_ppi_ops(struct kvm_vcpu *vcpu, u32 vintid);
+void vgic_v5_flush_ppi_state(struct kvm_vcpu *vcpu);
+void vgic_v5_fold_ppi_state(struct kvm_vcpu *vcpu);
 void vgic_v5_load(struct kvm_vcpu *vcpu);
 void vgic_v5_put(struct kvm_vcpu *vcpu);
 void vgic_v5_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
@@ -432,15 +435,6 @@ void vgic_its_invalidate_all_caches(struct kvm *kvm);
 int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
 int vgic_its_invall(struct kvm_vcpu *vcpu);
 
-bool system_supports_direct_sgis(void);
-bool vgic_supports_direct_msis(struct kvm *kvm);
-bool vgic_supports_direct_sgis(struct kvm *kvm);
-
-static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
-{
-       return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm);
-}
-
 int vgic_v4_init(struct kvm *kvm);
 void vgic_v4_teardown(struct kvm *kvm);
 void vgic_v4_configure_vsgis(struct kvm *kvm);
@@ -481,6 +475,19 @@ static inline bool vgic_host_has_gicv5(void)
        return kvm_vgic_global_state.type == VGIC_V5;
 }
 
+bool system_supports_direct_sgis(void);
+bool vgic_supports_direct_msis(struct kvm *kvm);
+bool vgic_supports_direct_sgis(struct kvm *kvm);
+
+static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
+{
+       /* GICv5 always supports direct IRQs */
+       if (vgic_is_v5(kvm))
+               return true;
+
+       return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm);
+}
+
 int vgic_its_debug_init(struct kvm_device *dev);
 void vgic_its_debug_destroy(struct kvm_device *dev);