Fix a potential race between decrementing an LPI's reference count and
evicting that structure from the LPI xarray.
LPI structures are maintained in the VGIC LPI xarray (dist->lpi_xa).
When the reference count of an LPI structure drops to zero,
vgic_release_lpi_locked() removes the structure from the xarray and
frees it under the xarray lock.
However, the release of an LPI can race with a concurrent LPI
re-registration with the same INTID via vgic_add_lpi() on another CPU,
since the reference count drop and the xarray eviction are not performed
in a single atomic step. This can happen e.g. if the guest issues a
DISCARD while the LPI is still referenced from a vCPU's active-pending
list (ap_list), and the same INTID is re-mapped via MAPTI.
Particularly, vgic_release_lpi_locked() is called from two distinct
paths: direct release via vgic_put_irq(), and deferred release via
vgic_release_deleted_lpis(). During direct release, the issue can result
in deleting a newly registered LPI from the xarray:
CPU0 (Releasing LPI) CPU1 (Adding new LPI)
==================== =====================
vgic_put_irq()
__vgic_put_irq()
refcount_dec_and_test()
vgic_add_lpi()
xa_lock_irqsave()
old_irq = xa_load(.., intid)
vgic_try_get_irq_ref(old_irq) == false
new IRQ inserted --> __xa_store(.., intid, ..)
xa_unlock_irqrestore()
xa_lock_irqsave();
vgic_release_lpi_locked()
__xa_erase(.., irq->intid) <-- BUG: new IRQ is erased
kfree_rcu(old_irq)
During the deferred release path, the old IRQ can be leaked:
CPU0 (Releasing LPI) CPU1 (Adding new LPI)
==================== =====================
vgic_put_irq_norelease()
__vgic_put_irq()
refcount_dec_and_test()
irq->pending_release = true
vgic_add_lpi()
xa_lock_irqsave()
old_irq = xa_load(.., intid)
vgic_try_get_irq_ref(oldirq) == false
BUG: old IRQ overwritten --> __xa_store(.., intid, ..)
xa_unlock_irqrestore()
vgic_release_deleted_lpis()
xa_lock_irqsave()
xa_for_each() { .. } <-- old IRQ with pending_release = true
is gone, so it cannot be released
To fix the direct release path, move the reference count drop inside
the xarray lock, making sure that vgic_add_lpi() never encounters the
to-be-released LPI.
In the deferred release path, the refcount drop must happen under a raw
spinlock, so the xarray lock cannot be grabbed, and the same solution
does not work. Instead, update vgic_add_lpi(), so that if it evicts
an LPI from the xarray, it takes on the responsibility of freeing it.
Consequently, an LPI may now be freed concurrently after a deferred
release drops the refcount, so accessing the pending_release field is no
longer safe from use-after-free. Delete all uses of the flag, and update
vgic_release_deleted_lpis() to identify orphaned LPIs purely based on
their refcount.
Reported-by: Claude:claude-opus-4-6
Fixes: 3a08a6ca7c37 ("KVM: arm64: vgic-v3: Use bare refcount for VGIC LPIs")
Fixes: d54594accf73 ("KVM: arm64: vgic-v3: Erase LPIs from xarray outside of raw spinlocks")
Signed-off-by: Carlos López <clopez@suse.de>
Link: https://patch.msgid.link/20260715105137.3973823-4-clopez@suse.de
Signed-off-by: Marc Zyngier <maz@kernel.org>
kfree(irq);
irq = oldirq;
} else {
- ret = xa_err(__xa_store(&dist->lpi_xa, intid, irq, 0));
- }
-
- xa_unlock_irqrestore(&dist->lpi_xa, flags);
+ /*
+ * The entry is either empty or contains a dead LPI (refcount=0)
+ * from the deferred release path, pending cleanup by
+ * vgic_release_deleted_lpis(). Evict and free it if present.
+ */
+ oldirq = __xa_store(&dist->lpi_xa, intid, irq, 0);
+ ret = xa_err(oldirq);
+ if (ret) {
+ xa_unlock_irqrestore(&dist->lpi_xa, flags);
+ kfree(irq);
- if (ret) {
- xa_release(&dist->lpi_xa, intid);
- kfree(irq);
+ return ERR_PTR(ret);
+ }
- return ERR_PTR(ret);
+ if (oldirq && !WARN_ON_ONCE(refcount_read(&oldirq->refcount)))
+ kfree_rcu(oldirq, rcu);
}
+ xa_unlock_irqrestore(&dist->lpi_xa, flags);
+
/*
* We "cache" the configuration table entries in our struct vgic_irq's.
* However we only have those structs for mapped IRQs, so we read in
static __must_check bool vgic_put_irq_norelease(struct kvm *kvm, struct vgic_irq *irq)
{
- if (!__vgic_put_irq(kvm, irq))
- return false;
-
- irq->pending_release = true;
- return true;
+ return __vgic_put_irq(kvm, irq);
}
void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)
guard(spinlock_irqsave)(&dist->lpi_xa.xa_lock);
}
- if (!__vgic_put_irq(kvm, irq))
+ if (!irq_is_lpi(kvm, irq->intid))
return;
- xa_lock_irqsave(&dist->lpi_xa, flags);
- vgic_release_lpi_locked(dist, irq);
- xa_unlock_irqrestore(&dist->lpi_xa, flags);
+ if (refcount_dec_and_lock_irqsave(&irq->refcount,
+ &dist->lpi_xa.xa_lock, &flags)) {
+ vgic_release_lpi_locked(dist, irq);
+ xa_unlock_irqrestore(&dist->lpi_xa, flags);
+ }
}
static void vgic_release_deleted_lpis(struct kvm *kvm)
xa_lock_irqsave(&dist->lpi_xa, flags);
xa_for_each(&dist->lpi_xa, intid, irq) {
- if (irq->pending_release)
+ if (!refcount_read(&irq->refcount))
vgic_release_lpi_locked(dist, irq);
}
* affinity reg (v3).
*/
- bool pending_release:1; /* Used for LPIs only, unreferenced IRQ
- * pending a release */
-
bool pending_latch:1; /* The pending latch state used to calculate
* the pending state for both level
* and edge triggered IRQs. */
projects / thirdparty / linux.git / commitdiff
