KVM: arm64: Invert KVM_PGTABLE_WALK_HANDLE_FAULT to fix pKVM walkers

Commit ddcadb297ce5 ("KVM: arm64: Ignore EAGAIN for walks outside of a
fault") introduced a new walker flag ('KVM_PGTABLE_WALK_HANDLE_FAULT')
to KVM's page-table code. When set, the walk logic maintains its
previous behaviour of terminating a walk as soon as the visitor callback
returns an error. However, when the flag is clear, the walk will
continue if the visitor returns -EAGAIN and the error is then suppressed
and returned as zero to the caller.

Clearing the flag is beneficial when write-protecting a range of IPAs
with kvm_pgtable_stage2_wrprotect() but is not useful in any other
cases, either because we are operating on a single page (e.g.
kvm_pgtable_stage2_mkyoung() or kvm_phys_addr_ioremap()) or because the
early termination is desirable (e.g. when mapping pages from a fault in
user_mem_abort()).

Subsequently, commit e912efed485a ("KVM: arm64: Introduce the EL1 pKVM
MMU") hooked up pKVM's hypercall interface to the MMU code at EL1 but
failed to propagate any of the walker flags. As a result, page-table
walks at EL2 fail to set KVM_PGTABLE_WALK_HANDLE_FAULT even when the
early termination semantics are desirable on the fault handling path.

Rather than complicate the pKVM hypercall interface, invert the flag so
that the whole thing can be simplified and only pass the new flag
('KVM_PGTABLE_WALK_IGNORE_EAGAIN') from the wrprotect code.

Cc: Fuad Tabba <tabba@google.com>
Cc: Quentin Perret <qperret@google.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Oliver Upton <oupton@kernel.org>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Fixes: fce886a60207 ("KVM: arm64: Plumb the pKVM MMU in KVM")
Signed-off-by: Will Deacon <will@kernel.org>
Reviewed-by: Quentin Perret <qperret@google.com>
Link: https://msgid.link/20260105154939.11041-2-will@kernel.org
Signed-off-by: Oliver Upton <oupton@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index be68b89692065a2c56972f6a7ec0c025d8eeda99..c0ad262a828902115e054c2cdfb4b1e66ea80b8f 100644 (file)
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -301,8 +301,8 @@ typedef bool (*kvm_pgtable_force_pte_cb_t)(u64 addr, u64 end,
  *                                     children.
  * @KVM_PGTABLE_WALK_SHARED:           Indicates the page-tables may be shared
  *                                     with other software walkers.
- * @KVM_PGTABLE_WALK_HANDLE_FAULT:     Indicates the page-table walk was
- *                                     invoked from a fault handler.
+ * @KVM_PGTABLE_WALK_IGNORE_EAGAIN:    Don't terminate the walk early if
+ *                                     the walker returns -EAGAIN.
  * @KVM_PGTABLE_WALK_SKIP_BBM_TLBI:    Visit and update table entries
  *                                     without Break-before-make's
  *                                     TLB invalidation.
@@ -315,7 +315,7 @@ enum kvm_pgtable_walk_flags {
        KVM_PGTABLE_WALK_TABLE_PRE              = BIT(1),
        KVM_PGTABLE_WALK_TABLE_POST             = BIT(2),
        KVM_PGTABLE_WALK_SHARED                 = BIT(3),
-       KVM_PGTABLE_WALK_HANDLE_FAULT           = BIT(4),
+       KVM_PGTABLE_WALK_IGNORE_EAGAIN          = BIT(4),
        KVM_PGTABLE_WALK_SKIP_BBM_TLBI          = BIT(5),
        KVM_PGTABLE_WALK_SKIP_CMO               = BIT(6),
 };

diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 947ac1a951a5b0f1c9b44339d69f7074cbfbd0a4..9abc0a6cf448109aba081022bbed15aa9889db7d 100644 (file)
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -144,7 +144,7 @@ static bool kvm_pgtable_walk_continue(const struct kvm_pgtable_walker *walker,
         * page table walk.
         */
        if (r == -EAGAIN)
-               return !(walker->flags & KVM_PGTABLE_WALK_HANDLE_FAULT);
+               return walker->flags & KVM_PGTABLE_WALK_IGNORE_EAGAIN;
 
        return !r;
 }
@@ -1262,7 +1262,8 @@ int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size)
 {
        return stage2_update_leaf_attrs(pgt, addr, size, 0,
                                        KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W,
-                                       NULL, NULL, 0);
+                                       NULL, NULL,
+                                       KVM_PGTABLE_WALK_IGNORE_EAGAIN);
 }
 
 void kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr,

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 124404eb208d3b5af06fdee7415a80caff6a565f..2caa97f87890fd5ef1203ebaeed71c9106761ca2 100644 (file)
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1563,14 +1563,12 @@ static void adjust_nested_exec_perms(struct kvm *kvm,
                *prot &= ~KVM_PGTABLE_PROT_PX;
 }
 
-#define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED)
-
 static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                      struct kvm_s2_trans *nested,
                      struct kvm_memory_slot *memslot, bool is_perm)
 {
        bool write_fault, exec_fault, writable;
-       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
+       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
        enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
        struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;
        unsigned long mmu_seq;
@@ -1665,7 +1663,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
        struct kvm_pgtable *pgt;
        struct page *page;
        vm_flags_t vm_flags;
-       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
+       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
 
        if (fault_is_perm)
                fault_granule = kvm_vcpu_trap_get_perm_fault_granule(vcpu);
@@ -1933,7 +1931,7 @@ out_unlock:
 /* Resolve the access fault by making the page young again. */
 static void handle_access_fault(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
 {
-       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
+       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_SHARED;
        struct kvm_s2_mmu *mmu;
 
        trace_kvm_access_fault(fault_ipa);