KVM: arm64: Handle guest_memfd-backed guest page faults

Add arm64 architecture support for handling guest page faults on memory
slots backed by guest_memfd.

This change introduces a new function, gmem_abort(), which encapsulates
the fault handling logic specific to guest_memfd-backed memory. The
kvm_handle_guest_abort() entry point is updated to dispatch to
gmem_abort() when a fault occurs on a guest_memfd-backed memory slot (as
determined by kvm_slot_has_gmem()).

Until guest_memfd gains support for huge pages, the fault granule for
these memory regions is restricted to PAGE_SIZE.

Reviewed-by: Gavin Shan <gshan@redhat.com>
Reviewed-by: James Houghton <jthoughton@google.com>
Reviewed-by: Marc Zyngier <maz@kernel.org>
Signed-off-by: Fuad Tabba <tabba@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-ID: <20250729225455.670324-19-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c
index 6ed69d309f4543caf3853b1720d2e348f1389f3c..5d73abc9bebc48b0680d573d21979df67b4267aa 100644 (file)
--- a/arch/arm64/kvm/mmu.c
+++ b/arch/arm64/kvm/mmu.c
@@ -1519,6 +1519,82 @@ static void adjust_nested_fault_perms(struct kvm_s2_trans *nested,
        *prot |= kvm_encode_nested_level(nested);
 }
 
+#define KVM_PGTABLE_WALK_MEMABORT_FLAGS (KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED)
+
+static int gmem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
+                     struct kvm_s2_trans *nested,
+                     struct kvm_memory_slot *memslot, bool is_perm)
+{
+       bool write_fault, exec_fault, writable;
+       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
+       enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
+       struct kvm_pgtable *pgt = vcpu->arch.hw_mmu->pgt;
+       unsigned long mmu_seq;
+       struct page *page;
+       struct kvm *kvm = vcpu->kvm;
+       void *memcache;
+       kvm_pfn_t pfn;
+       gfn_t gfn;
+       int ret;
+
+       ret = prepare_mmu_memcache(vcpu, true, &memcache);
+       if (ret)
+               return ret;
+
+       if (nested)
+               gfn = kvm_s2_trans_output(nested) >> PAGE_SHIFT;
+       else
+               gfn = fault_ipa >> PAGE_SHIFT;
+
+       write_fault = kvm_is_write_fault(vcpu);
+       exec_fault = kvm_vcpu_trap_is_exec_fault(vcpu);
+
+       VM_WARN_ON_ONCE(write_fault && exec_fault);
+
+       mmu_seq = kvm->mmu_invalidate_seq;
+       /* Pairs with the smp_wmb() in kvm_mmu_invalidate_end(). */
+       smp_rmb();
+
+       ret = kvm_gmem_get_pfn(kvm, memslot, gfn, &pfn, &page, NULL);
+       if (ret) {
+               kvm_prepare_memory_fault_exit(vcpu, fault_ipa, PAGE_SIZE,
+                                             write_fault, exec_fault, false);
+               return ret;
+       }
+
+       writable = !(memslot->flags & KVM_MEM_READONLY);
+
+       if (nested)
+               adjust_nested_fault_perms(nested, &prot, &writable);
+
+       if (writable)
+               prot |= KVM_PGTABLE_PROT_W;
+
+       if (exec_fault ||
+           (cpus_have_final_cap(ARM64_HAS_CACHE_DIC) &&
+            (!nested || kvm_s2_trans_executable(nested))))
+               prot |= KVM_PGTABLE_PROT_X;
+
+       kvm_fault_lock(kvm);
+       if (mmu_invalidate_retry(kvm, mmu_seq)) {
+               ret = -EAGAIN;
+               goto out_unlock;
+       }
+
+       ret = KVM_PGT_FN(kvm_pgtable_stage2_map)(pgt, fault_ipa, PAGE_SIZE,
+                                                __pfn_to_phys(pfn), prot,
+                                                memcache, flags);
+
+out_unlock:
+       kvm_release_faultin_page(kvm, page, !!ret, writable);
+       kvm_fault_unlock(kvm);
+
+       if (writable && !ret)
+               mark_page_dirty_in_slot(kvm, memslot, gfn);
+
+       return ret != -EAGAIN ? ret : 0;
+}
+
 static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
                          struct kvm_s2_trans *nested,
                          struct kvm_memory_slot *memslot, unsigned long hva,
@@ -1544,7 +1620,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
        struct kvm_pgtable *pgt;
        struct page *page;
        vm_flags_t vm_flags;
-       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
+       enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_MEMABORT_FLAGS;
 
        if (fault_is_perm)
                fault_granule = kvm_vcpu_trap_get_perm_fault_granule(vcpu);
@@ -1989,8 +2065,12 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
        VM_WARN_ON_ONCE(kvm_vcpu_trap_is_permission_fault(vcpu) &&
                        !write_fault && !kvm_vcpu_trap_is_exec_fault(vcpu));
 
-       ret = user_mem_abort(vcpu, fault_ipa, nested, memslot, hva,
-                            esr_fsc_is_permission_fault(esr));
+       if (kvm_slot_has_gmem(memslot))
+               ret = gmem_abort(vcpu, fault_ipa, nested, memslot,
+                                esr_fsc_is_permission_fault(esr));
+       else
+               ret = user_mem_abort(vcpu, fault_ipa, nested, memslot, hva,
+                                    esr_fsc_is_permission_fault(esr));
        if (ret == 0)
                ret = 1;
 out: