vfio: refactor vfio_pci_mmap_huge_fault function

Refactor vfio_pci_mmap_huge_fault to take out the implementation
to map the VMA to the PTE/PMD/PUD as a separate function.

Export the new function to be used by nvgrace-gpu module.

Move the alignment check code to verify that pfn and VMA VA is
aligned to the page order to the header file and make it inline.

No functional change is intended.

Cc: Shameer Kolothum <skolothumtho@nvidia.com>
Cc: Alex Williamson <alex@shazbot.org>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Reviewed-by: Shameer Kolothum <skolothumtho@nvidia.com>
Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
Link: https://lore.kernel.org/r/20251127170632.3477-2-ankita@nvidia.com
Signed-off-by: Alex Williamson <alex@shazbot.org>

diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 2b01bfbce3eae037b6036b43d5b73ae981caf0f0..9bb700d25ccbed87bf33cfb1816662209f675d41 100644 (file)
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -1652,49 +1652,49 @@ static unsigned long vma_to_pfn(struct vm_area_struct *vma)
        return (pci_resource_start(vdev->pdev, index) >> PAGE_SHIFT) + pgoff;
 }
 
-static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
-                                          unsigned int order)
+vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
+                                  struct vm_fault *vmf,
+                                  unsigned long pfn,
+                                  unsigned int order)
 {
-       struct vm_area_struct *vma = vmf->vma;
-       struct vfio_pci_core_device *vdev = vma->vm_private_data;
-       unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1);
-       unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
-       unsigned long pfn = vma_to_pfn(vma) + pgoff;
-       vm_fault_t ret = VM_FAULT_SIGBUS;
-
-       if (order && (addr < vma->vm_start ||
-                     addr + (PAGE_SIZE << order) > vma->vm_end ||
-                     pfn & ((1 << order) - 1))) {
-               ret = VM_FAULT_FALLBACK;
-               goto out;
-       }
-
-       down_read(&vdev->memory_lock);
+       lockdep_assert_held_read(&vdev->memory_lock);
 
        if (vdev->pm_runtime_engaged || !__vfio_pci_memory_enabled(vdev))
-               goto out_unlock;
+               return VM_FAULT_SIGBUS;
 
        switch (order) {
        case 0:
-               ret = vmf_insert_pfn(vma, vmf->address, pfn);
-               break;
+               return vmf_insert_pfn(vmf->vma, vmf->address, pfn);
 #ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
        case PMD_ORDER:
-               ret = vmf_insert_pfn_pmd(vmf, pfn, false);
-               break;
+               return vmf_insert_pfn_pmd(vmf, pfn, false);
 #endif
 #ifdef CONFIG_ARCH_SUPPORTS_PUD_PFNMAP
        case PUD_ORDER:
-               ret = vmf_insert_pfn_pud(vmf, pfn, false);
+               return vmf_insert_pfn_pud(vmf, pfn, false);
                break;
 #endif
        default:
-               ret = VM_FAULT_FALLBACK;
+               return VM_FAULT_FALLBACK;
+       }
+}
+EXPORT_SYMBOL_GPL(vfio_pci_vmf_insert_pfn);
+
+static vm_fault_t vfio_pci_mmap_huge_fault(struct vm_fault *vmf,
+                                          unsigned int order)
+{
+       struct vm_area_struct *vma = vmf->vma;
+       struct vfio_pci_core_device *vdev = vma->vm_private_data;
+       unsigned long addr = vmf->address & ~((PAGE_SIZE << order) - 1);
+       unsigned long pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
+       unsigned long pfn = vma_to_pfn(vma) + pgoff;
+       vm_fault_t ret = VM_FAULT_FALLBACK;
+
+       if (is_aligned_for_order(vma, addr, pfn, order)) {
+               scoped_guard(rwsem_read, &vdev->memory_lock)
+                       ret = vfio_pci_vmf_insert_pfn(vdev, vmf, pfn, order);
        }
 
-out_unlock:
-       up_read(&vdev->memory_lock);
-out:
        dev_dbg_ratelimited(&vdev->pdev->dev,
                           "%s(,order = %d) BAR %ld page offset 0x%lx: 0x%x\n",
                            __func__, order,

diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index a1eddd55dab888dd9eb6e58e7629644378659cb1..5569488ec4dcb998fcf9906d146226d6ce959c8a 100644 (file)
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -170,6 +170,9 @@ ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
                size_t count, loff_t *ppos);
 ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
                size_t count, loff_t *ppos);
+vm_fault_t vfio_pci_vmf_insert_pfn(struct vfio_pci_core_device *vdev,
+                                  struct vm_fault *vmf, unsigned long pfn,
+                                  unsigned int order);
 int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);
 void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
 int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);
@@ -212,4 +215,14 @@ VFIO_IOREAD_DECLARATION(32)
 VFIO_IOREAD_DECLARATION(64)
 #endif
 
+static inline bool is_aligned_for_order(struct vm_area_struct *vma,
+                                       unsigned long addr,
+                                       unsigned long pfn,
+                                       unsigned int order)
+{
+       return !(order && (addr < vma->vm_start ||
+                          addr + (PAGE_SIZE << order) > vma->vm_end ||
+                          !IS_ALIGNED(pfn, 1 << order)));
+}
+
 #endif /* VFIO_PCI_CORE_H */