drm/amdgpu: use hmm_pfns instead of array of pages

we dont need to allocate local array of pages to hold
the pages returned by the hmm, instead we could use
the hmm_range structure itself to get to hmm_pfn
and get the required pages directly.

This avoids call to alloc/free quite a lot.

Signed-off-by: Sunil Khatri <sunil.khatri@amd.com>
Suggested-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Acked-by: Felix Kuehling <felix.kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index c3b34a4103751743d32a1f588bf31541a5eb40f6..7c54fe6b0f5dba4049853e9171277077a1d06c74 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1089,7 +1089,7 @@ static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
                return 0;
        }
 
-       ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages, &range);
+       ret = amdgpu_ttm_tt_get_user_pages(bo, &range);
        if (ret) {
                if (ret == -EAGAIN)
                        pr_debug("Failed to get user pages, try again\n");
@@ -1103,6 +1103,9 @@ static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr,
                pr_err("%s: Failed to reserve BO\n", __func__);
                goto release_out;
        }
+
+       amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
+
        amdgpu_bo_placement_from_domain(bo, mem->domain);
        ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
        if (ret)
@@ -2565,8 +2568,7 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
                }
 
                /* Get updated user pages */
-               ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
-                                                  &mem->range);
+               ret = amdgpu_ttm_tt_get_user_pages(bo, &mem->range);
                if (ret) {
                        pr_debug("Failed %d to get user pages\n", ret);
 
@@ -2595,6 +2597,8 @@ static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
                        ret = 0;
                }
 
+               amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->range);
+
                mutex_lock(&process_info->notifier_lock);
 
                /* Mark the BO as valid unless it was invalidated

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
index 555cd6d877c301e2ecb2f8e19371fcb1e3cfd831..a716c9886c7407848b40a79ee202e1f85659460d 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.h
@@ -38,7 +38,6 @@ struct amdgpu_bo_list_entry {
        struct amdgpu_bo                *bo;
        struct amdgpu_bo_va             *bo_va;
        uint32_t                        priority;
-       struct page                     **user_pages;
        struct hmm_range                *range;
        bool                            user_invalidated;
 };

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index defb511acc5a04d5c5a31ae48db7c1dadcc33b7c..744e6ff698146be9036b5c7d9ed058dbb958e5b7 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -29,6 +29,7 @@
 #include <linux/pagemap.h>
 #include <linux/sync_file.h>
 #include <linux/dma-buf.h>
+#include <linux/hmm.h>
 
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_syncobj.h>
@@ -885,24 +886,12 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                struct amdgpu_bo *bo = e->bo;
                int i;
 
-               e->user_pages = kvcalloc(bo->tbo.ttm->num_pages,
-                                        sizeof(struct page *),
-                                        GFP_KERNEL);
-               if (!e->user_pages) {
-                       drm_err(adev_to_drm(p->adev), "kvmalloc_array failure\n");
-                       r = -ENOMEM;
-                       goto out_free_user_pages;
-               }
-
-               r = amdgpu_ttm_tt_get_user_pages(bo, e->user_pages, &e->range);
-               if (r) {
-                       kvfree(e->user_pages);
-                       e->user_pages = NULL;
+               r = amdgpu_ttm_tt_get_user_pages(bo, &e->range);
+               if (r)
                        goto out_free_user_pages;
-               }
 
                for (i = 0; i < bo->tbo.ttm->num_pages; i++) {
-                       if (bo->tbo.ttm->pages[i] != e->user_pages[i]) {
+                       if (bo->tbo.ttm->pages[i] != hmm_pfn_to_page(e->range->hmm_pfns[i])) {
                                userpage_invalidated = true;
                                break;
                        }
@@ -946,7 +935,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                }
 
                if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) &&
-                   e->user_invalidated && e->user_pages) {
+                   e->user_invalidated) {
                        amdgpu_bo_placement_from_domain(e->bo,
                                                        AMDGPU_GEM_DOMAIN_CPU);
                        r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement,
@@ -955,11 +944,8 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                                goto out_free_user_pages;
 
                        amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm,
-                                                    e->user_pages);
+                                                    e->range);
                }
-
-               kvfree(e->user_pages);
-               e->user_pages = NULL;
        }
 
        amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
@@ -1001,11 +987,7 @@ out_free_user_pages:
        amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
                struct amdgpu_bo *bo = e->bo;
 
-               if (!e->user_pages)
-                       continue;
                amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, e->range);
-               kvfree(e->user_pages);
-               e->user_pages = NULL;
                e->range = NULL;
        }
        mutex_unlock(&p->bo_list->bo_list_mutex);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 1679c2c3d5053fc993f66c120e00c22ac87af6ba..b7ebae289beabb4988d3f844139ce99513e17ed4 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -572,8 +572,7 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
                goto release_object;
 
        if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
-               r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
-                                                &range);
+               r = amdgpu_ttm_tt_get_user_pages(bo, &range);
                if (r)
                        goto release_object;
 
@@ -581,6 +580,8 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
                if (r)
                        goto user_pages_done;
 
+               amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
+
                amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
                r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
                amdgpu_bo_unreserve(bo);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
index e36fede7f74c3b2f3096721cd9d61bc7a023bdb2..2c6a6b858112f3167f947a53bd49c58f54b93d3d 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.c
@@ -167,13 +167,12 @@ void amdgpu_hmm_unregister(struct amdgpu_bo *bo)
 
 int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
                               uint64_t start, uint64_t npages, bool readonly,
-                              void *owner, struct page **pages,
+                              void *owner,
                               struct hmm_range **phmm_range)
 {
        struct hmm_range *hmm_range;
        unsigned long end;
        unsigned long timeout;
-       unsigned long i;
        unsigned long *pfns;
        int r = 0;
 
@@ -222,14 +221,6 @@ retry:
        hmm_range->start = start;
        hmm_range->hmm_pfns = pfns;
 
-       /*
-        * Due to default_flags, all pages are HMM_PFN_VALID or
-        * hmm_range_fault() fails. FIXME: The pages cannot be touched outside
-        * the notifier_lock, and mmu_interval_read_retry() must be done first.
-        */
-       for (i = 0; pages && i < npages; i++)
-               pages[i] = hmm_pfn_to_page(pfns[i]);
-
        *phmm_range = hmm_range;
 
        return 0;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h
index e2edcd010cccbf307f6b1d8c4d0cc9d24c4c70a5..953e1d06de20efc17bf5e7e830ebdfe7c832e1d6 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_hmm.h
@@ -33,7 +33,7 @@
 
 int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
                               uint64_t start, uint64_t npages, bool readonly,
-                              void *owner, struct page **pages,
+                              void *owner,
                               struct hmm_range **phmm_range);
 bool amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range);
 

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 71b6691edab42c77c56a1aa218a3c064a2a35542..aa9ee5dffa45514b0133e25a521b9a8b66e5ec3d 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -708,7 +708,7 @@ struct amdgpu_ttm_tt {
  * Calling function must call amdgpu_ttm_tt_userptr_range_done() once and only
  * once afterwards to stop HMM tracking
  */
-int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages,
+int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
                                 struct hmm_range **range)
 {
        struct ttm_tt *ttm = bo->tbo.ttm;
@@ -745,7 +745,7 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages,
 
        readonly = amdgpu_ttm_tt_is_readonly(ttm);
        r = amdgpu_hmm_range_get_pages(&bo->notifier, start, ttm->num_pages,
-                                      readonly, NULL, pages, range);
+                                      readonly, NULL, range);
 out_unlock:
        mmap_read_unlock(mm);
        if (r)
@@ -797,12 +797,12 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm,
  * that backs user memory and will ultimately be mapped into the device
  * address space.
  */
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages)
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct hmm_range *range)
 {
        unsigned long i;
 
        for (i = 0; i < ttm->num_pages; ++i)
-               ttm->pages[i] = pages ? pages[i] : NULL;
+               ttm->pages[i] = range ? hmm_pfn_to_page(range->hmm_pfns[i]) : NULL;
 }
 
 /*

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index 1c5c7836ce4f88328cdb6c1982825dab88f14578..0be2728aa872947ebecf9807fe6f482a2514a501 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -191,7 +191,7 @@ void amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo);
 uint64_t amdgpu_ttm_domain_start(struct amdgpu_device *adev, uint32_t type);
 
 #if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
-int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo, struct page **pages,
+int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
                                 struct hmm_range **range);
 void amdgpu_ttm_tt_discard_user_pages(struct ttm_tt *ttm,
                                      struct hmm_range *range);
@@ -199,7 +199,6 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm,
                                       struct hmm_range *range);
 #else
 static inline int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo *bo,
-                                              struct page **pages,
                                               struct hmm_range **range)
 {
        return -EPERM;
@@ -215,7 +214,7 @@ static inline bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm,
 }
 #endif
 
-void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct page **pages);
+void amdgpu_ttm_tt_set_user_pages(struct ttm_tt *ttm, struct hmm_range *range);
 int amdgpu_ttm_tt_get_userptr(const struct ttm_buffer_object *tbo,
                              uint64_t *user_addr);
 int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 68ba239b2e5db07c23f5210510013477a7e036a4..273f42e3afddf0816b60a8a7b7c5439d1ee8d800 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1738,7 +1738,7 @@ static int svm_range_validate_and_map(struct mm_struct *mm,
 
                        WRITE_ONCE(p->svms.faulting_task, current);
                        r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages,
-                                                      readonly, owner, NULL,
+                                                      readonly, owner,
                                                       &hmm_range);
                        WRITE_ONCE(p->svms.faulting_task, NULL);
                        if (r)