drm/amdgpu: use GFP_NOWAIT for memory allocations

[ Upstream commit 16590745b571c07869ef8958e0bbe44ab6f08d1f ]

In the critical submission path memory allocations can't wait for
reclaim since that can potentially wait for submissions to finish.

Finally clean that up and mark most memory allocations in the critical
path with GFP_NOWAIT. The only exception left is the dma_fence_array()
used when no VMID is available, but that will be cleaned up later on.

Signed-off-by: Christian König <christian.koenig@amd.com>
Acked-by: Srinivasan Shanmugam <srinivasan.shanmugam@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 1e998f972c308bb019c2b5131b7a2ff3752e2606..70224b9f54f2f4c9c80559e0c662a26b3942720c 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -499,7 +499,7 @@ static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
        if (ret)
                return ret;
 
-       return amdgpu_sync_fence(sync, vm->last_update);
+       return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL);
 }
 
 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
@@ -1263,7 +1263,7 @@ static int unmap_bo_from_gpuvm(struct kgd_mem *mem,
 
        (void)amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
 
-       (void)amdgpu_sync_fence(sync, bo_va->last_pt_update);
+       (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
 
        return 0;
 }
@@ -1287,7 +1287,7 @@ static int update_gpuvm_pte(struct kgd_mem *mem,
                return ret;
        }
 
-       return amdgpu_sync_fence(sync, bo_va->last_pt_update);
+       return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL);
 }
 
 static int map_bo_to_gpuvm(struct kgd_mem *mem,
@@ -2969,7 +2969,7 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu *
                }
                dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
                                        DMA_RESV_USAGE_KERNEL, fence) {
-                       ret = amdgpu_sync_fence(&sync_obj, fence);
+                       ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL);
                        if (ret) {
                                pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
                                goto validate_map_fail;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 5cc5f59e30184ff956823d8ff44040e211ec93eb..4a5b406601fa20c789edc9095ca1813aa78081c1 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -428,7 +428,7 @@ static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
                        dma_fence_put(old);
                }
 
-               r = amdgpu_sync_fence(&p->sync, fence);
+               r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
                dma_fence_put(fence);
                if (r)
                        return r;
@@ -450,7 +450,7 @@ static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
                return r;
        }
 
-       r = amdgpu_sync_fence(&p->sync, fence);
+       r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL);
        dma_fence_put(fence);
        return r;
 }
@@ -1124,7 +1124,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
        if (r)
                return r;
 
-       r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update);
+       r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update,
+                             GFP_KERNEL);
        if (r)
                return r;
 
@@ -1135,7 +1136,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
                if (r)
                        return r;
 
-               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
+               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
+                                     GFP_KERNEL);
                if (r)
                        return r;
        }
@@ -1154,7 +1156,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
                if (r)
                        return r;
 
-               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update);
+               r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update,
+                                     GFP_KERNEL);
                if (r)
                        return r;
        }
@@ -1167,7 +1170,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
        if (r)
                return r;
 
-       r = amdgpu_sync_fence(&p->sync, vm->last_update);
+       r = amdgpu_sync_fence(&p->sync, vm->last_update, GFP_KERNEL);
        if (r)
                return r;
 
@@ -1248,7 +1251,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
                        continue;
                }
 
-               r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence);
+               r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence,
+                                     GFP_KERNEL);
                dma_fence_put(fence);
                if (r)
                        return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
index 9008b7388e8979adb6e6f2ea46659453aea0f73a..92ab821afc06ae561929cfed74c72ad0ac8d87b2 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ids.c
@@ -209,7 +209,7 @@ static int amdgpu_vmid_grab_idle(struct amdgpu_ring *ring,
                return 0;
        }
 
-       fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_KERNEL);
+       fences = kmalloc_array(id_mgr->num_ids, sizeof(void *), GFP_NOWAIT);
        if (!fences)
                return -ENOMEM;
 
@@ -313,7 +313,8 @@ static int amdgpu_vmid_grab_reserved(struct amdgpu_vm *vm,
        /* Good we can use this VMID. Remember this submission as
        * user of the VMID.
        */
-       r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished);
+       r = amdgpu_sync_fence(&(*id)->active, &job->base.s_fence->finished,
+                             GFP_NOWAIT);
        if (r)
                return r;
 
@@ -372,7 +373,8 @@ static int amdgpu_vmid_grab_used(struct amdgpu_vm *vm,
                 * user of the VMID.
                 */
                r = amdgpu_sync_fence(&(*id)->active,
-                                     &job->base.s_fence->finished);
+                                     &job->base.s_fence->finished,
+                                     GFP_NOWAIT);
                if (r)
                        return r;
 
@@ -424,7 +426,8 @@ int amdgpu_vmid_grab(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
 
                        /* Remember this submission as user of the VMID */
                        r = amdgpu_sync_fence(&id->active,
-                                             &job->base.s_fence->finished);
+                                             &job->base.s_fence->finished,
+                                             GFP_NOWAIT);
                        if (r)
                                goto error;
 

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 6fa20980a0b15e5be41ba1f6873943e25e5158c2..e4251d0691c9cbbfcfa49366edee6f1dfb10303f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -1335,14 +1335,14 @@ int amdgpu_mes_ctx_map_meta_data(struct amdgpu_device *adev,
                DRM_ERROR("failed to do vm_bo_update on meta data\n");
                goto error_del_bo_va;
        }
-       amdgpu_sync_fence(&sync, bo_va->last_pt_update);
+       amdgpu_sync_fence(&sync, bo_va->last_pt_update, GFP_KERNEL);
 
        r = amdgpu_vm_update_pdes(adev, vm, false);
        if (r) {
                DRM_ERROR("failed to update pdes on meta data\n");
                goto error_del_bo_va;
        }
-       amdgpu_sync_fence(&sync, vm->last_update);
+       amdgpu_sync_fence(&sync, vm->last_update, GFP_KERNEL);
 
        amdgpu_sync_wait(&sync, false);
        drm_exec_fini(&exec);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index d75715b3f1870f32e0de7d29fff915a636e9d101..34fc742fda91d5b8aa7471b3e72af0e799a749bf 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -152,7 +152,8 @@ static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f)
  *
  * Add the fence to the sync object.
  */
-int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
+int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f,
+                     gfp_t flags)
 {
        struct amdgpu_sync_entry *e;
 
@@ -162,7 +163,7 @@ int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f)
        if (amdgpu_sync_add_later(sync, f))
                return 0;
 
-       e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
+       e = kmem_cache_alloc(amdgpu_sync_slab, flags);
        if (!e)
                return -ENOMEM;
 
@@ -249,7 +250,7 @@ int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
                        struct dma_fence *tmp = dma_fence_chain_contained(f);
 
                        if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) {
-                               r = amdgpu_sync_fence(sync, f);
+                               r = amdgpu_sync_fence(sync, f, GFP_KERNEL);
                                dma_fence_put(f);
                                if (r)
                                        return r;
@@ -281,7 +282,7 @@ int amdgpu_sync_kfd(struct amdgpu_sync *sync, struct dma_resv *resv)
                if (fence_owner != AMDGPU_FENCE_OWNER_KFD)
                        continue;
 
-               r = amdgpu_sync_fence(sync, f);
+               r = amdgpu_sync_fence(sync, f, GFP_KERNEL);
                if (r)
                        break;
        }
@@ -388,7 +389,7 @@ int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone)
        hash_for_each_safe(source->fences, i, tmp, e, node) {
                f = e->fence;
                if (!dma_fence_is_signaled(f)) {
-                       r = amdgpu_sync_fence(clone, f);
+                       r = amdgpu_sync_fence(clone, f, GFP_KERNEL);
                        if (r)
                                return r;
                } else {

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
index a91a8eaf808b198f148fc9d798cdb67799812311..51eb4382c91ebd216b3be4ae80d8923ff9c5c8db 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.h
@@ -47,7 +47,8 @@ struct amdgpu_sync {
 };
 
 void amdgpu_sync_create(struct amdgpu_sync *sync);
-int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f);
+int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f,
+                     gfp_t flags);
 int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync,
                     struct dma_resv *resv, enum amdgpu_sync_mode mode,
                     void *owner);