drm/amdgpu: pass all the sdma scheds to amdgpu_mman

This will allow the use of all of them for clear/fill buffer
operations.
Since drm_sched_entity_init requires a scheduler array, we
store schedulers rather than rings. For the few places that need
access to a ring, we can get it from the sched using container_of.

Since the code is the same for all sdma versions, add a new
helper amdgpu_sdma_set_buffer_funcs_scheds to set buffer_funcs_scheds
based on the number of sdma instances.

Note: the new sched array is identical to the amdgpu_vm_manager one.
These 2 could be merged.

Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
Acked-by: Felix Kuehling <felix.kuehling@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 8bc591deb546d10ce5f7c5bb37a8fa9f17153e63..695dac53fe08218f6b4d63b550565ab93db34812 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -1462,6 +1462,8 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring);
 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset);
 void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
                                   const struct amdgpu_vm_pte_funcs *vm_pte_funcs);
+void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
+                                        const struct amdgpu_buffer_funcs *buffer_funcs);
 
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 413145a958fc1e675d5998a1d2dd36e617d71388..86ba5aa39462efdabdc2582279136818a568a774 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -3700,7 +3700,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
        adev->num_rings = 0;
        RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
        adev->mman.buffer_funcs = NULL;
-       adev->mman.buffer_funcs_ring = NULL;
+       adev->mman.num_buffer_funcs_scheds = 0;
        adev->vm_manager.vm_pte_funcs = NULL;
        adev->vm_manager.vm_pte_num_scheds = 0;
        adev->gmc.gmc_funcs = NULL;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 285e217fba040b039a2a667c9b8000023611d146..bfeb1b0ec935422192b2197e2acfdcf264c29396 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -708,12 +708,14 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
                              uint32_t vmhub, uint32_t flush_type)
 {
-       struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+       struct amdgpu_ring *ring;
        struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
        struct dma_fence *fence;
        struct amdgpu_job *job;
        int r;
 
+       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
+
        if (!hub->sdma_invalidation_workaround || vmid ||
            !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready ||
            !ring->sched.ready) {

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 0dc68fb9d88e579ebb006216c0133da970da315c..24c7b0b48418bd5e0cdcd190d7b8433648e07e76 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -168,7 +168,7 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit
 {
        struct amdgpu_ring *ring;
 
-       ring = adev->mman.buffer_funcs_ring;
+       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
        amdgpu_ring_pad_ib(ring, &job->ibs[0]);
        WARN_ON(job->ibs[0].length_dw > num_dw);
 
@@ -2298,18 +2298,17 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
                return;
 
        if (enable) {
-               struct amdgpu_ring *ring;
                struct drm_gpu_scheduler *sched;
 
-               if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) {
+               if (!adev->mman.num_buffer_funcs_scheds ||
+                   !adev->mman.buffer_funcs_scheds[0]->ready) {
                        dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
                        return;
                }
 
                num_clear_entities = 1;
                num_move_entities = 1;
-               ring = adev->mman.buffer_funcs_ring;
-               sched = &ring->sched;
+               sched = adev->mman.buffer_funcs_scheds[0];
                r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
                                                  &adev->mman.default_entity,
                                                  DRM_SCHED_PRIORITY_KERNEL,
@@ -2446,7 +2445,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
        unsigned int i;
        int r;
 
-       ring = adev->mman.buffer_funcs_ring;
+       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 
        if (!ring->sched.ready) {
                dev_err(adev->dev,
@@ -2679,6 +2678,31 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
        return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
 }
 
+void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
+                                        const struct amdgpu_buffer_funcs *buffer_funcs)
+{
+       struct drm_gpu_scheduler *sched;
+       struct amdgpu_vmhub *hub;
+       int i;
+
+       adev->mman.buffer_funcs = buffer_funcs;
+
+       for (i = 0; i < adev->sdma.num_instances; i++) {
+               if (adev->sdma.has_page_queue)
+                       sched = &adev->sdma.instance[i].page.sched;
+               else
+                       sched = &adev->sdma.instance[i].ring.sched;
+               adev->mman.buffer_funcs_scheds[i] = sched;
+       }
+
+       /* Navi1x's workaround requires us to limit to a single SDMA sched
+        * for ttm.
+        */
+       hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+       adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ?
+               1 : adev->sdma.num_instances;
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index f2f23a42b3cc45c9ec0fd969d4c9f9dd48b24183..471a79c153cf948c95002f79bd70f616ff36b713 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -87,7 +87,8 @@ struct amdgpu_mman {
 
        /* buffer handling */
        const struct amdgpu_buffer_funcs        *buffer_funcs;
-       struct amdgpu_ring                      *buffer_funcs_ring;
+       struct drm_gpu_scheduler                *buffer_funcs_scheds[AMDGPU_MAX_RINGS];
+       u32                                     num_buffer_funcs_scheds;
        bool                                    buffer_funcs_enabled;
 
        /* @default_entity: for workarounds, has no gart windows */

diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 22780c09177d88589b79ef01e487d9e672eff436..26276dcfd458e57a5b42aed537f3241c052615cf 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -1340,8 +1340,7 @@ static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {
 
 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &cik_sdma_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version cik_sdma_ip_block =

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 0090ace49024f43151323416295094d81a19bd76..c6a059ca59e5bd1cbedae0c8294450f16d9d61d9 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -1235,8 +1235,7 @@ static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = {
 
 static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v2_4_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v2_4_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 2526d393162ace11e50668fbe27937a886c1ee0d..cb516a25210d87fb9c44f2e6f712cb8359e8b9dc 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1677,8 +1677,7 @@ static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
 
 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v3_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v3_0_ip_block =

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 44f0f23e11484384c1431ff0b4b002e02183c304..d56be26f216bb099a36a8d245214f72d52645690 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -2626,13 +2626,9 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_buffer_funcs = {
 static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
        if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= IP_VERSION(4, 4, 0))
-               adev->mman.buffer_funcs = &sdma_v4_4_buffer_funcs;
+               amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_buffer_funcs);
        else
-               adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
-       if (adev->sdma.has_page_queue)
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
-       else
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+               amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_0_buffer_funcs);
 }
 
 static void sdma_v4_0_get_ras_error_count(uint32_t value,

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index 78bdfed0a7fd3cfa911fa75e4ce7b83292d45809..67e9697301b47da6675d2f29a9e23801b3bb3aa3 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -2316,11 +2316,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
 
 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
-       if (adev->sdma.has_page_queue)
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
-       else
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_2_buffer_funcs);
 }
 
 /**

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index 52f4e9e099cbf2f5dd5860768fb6929848377d97..86f5eb784d572335b82944f1660e896453592cb3 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -2052,10 +2052,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = {
 
 static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       if (adev->mman.buffer_funcs == NULL) {
-               adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs;
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
-       }
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v5_0_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index b4fb90cc8f7d970a8ee48e37df58b24d3ddb9e66..3fec838374b2373681d72d30778ba16a3d9ebbc3 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -2056,10 +2056,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
 
 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       if (adev->mman.buffer_funcs == NULL) {
-               adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
-               adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
-       }
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_2_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 0f530bb8a9a36044c705dfc3501f0fa62079a107..eaf5b4656df155e0ba5100d4c924b06c94a9aecf 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1895,8 +1895,7 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
 
 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v6_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 9ed817b69a3b74f803d0545bab22c6160cc98cee..7161e818bfd616f3e11b13043932277d738e3fea 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1845,8 +1845,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
 
 static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index 061934a2e93a38527ffce901b7d720ffee0f07fc..4775bbf714a160c1e6dbe1b4aa4105144ec68559 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -1776,8 +1776,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
 
 static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {

diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3e58feb2d5e4f107104e6256b2797e732a1d7a03..155067c20a0ed067bb50a4dc74eb07437a464af6 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -833,8 +833,7 @@ static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = {
 
 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
 {
-       adev->mman.buffer_funcs = &si_dma_buffer_funcs;
-       adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+       amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version si_dma_ip_block =

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 964efa32590893137f4edf172560e538297353e6..28dc6886c1ff101512473f13b29c9e834d1bbf6a 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -129,13 +129,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
                             struct dma_fence **mfence)
 {
        const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
-       struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+       struct amdgpu_ring *ring;
        struct amdgpu_ttm_buffer_entity *entity;
        u64 gart_s, gart_d;
        struct dma_fence *next;
        u64 size;
        int r;
 
+       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
        entity = &adev->mman.move_entities[0];
 
        mutex_lock(&entity->lock);