drm/amdgpu: Move VM PTE MTYPE override to per-PTE granularity

Refactor the NUMA-aware MTYPE override for VM page table entries:

Move the override_vm_pte_flags call from the centralized
amdgpu_vm_pte_update_flags() into the individual CPU and SDMA update
backends, enabling per-PTE MTYPE override including for scattered
pages (pages_addr path).

Move APU, IP version, and direct-mapped eligibility checks from
runtime (gmc_v9_0_override_vm_pte_flags) to init time
(gmc_v9_0_set_gmc_funcs), to avoid repeated runtime checks on
every PTE update.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 6ab4c1e297fce7e966e9306c357a5802301db7b2..c6d7a9e54eb3714ac4370be511a6f47b116c5381 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -365,6 +365,8 @@ struct amdgpu_gmc {
        bool flush_tlb_needs_extra_type_0;
        bool flush_tlb_needs_extra_type_2;
        bool flush_pasid_uses_kiq;
+
+       bool override_pte;
 };
 
 #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9ba9de16a27a2324334763d16afce1653a16a2f1..2f2ac3c4c2f7202431462ab45c32a9443c0f8310 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1163,7 +1163,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
        params.pages_addr = pages_addr;
        params.unlocked = unlocked;
        params.needs_flush = flush_tlb;
-       params.allow_override = allow_override;
+       params.override_pte = allow_override && adev->gmc.override_pte;
        INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
        amdgpu_vm_eviction_lock(vm);

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index d083d7aab75c6e47fab1284d9a1496c8b8a75f63..f4b0e9da3fa2c4595a473911f937239f66515e9c 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -296,10 +296,10 @@ struct amdgpu_vm_update_params {
        bool needs_flush;
 
        /**
-        * @allow_override: true for memory that is not uncached: allows MTYPE
-        * to be overridden for NUMA local memory.
+        * @override_pte: true for memory that is not uncached and gmc override function is
+        * implemented to allow MTYPE to be overridden for NUMA local memory.
         */
-       bool allow_override;
+       bool override_pte;
 
        /**
         * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index f078db3fef79e104beee9e717e2a4f4fdcddaf25..b31ff6f56f0d615ae74204174229e0597423ba76 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -88,12 +88,21 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p,
 
        trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->immediate);
 
+       if (!p->pages_addr && p->override_pte)
+               amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
+
        for (i = 0; i < count; i++) {
+               u64 oflags = flags;
+
                value = p->pages_addr ?
                        amdgpu_vm_map_gart(p->pages_addr, addr) :
                        addr;
+
+               if (p->pages_addr && p->override_pte)
+                       amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, value, &oflags);
+
                amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe,
-                                      i, value, flags);
+                                      i, value, oflags);
                addr += incr;
        }
        return 0;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index a930f1522f962f08a70dedf9d088c3f319c1429f..b2494381b98a1056cdfbe22ae9a5afc937cb7343 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -710,15 +710,6 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
        if (level == AMDGPU_VM_PTB)
                amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
-       /* APUs mapping system memory may need different MTYPEs on different
-        * NUMA nodes. Only do this for contiguous ranges that can be assumed
-        * to be on the same NUMA node.
-        */
-       if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
-           adev->gmc.gmc_funcs->override_vm_pte_flags &&
-           num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-               amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
-
        params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
                                         flags);
 }

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 36805dcfa15988f1a57f508d9d39c6c8fbe4dbac..2eb64df6daa94fb8204b2b1f98891dfc3a55b04e 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -257,6 +257,9 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
                }
 
                if (!p->pages_addr) {
+                       if (p->override_pte)
+                               amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
+
                        /* set page commands needed */
                        amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count,
                                                incr, flags);
@@ -275,8 +278,14 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
                p->num_dw_left -= nptes * 2;
                pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]);
                for (i = 0; i < nptes; ++i, addr += incr) {
+                       u64 oflags = flags;
+
                        pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr);
-                       pte[i] |= flags;
+
+                       if (p->override_pte)
+                               amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, pte[i], &oflags);
+
+                       pte[i] |= oflags;
                }
 
                amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes);

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index e7b78027002beefb52f377119aa864b448e9d865..aca7841173f35469494d48e7d233bd6f7c9b8177 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1204,21 +1204,6 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 {
        int local_node, nid;
 
-       /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
-        * memory can use more efficient MTYPEs.
-        */
-       if (!(adev->flags & AMD_IS_APU) ||
-           amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3))
-               return;
-
-       /* Only direct-mapped memory allows us to determine the NUMA node from
-        * the DMA address.
-        */
-       if (!adev->ram_is_direct_mapped) {
-               dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
-               return;
-       }
-
        /* MTYPE_NC is the same default and can be overridden.
         * MTYPE_UC will be present if the memory is extended-coherent
         * and can also be overridden.
@@ -1231,11 +1216,7 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
                return;
        }
 
-       /* FIXME: Only supported on native mode for now. For carve-out, the
-        * NUMA affinity of the GPU/VM needs to come from the PCI info because
-        * memory partitions are not associated with different NUMA nodes.
-        */
-       if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
+       if (vm->mem_id >= 0) {
                local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
        } else {
                dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
@@ -1344,6 +1325,20 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
 {
        adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
+
+       /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes, local system
+        * memory can use more efficient MTYPEs.
+        *
+        * APUs mapping system memory may need different MTYPEs on different
+        * NUMA nodes.
+        *
+        * Only direct-mapped memory allows us to determine the NUMA node from
+        * the DMA address.
+        */
+       adev->gmc.override_pte = adev->gmc.is_app_apu &&
+                                num_possible_nodes() > 1 &&
+                                amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
+                                adev->ram_is_direct_mapped;
 }
 
 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)