drm/amdgpu/gmc12.1: implement tlb inv semaphore

Needed to properly lock the interface before using it.

Cc: Sonny Jiang <sonny.jiang@amd.com>
Reviewed-by: Sonny Jiang <sonny.jiang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
(cherry picked from commit 8e37aa0bd56ba75801a6a21bed45f96372cd9fdc)
Cc: stable@vger.kernel.org

diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index 855cd29cbffaacf96ee99c5d55702fde723510ff..bdf6788b85b58f67eb6c9001ca923512ac4b4380 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -267,9 +267,24 @@ static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev,
  * by the amdgpu vm/hsa code.
  */
 
+/**
+ * gmc_v12_1_use_invalidate_semaphore - judge whether to use semaphore
+ *
+ * @adev: amdgpu_device pointer
+ * @vmhub: vmhub type
+ *
+ */
+static bool gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device *adev,
+                                      uint32_t vmhub)
+{
+       return ((!AMDGPU_IS_GFXHUB(vmhub)) &&
+               (!amdgpu_sriov_vf(adev)));
+}
+
 static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
                                   unsigned int vmhub, uint32_t flush_type)
 {
+       bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(adev, vmhub);
        struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
        u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
        u32 tmp;
@@ -283,6 +298,19 @@ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
 
        spin_lock(&adev->gmc.invalidate_lock);
 
+       if (use_semaphore) {
+               for (i = 0; i < adev->usec_timeout; i++) {
+                       /* a read return value of 1 means semaphore acuqire */
+                       tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, hub_ip);
+                       if (tmp & 0x1)
+                               break;
+                       udelay(1);
+               }
+
+               if (i >= adev->usec_timeout)
+                       DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
+       }
+
        WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip);
 
        /* Wait for ACK with a delay.*/
@@ -296,6 +324,9 @@ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
                udelay(1);
        }
 
+       if (use_semaphore)
+               WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0, hub_ip);
+
        /* Issue additional private vm invalidation to MMHUB */
        if (!AMDGPU_IS_GFXHUB(vmhub) &&
            (hub->vm_l2_bank_select_reserved_cid2) &&
@@ -431,10 +462,17 @@ static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
 static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
                                             unsigned vmid, uint64_t pd_addr)
 {
+       bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(ring->adev, ring->vm_hub);
        struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
        uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
        unsigned eng = ring->vm_inv_eng;
 
+       if (use_semaphore)
+               /* a read return value of 1 means semaphore acuqire */
+               amdgpu_ring_emit_reg_wait(ring,
+                                         hub->vm_inv_eng0_sem +
+                                         hub->eng_distance * eng, 0x1, 0x1);
+
        amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
                              (hub->ctx_addr_distance * vmid),
                              lower_32_bits(pd_addr));
@@ -449,6 +487,14 @@ static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
                                            hub->eng_distance * eng,
                                            req, 1 << vmid);
 
+       if (use_semaphore)
+               /*
+                * add semaphore release after invalidation,
+                * write with 0 means semaphore release
+                */
+               amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
+                                     hub->eng_distance * eng, 0);
+
        return pd_addr;
 }