drm/amdgpu: trigger flr_work if reading pf2vf data failed

[ Upstream commit ab66c832847fcdffc97d4591ba5547e3990d9d33 ]

if reading pf2vf data failed 30 times continuously, it means something is
wrong. Need to trigger flr_work to recover the issue.

also use dev_err to print the error message to get which device has
issue and add warning message if waiting IDH_FLR_NOTIFICATION_CMPL
timeout.

Signed-off-by: Zhigang Luo <Zhigang.Luo@amd.com>
Acked-by: Hawking Zhang <Hawking.Zhang@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Stable-dep-of: d0ce1aaa8531 ("Revert "drm/amd: Stop evicting resources on APUs in suspend"")
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 52f4ef4e8b4b05e1f40324a80845e442d9b7ad59..aac0e49f77c7ff98070097a38ba68380c09c7b65 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -139,6 +139,8 @@ const char *amdgpu_asic_name[] = {
        "LAST",
 };
 
+static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev);
+
 /**
  * DOC: pcie_replay_count
  *
@@ -4638,6 +4640,8 @@ static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
 retry:
        amdgpu_amdkfd_pre_reset(adev);
 
+       amdgpu_device_stop_pending_resets(adev);
+
        if (from_hypervisor)
                r = amdgpu_virt_request_full_gpu(adev, true);
        else
@@ -5509,11 +5513,12 @@ retry:  /* Rest of adevs pre asic reset from XGMI hive. */
                        tmp_adev->asic_reset_res = r;
                }
 
-               /*
-                * Drop all pending non scheduler resets. Scheduler resets
-                * were already dropped during drm_sched_stop
-                */
-               amdgpu_device_stop_pending_resets(tmp_adev);
+               if (!amdgpu_sriov_vf(tmp_adev))
+                       /*
+                       * Drop all pending non scheduler resets. Scheduler resets
+                       * were already dropped during drm_sched_stop
+                       */
+                       amdgpu_device_stop_pending_resets(tmp_adev);
        }
 
        tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index d7b76a3d2d5588e9f242389341cea39dd4bd8412..6174bef0ecb886659c7103cb29a34260ed235111 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -32,6 +32,7 @@
 
 #include "amdgpu.h"
 #include "amdgpu_ras.h"
+#include "amdgpu_reset.h"
 #include "vi.h"
 #include "soc15.h"
 #include "nv.h"
@@ -456,7 +457,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
                return -EINVAL;
 
        if (pf2vf_info->size > 1024) {
-               DRM_ERROR("invalid pf2vf message size\n");
+               dev_err(adev->dev, "invalid pf2vf message size: 0x%x\n", pf2vf_info->size);
                return -EINVAL;
        }
 
@@ -467,7 +468,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
                        adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
                        adev->virt.fw_reserve.checksum_key, checksum);
                if (checksum != checkval) {
-                       DRM_ERROR("invalid pf2vf message\n");
+                       dev_err(adev->dev,
+                               "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
+                               checksum, checkval);
                        return -EINVAL;
                }
 
@@ -481,7 +484,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
                        adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
                        0, checksum);
                if (checksum != checkval) {
-                       DRM_ERROR("invalid pf2vf message\n");
+                       dev_err(adev->dev,
+                               "invalid pf2vf message: header checksum=0x%x calculated checksum=0x%x\n",
+                               checksum, checkval);
                        return -EINVAL;
                }
 
@@ -517,7 +522,7 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
                        ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid;
                break;
        default:
-               DRM_ERROR("invalid pf2vf version\n");
+               dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
                return -EINVAL;
        }
 
@@ -617,8 +622,21 @@ static void amdgpu_virt_update_vf2pf_work_item(struct work_struct *work)
        int ret;
 
        ret = amdgpu_virt_read_pf2vf_data(adev);
-       if (ret)
+       if (ret) {
+               adev->virt.vf2pf_update_retry_cnt++;
+               if ((adev->virt.vf2pf_update_retry_cnt >= AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT) &&
+                   amdgpu_sriov_runtime(adev) && !amdgpu_in_reset(adev)) {
+                       if (amdgpu_reset_domain_schedule(adev->reset_domain,
+                                                         &adev->virt.flr_work))
+                               return;
+                       else
+                               dev_err(adev->dev, "Failed to queue work! at %s", __func__);
+               }
+
                goto out;
+       }
+
+       adev->virt.vf2pf_update_retry_cnt = 0;
        amdgpu_virt_write_vf2pf_data(adev);
 
 out:
@@ -639,6 +657,7 @@ void amdgpu_virt_init_data_exchange(struct amdgpu_device *adev)
        adev->virt.fw_reserve.p_pf2vf = NULL;
        adev->virt.fw_reserve.p_vf2pf = NULL;
        adev->virt.vf2pf_update_interval_ms = 0;
+       adev->virt.vf2pf_update_retry_cnt = 0;
 
        if (adev->mman.fw_vram_usage_va != NULL) {
                /* go through this logic in ip_init and reset to init workqueue*/

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index dc6aaa4d67be7e1d66afa5e84b6990a0ceca6cad..fc2859726f0a691d7c5c6fbff6f740a0a3c7df67 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -51,6 +51,8 @@
 /* tonga/fiji use this offset */
 #define mmBIF_IOV_FUNC_IDENTIFIER 0x1503
 
+#define AMDGPU_VF2PF_UPDATE_MAX_RETRY_LIMIT 30
+
 enum amdgpu_sriov_vf_mode {
        SRIOV_VF_MODE_BARE_METAL = 0,
        SRIOV_VF_MODE_ONE_VF,
@@ -250,6 +252,7 @@ struct amdgpu_virt {
        /* vf2pf message */
        struct delayed_work vf2pf_work;
        uint32_t vf2pf_update_interval_ms;
+       int vf2pf_update_retry_cnt;
 
        /* multimedia bandwidth config */
        bool     is_mm_bw_enabled;

diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 12906ba74462fb65669392bc826663e8fbb60d09..d39c6baae007abe0e29ff6dc0eae35a1a4c07e71 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -276,6 +276,8 @@ static void xgpu_ai_mailbox_flr_work(struct work_struct *work)
                timeout -= 10;
        } while (timeout > 1);
 
+       dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n");
+
 flr_done:
        atomic_set(&adev->reset_domain->in_gpu_reset, 0);
        up_write(&adev->reset_domain->sem);

diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
index e07757eea7adf95bb43b1a330166b8e84a75468b..a311a2425ec01ee85ff63c8ef9fd838fe5f1cf45 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
@@ -300,6 +300,8 @@ static void xgpu_nv_mailbox_flr_work(struct work_struct *work)
                timeout -= 10;
        } while (timeout > 1);
 
+       dev_warn(adev->dev, "waiting IDH_FLR_NOTIFICATION_CMPL timeout\n");
+
 flr_done:
        atomic_set(&adev->reset_domain->in_gpu_reset, 0);
        up_write(&adev->reset_domain->sem);