accel/amdxdna: Check for device hang on job timeout

A job timeout does not necessarily indicate that the device is hung, as
it may still be processing other jobs.

Track whether any jobs have been successfully submitted or completed,
and use this information to determine if the device is making forward
progress. If so, return DRM_GPU_SCHED_STAT_NO_HANG instead of treating
the timeout as a device hang.

In the meanwhile the timeout interval is changed to 2 seconds which meets
the userspace requirement.

Reviewed-by: Mario Limonciello (AMD) <superm1@kernel.org>
Signed-off-by: Lizhi Hou <lizhi.hou@amd.com>
Link: https://patch.msgid.link/20260409175826.195665-1-lizhi.hou@amd.com

diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index f97755d60fa35bb4bf7d9fb4b5614771b9914da7..ddcf06a6b80c4b9cbf78c9faaee25653b02e7ffc 100644 (file)
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -27,7 +27,9 @@ static bool force_cmdlist = true;
 module_param(force_cmdlist, bool, 0600);
 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)");
 
-#define HWCTX_MAX_TIMEOUT      60000 /* milliseconds */
+uint tdr_timeout_ms = 2000;
+module_param(tdr_timeout_ms, int, 0400);
+MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)");
 
 struct aie2_ctx_health {
        struct amdxdna_ctx_health header;
@@ -39,6 +41,24 @@ struct aie2_ctx_health {
        u32 fatal_error_app_module;
 };
 
+static inline void aie2_tdr_signal(struct amdxdna_dev *xdna)
+{
+       WRITE_ONCE(xdna->dev_handle->tdr_status, AIE2_TDR_SIGNALED);
+}
+
+static bool aie2_tdr_detect(struct amdxdna_dev *xdna)
+{
+       struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+
+       if (READ_ONCE(ndev->tdr_status) == AIE2_TDR_WAIT) {
+               XDNA_ERR(xdna, "TDR timeout detected");
+               return true;
+       }
+
+       WRITE_ONCE(ndev->tdr_status, AIE2_TDR_WAIT);
+       return false;
+}
+
 static void aie2_job_release(struct kref *ref)
 {
        struct amdxdna_sched_job *job;
@@ -177,6 +197,7 @@ aie2_sched_notify(struct amdxdna_sched_job *job)
 
        trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq);
 
+       aie2_tdr_signal(job->hwctx->client->xdna);
        job->hwctx->priv->completed++;
        dma_fence_signal(fence);
 
@@ -385,6 +406,8 @@ out:
                aie2_job_put(job);
                mmput(job->mm);
                fence = ERR_PTR(ret);
+       } else {
+               aie2_tdr_signal(hwctx->client->xdna);
        }
        trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq);
 
@@ -415,9 +438,12 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job)
 
        xdna = hwctx->client->xdna;
        trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq);
-       job->job_timeout = true;
 
-       mutex_lock(&xdna->dev_lock);
+       guard(mutex)(&xdna->dev_lock);
+
+       if (!aie2_tdr_detect(xdna))
+               return DRM_GPU_SCHED_STAT_NO_HANG;
+
        report = kzalloc_obj(*report);
        if (!report)
                goto reset_hwctx;
@@ -429,10 +455,10 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job)
                job->aie2_job_health = report;
 
 reset_hwctx:
+       job->job_timeout = true;
        aie2_hwctx_stop(xdna, hwctx, sched_job);
 
        aie2_hwctx_restart(xdna, hwctx);
-       mutex_unlock(&xdna->dev_lock);
 
        return DRM_GPU_SCHED_STAT_RESET;
 }
@@ -608,7 +634,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
                .ops = &sched_ops,
                .num_rqs = DRM_SCHED_PRIORITY_COUNT,
                .credit_limit = HWCTX_MAX_CMDS,
-               .timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT),
+               .timeout = msecs_to_jiffies(tdr_timeout_ms),
                .name = "amdxdna_js",
                .dev = xdna->ddev.dev,
        };

diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index 77ba125e4d72d5e2e07d36238d917fc314508d9e..f83deca2b51ac369a47467a93bb7617ca56d9a2b 100644 (file)
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -165,6 +165,11 @@ struct aie2_exec_msg_ops {
        u32 (*get_chain_msg_op)(u32 cmd_op);
 };
 
+enum aie2_tdr_status {
+       AIE2_TDR_WAIT,
+       AIE2_TDR_SIGNALED,
+};
+
 struct amdxdna_dev_hdl {
        struct aie_device               aie;
        const struct amdxdna_dev_priv   *priv;
@@ -197,6 +202,7 @@ struct amdxdna_dev_hdl {
        u32                             hwctx_num;
 
        struct amdxdna_async_error      last_async_err;
+       enum aie2_tdr_status            tdr_status;
 };
 
 struct aie2_hw_ops {