drm/amdgpu/gfx: move fault and EOP IRQ get/put to hw_init/hw_fini

priv_reg / priv_inst / bad_op and (on v11+) userq EOP IRQs are
acquired in late_init but released in hw_fini.  This split forced
gfx_v9_0_hw_fini() to defensively guard each put with
amdgpu_irq_enabled() because hw_fini runs on paths that may not
reach late_init.

amdgpu_ip_block_hw_fini() only runs after hw_init returns success,
and suspend / resume cycle the refs through the same path, so
hw_init / hw_fini pair without any extra tracking.  Move the gets
there and drop the guards.

While here, fix the pre-existing partial-failure leak in
set_userq_eop_interrupts() (gfx11 / 12_0 / 12_1).  amdgpu_irq_get()
increments the refcount before calling .set, so a failure partway
through the loop leaves earlier successful gets stranded.  Track
the loop position and roll back on the enable path.

Signed-off-by: Yunxiang Li <Yunxiang.Li@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 58c69dcb527f7182afe24a0aa7e17a245955d4c8..0780c5e5de4ff975358e762186319f2177a4afb2 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -7530,6 +7530,24 @@ static int gfx_v10_0_hw_init(struct amdgpu_ip_block *ip_block)
        if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(10, 3, 0) && !amdgpu_sriov_vf(adev))
                gfx_v10_3_set_power_brake_sequence(adev);
 
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
+       if (r)
+               goto err_bad_op;
+
+       return 0;
+
+err_bad_op:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
        return r;
 }
 
@@ -7539,9 +7557,9 @@ static int gfx_v10_0_hw_fini(struct amdgpu_ip_block *ip_block)
 
        cancel_delayed_work_sync(&adev->gfx.idle_work);
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
        amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        /* WA added for Vangogh asic fixing the SMU suspend failure
         * It needs to set power gating again during gfxoff control
@@ -7837,26 +7855,6 @@ static int gfx_v10_0_early_init(struct amdgpu_ip_block *ip_block)
        return gfx_v10_0_init_microcode(adev);
 }
 
-static int gfx_v10_0_late_init(struct amdgpu_ip_block *ip_block)
-{
-       struct amdgpu_device *adev = ip_block->adev;
-       int r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
-       if (r)
-               return r;
-
-       return 0;
-}
-
 static bool gfx_v10_0_is_rlc_enabled(struct amdgpu_device *adev)
 {
        uint32_t rlc_cntl;
@@ -9805,7 +9803,6 @@ static void gfx_v10_0_ring_end_use(struct amdgpu_ring *ring)
 static const struct amd_ip_funcs gfx_v10_0_ip_funcs = {
        .name = "gfx_v10_0",
        .early_init = gfx_v10_0_early_init,
-       .late_init = gfx_v10_0_late_init,
        .sw_init = gfx_v10_0_sw_init,
        .sw_fini = gfx_v10_0_sw_fini,
        .hw_init = gfx_v10_0_hw_init,

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index 1941bfbcbfbfff1ec6140d2c99bc055ee938ef50..f856b0cf5becf1014bfb151f4c01fad149f51398 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -4814,6 +4814,78 @@ static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev)
        WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data);
 }
 
+static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
+                                             bool enable)
+{
+       unsigned int irq_type;
+       int m, p, r;
+
+       if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
+               for (m = 0; m < adev->gfx.me.num_me; m++) {
+                       for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
+                               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+                               if (enable)
+                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
+                               else
+                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+                               if (r) {
+                                       if (!enable)
+                                               return r;
+                                       goto err_gfx;
+                               }
+                       }
+               }
+       }
+
+       if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
+               for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
+                       for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
+                               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                                       + (m * adev->gfx.mec.num_pipe_per_mec)
+                                       + p;
+                               if (enable)
+                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
+                               else
+                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+                               if (r) {
+                                       if (!enable)
+                                               return r;
+                                       goto err_compute;
+                               }
+                       }
+               }
+       }
+
+       return 0;
+
+err_compute:
+       for (p--; p >= 0; p--) {
+               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                       + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+               amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+       }
+       for (m--; m >= 0; m--) {
+               for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
+                       irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                               + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+                       amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+               }
+       }
+       m = adev->gfx.me.num_me;
+err_gfx:
+       for (p--; p >= 0; p--) {
+               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+               amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+       }
+       for (m--; m >= 0; m--) {
+               for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) {
+                       irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+                       amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+               }
+       }
+       return r;
+}
+
 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
 {
        int r;
@@ -4911,50 +4983,31 @@ static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
        if (!adev->gfx.imu_fw_version)
                adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0);
 
-       return r;
-}
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
 
-static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
-                                             bool enable)
-{
-       unsigned int irq_type;
-       int m, p, r;
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
 
-       if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
-               for (m = 0; m < adev->gfx.me.num_me; m++) {
-                       for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
-                               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
-                               if (enable)
-                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               else
-                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               if (r)
-                                       return r;
-                       }
-               }
-       }
+       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
+       if (r)
+               goto err_bad_op;
 
-       if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
-               for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
-                       for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
-                               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
-                                       + (m * adev->gfx.mec.num_pipe_per_mec)
-                                       + p;
-                               if (enable)
-                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               else
-                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               if (r)
-                                       return r;
-                       }
-               }
-       }
+       r = gfx_v11_0_set_userq_eop_interrupts(adev, true);
+       if (r)
+               goto err_userq_eop;
 
        return 0;
+
+err_userq_eop:
+       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+err_bad_op:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
+       return r;
 }
 
 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -4963,10 +5016,10 @@ static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
 
        cancel_delayed_work_sync(&adev->gfx.idle_work);
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
        gfx_v11_0_set_userq_eop_interrupts(adev, false);
+       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        if (!adev->no_hw_access) {
                if (amdgpu_async_gfx_ring &&
@@ -5356,30 +5409,6 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block)
        return gfx_v11_0_init_microcode(adev);
 }
 
-static int gfx_v11_0_late_init(struct amdgpu_ip_block *ip_block)
-{
-       struct amdgpu_device *adev = ip_block->adev;
-       int r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
-       if (r)
-               return r;
-
-       r = gfx_v11_0_set_userq_eop_interrupts(adev, true);
-       if (r)
-               return r;
-
-       return 0;
-}
-
 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev)
 {
        uint32_t rlc_cntl;
@@ -7211,7 +7240,6 @@ static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring)
 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = {
        .name = "gfx_v11_0",
        .early_init = gfx_v11_0_early_init,
-       .late_init = gfx_v11_0_late_init,
        .sw_init = gfx_v11_0_sw_init,
        .sw_fini = gfx_v11_0_sw_fini,
        .hw_init = gfx_v11_0_hw_init,

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index f47928dcd8480aca3a8da647756f1fdcc8ceb8c6..f66293fc675e7172716afd74318e0513344bb068 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -3655,6 +3655,78 @@ static void gfx_v12_0_init_golden_registers(struct amdgpu_device *adev)
        }
 }
 
+static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
+                                             bool enable)
+{
+       unsigned int irq_type;
+       int m, p, r;
+
+       if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
+               for (m = 0; m < adev->gfx.me.num_me; m++) {
+                       for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
+                               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+                               if (enable)
+                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
+                               else
+                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+                               if (r) {
+                                       if (!enable)
+                                               return r;
+                                       goto err_gfx;
+                               }
+                       }
+               }
+       }
+
+       if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
+               for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
+                       for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
+                               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                                       + (m * adev->gfx.mec.num_pipe_per_mec)
+                                       + p;
+                               if (enable)
+                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
+                               else
+                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+                               if (r) {
+                                       if (!enable)
+                                               return r;
+                                       goto err_compute;
+                               }
+                       }
+               }
+       }
+
+       return 0;
+
+err_compute:
+       for (p--; p >= 0; p--) {
+               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                       + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+               amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+       }
+       for (m--; m >= 0; m--) {
+               for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
+                       irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                               + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+                       amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+               }
+       }
+       m = adev->gfx.me.num_me;
+err_gfx:
+       for (p--; p >= 0; p--) {
+               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+               amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+       }
+       for (m--; m >= 0; m--) {
+               for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) {
+                       irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
+                       amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+               }
+       }
+       return r;
+}
+
 static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
 {
        int r;
@@ -3742,50 +3814,31 @@ static int gfx_v12_0_hw_init(struct amdgpu_ip_block *ip_block)
        if (r)
                return r;
 
-       return r;
-}
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
 
-static int gfx_v12_0_set_userq_eop_interrupts(struct amdgpu_device *adev,
-                                             bool enable)
-{
-       unsigned int irq_type;
-       int m, p, r;
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
 
-       if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) {
-               for (m = 0; m < adev->gfx.me.num_me; m++) {
-                       for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) {
-                               irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p;
-                               if (enable)
-                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               else
-                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               if (r)
-                                       return r;
-                       }
-               }
-       }
+       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
+       if (r)
+               goto err_bad_op;
 
-       if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) {
-               for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
-                       for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
-                               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
-                                       + (m * adev->gfx.mec.num_pipe_per_mec)
-                                       + p;
-                               if (enable)
-                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               else
-                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               if (r)
-                                       return r;
-                       }
-               }
-       }
+       r = gfx_v12_0_set_userq_eop_interrupts(adev, true);
+       if (r)
+               goto err_userq_eop;
 
        return 0;
+
+err_userq_eop:
+       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+err_bad_op:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
+       return r;
 }
 
 static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -3795,10 +3848,10 @@ static int gfx_v12_0_hw_fini(struct amdgpu_ip_block *ip_block)
 
        cancel_delayed_work_sync(&adev->gfx.idle_work);
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
        gfx_v12_0_set_userq_eop_interrupts(adev, false);
+       amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        if (!adev->no_hw_access) {
                if (amdgpu_async_gfx_ring) {
@@ -3927,30 +3980,6 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block)
        return gfx_v12_0_init_microcode(adev);
 }
 
-static int gfx_v12_0_late_init(struct amdgpu_ip_block *ip_block)
-{
-       struct amdgpu_device *adev = ip_block->adev;
-       int r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
-       if (r)
-               return r;
-
-       r = gfx_v12_0_set_userq_eop_interrupts(adev, true);
-       if (r)
-               return r;
-
-       return 0;
-}
-
 static bool gfx_v12_0_is_rlc_enabled(struct amdgpu_device *adev)
 {
        uint32_t rlc_cntl;
@@ -5440,7 +5469,6 @@ static void gfx_v12_0_ring_end_use(struct amdgpu_ring *ring)
 static const struct amd_ip_funcs gfx_v12_0_ip_funcs = {
        .name = "gfx_v12_0",
        .early_init = gfx_v12_0_early_init,
-       .late_init = gfx_v12_0_late_init,
        .sw_init = gfx_v12_0_sw_init,
        .sw_fini = gfx_v12_0_sw_fini,
        .hw_init = gfx_v12_0_hw_init,

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index 033f15e21ad333225d26382fe226cb6c431b9c43..61c3577f829fe29cfe3f3f2e8e1a5438f88b9f5f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -2735,6 +2735,50 @@ static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev)
        }
 }
 
+static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev,
+                                             bool enable)
+{
+       unsigned int irq_type;
+       int m, p, r;
+
+       if (!adev->gfx.disable_kq)
+               return 0;
+
+       for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
+               for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
+                       irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                               + (m * adev->gfx.mec.num_pipe_per_mec)
+                               + p;
+                       if (enable)
+                               r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type);
+                       else
+                               r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+                       if (r) {
+                               if (!enable)
+                                       return r;
+                               goto err_unwind;
+                       }
+               }
+       }
+
+       return 0;
+
+err_unwind:
+       for (p--; p >= 0; p--) {
+               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                       + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+               amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+       }
+       for (m--; m >= 0; m--) {
+               for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) {
+                       irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
+                               + (m * adev->gfx.mec.num_pipe_per_mec) + p;
+                       amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type);
+               }
+       }
+       return r;
+}
+
 static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
 {
        int r, i, num_xcc;
@@ -2803,6 +2847,24 @@ static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block)
        if (r)
                return r;
 
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
+
+       r = gfx_v12_1_set_userq_eop_interrupts(adev, true);
+       if (r)
+               goto err_userq_eop;
+
+       return 0;
+
+err_userq_eop:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
        return r;
 }
 
@@ -2828,41 +2890,14 @@ static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev,
        gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id);
 }
 
-static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev,
-                                             bool enable)
-{
-       unsigned int irq_type;
-       int m, p, r;
-
-       if (adev->gfx.disable_kq) {
-               for (m = 0; m < adev->gfx.mec.num_mec; ++m) {
-                       for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) {
-                               irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
-                                       + (m * adev->gfx.mec.num_pipe_per_mec)
-                                       + p;
-                               if (enable)
-                                       r = amdgpu_irq_get(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               else
-                                       r = amdgpu_irq_put(adev, &adev->gfx.eop_irq,
-                                                          irq_type);
-                               if (r)
-                                       return r;
-                       }
-               }
-       }
-
-       return 0;
-}
-
 static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
 {
        struct amdgpu_device *adev = ip_block->adev;
        int i, num_xcc;
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
        gfx_v12_1_set_userq_eop_interrupts(adev, false);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        num_xcc = NUM_XCC(adev->gfx.xcc_mask);
        for (i = 0; i < num_xcc; i++) {
@@ -2963,26 +2998,6 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block)
        return gfx_v12_1_init_microcode(adev);
 }
 
-static int gfx_v12_1_late_init(struct amdgpu_ip_block *ip_block)
-{
-       struct amdgpu_device *adev = ip_block->adev;
-       int r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = gfx_v12_1_set_userq_eop_interrupts(adev, true);
-       if (r)
-               return r;
-
-       return 0;
-}
-
 static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev)
 {
        uint32_t rlc_cntl;
@@ -3876,7 +3891,6 @@ static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
 static const struct amd_ip_funcs gfx_v12_1_ip_funcs = {
        .name = "gfx_v12_1",
        .early_init = gfx_v12_1_early_init,
-       .late_init = gfx_v12_1_late_init,
        .sw_init = gfx_v12_1_sw_init,
        .sw_fini = gfx_v12_1_sw_fini,
        .hw_init = gfx_v12_1_hw_init,

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 60376d43e81d1b5a83ef81440fb47266c579f425..47721d0c37812b3a9f4205dea09aaab3938eafbe 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4050,6 +4050,24 @@ static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
            !amdgpu_sriov_vf(adev))
                gfx_v9_4_2_set_power_brake_sequence(adev);
 
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
+       if (r)
+               goto err_bad_op;
+
+       return 0;
+
+err_bad_op:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
        return r;
 }
 
@@ -4057,9 +4075,9 @@ static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
 {
        struct amdgpu_device *adev = ip_block->adev;
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
        amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        /* DF freeze and kcq disable will fail */
        if (!amdgpu_ras_intr_triggered())
@@ -4860,18 +4878,6 @@ static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block)
        struct amdgpu_device *adev = ip_block->adev;
        int r;
 
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
-       if (r)
-               return r;
-
        r = gfx_v9_0_ecc_late_init(ip_block);
        if (r)
                return r;

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index 9f76e1af8a553d4ca5d2f3fa157080b6b29d6d5e..510266ba0c38847e7e9c5633b4403760fec38817 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -2371,6 +2371,24 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
        if (r)
                return r;
 
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
+       if (r)
+               return r;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
+       if (r)
+               goto err_priv_inst;
+
+       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
+       if (r)
+               goto err_bad_op;
+
+       return 0;
+
+err_bad_op:
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+err_priv_inst:
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
        return r;
 }
 
@@ -2446,9 +2464,9 @@ static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
        if (adev->psp.ptl.hw_supported && !amdgpu_in_reset(adev))
                gfx_v9_4_3_perf_monitor_ptl_init(adev, false);
 
-       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
-       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
        amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
+       amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 
        num_xcc = NUM_XCC(adev->gfx.xcc_mask);
        for (i = 0; i < num_xcc; i++) {
@@ -2611,19 +2629,6 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block)
 static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block)
 {
        struct amdgpu_device *adev = ip_block->adev;
-       int r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
-       if (r)
-               return r;
-
-       r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0);
-       if (r)
-               return r;
 
        if (adev->gfx.ras &&
            adev->gfx.ras->enable_watchdog_timer)