queue_work(system_percpu_wq, &offload_work->work);
}
-static void dm_vupdate_high_irq(void *interrupt_params)
-{
- struct common_irq_params *irq_params = interrupt_params;
- struct amdgpu_device *adev = irq_params->adev;
- struct amdgpu_crtc *acrtc;
- struct drm_device *drm_dev;
- struct drm_vblank_crtc *vblank;
- ktime_t frame_duration_ns, previous_timestamp;
- unsigned long flags;
- int vrr_active;
-
- acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
-
- if (acrtc) {
- vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
- drm_dev = acrtc->base.dev;
- vblank = drm_crtc_vblank_crtc(&acrtc->base);
- previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
- frame_duration_ns = vblank->time - previous_timestamp;
-
- if (frame_duration_ns > 0) {
- trace_amdgpu_refresh_rate_track(acrtc->base.index,
- frame_duration_ns,
- ktime_divns(NSEC_PER_SEC, frame_duration_ns));
- atomic64_set(&irq_params->previous_timestamp, vblank->time);
- }
-
- drm_dbg_vbl(drm_dev,
- "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
- vrr_active);
-
- /* Core vblank handling is done here after end of front-porch in
- * vrr mode, as vblank timestamping will give valid results
- * while now done after front-porch. This will also deliver
- * page-flip completion events that have been queued to us
- * if a pageflip happened inside front-porch.
- */
- if (vrr_active && acrtc->dm_irq_params.stream) {
- bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
- bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
- bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
- == VRR_STATE_ACTIVE_VARIABLE;
-
- amdgpu_dm_crtc_handle_vblank(acrtc);
-
- /* BTR processing for pre-DCE12 ASICs */
- if (adev->family < AMDGPU_FAMILY_AI) {
- spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
- mod_freesync_handle_v_update(
- adev->dm.freesync_module,
- acrtc->dm_irq_params.stream,
- &acrtc->dm_irq_params.vrr_params);
-
- if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
- schedule_dc_vmin_vmax(adev,
- acrtc->dm_irq_params.stream,
- &acrtc->dm_irq_params.vrr_params.adjust);
- }
- spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
- }
- }
- }
-}
-
/**
- * dm_crtc_high_irq() - Handles CRTC interrupt
- * @interrupt_params: used for determining the CRTC instance
+ * dm_crtc_high_irq_handler() - Common OTG vblank/flip event handling
+ * @adev: amdgpu device
+ * @acrtc: the CRTC to service
*
- * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK
- * event handler.
+ * Performs writeback completion, vblank event handling, CRC processing, VRR BTR
+ * updates and pageflip completion delivery.
+ *
+ * On DCN this is driven by VUPDATE_NO_LOCK (the register latch point) from
+ * dm_vupdate_high_irq(); on DCE it is driven by VLINE0 at the start of vblank
+ * from dm_crtc_high_irq().
*/
-static void dm_crtc_high_irq(void *interrupt_params)
+static void dm_crtc_high_irq_handler(struct amdgpu_device *adev,
+ struct amdgpu_crtc *acrtc)
{
- struct common_irq_params *irq_params = interrupt_params;
- struct amdgpu_device *adev = irq_params->adev;
struct drm_writeback_job *job;
- struct amdgpu_crtc *acrtc;
unsigned long flags;
int vrr_active;
-
- acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
- if (!acrtc)
- return;
+ bool is_dcn = amdgpu_ip_version(adev, DCE_HWIP, 0) != 0;
if (acrtc->wb_conn) {
spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags);
vrr_active, acrtc->dm_irq_params.active_planes);
/**
- * Core vblank handling at start of front-porch is only possible
- * in non-vrr mode, as only there vblank timestamping will give
- * valid results while done in front-porch. Otherwise defer it
- * to dm_vupdate_high_irq after end of front-porch.
+ * Core vblank handling.
+ *
+ * On DCN this handler runs at VUPDATE_NO_LOCK, the register latch
+ * point, which is the correct place to timestamp both VRR and non-VRR
+ * vblanks.
+ *
+ * On DCE this handler runs at the start of front-porch, where only
+ * non-VRR timestamping is valid; VRR vblank is deferred to
+ * dm_vupdate_high_irq() after end of front-porch.
*/
- if (!vrr_active)
+ if (is_dcn || !vrr_active)
amdgpu_dm_crtc_handle_vblank(acrtc);
/**
}
/*
- * If there aren't any active_planes then DCH HUBP may be clock-gated.
- * In that case, pageflip completion interrupts won't fire and pageflip
- * completion events won't get delivered. Prevent this by sending
- * pending pageflip events from here if a flip is still pending.
+ * Deliver pageflip completion events (DCN only).
+ *
+ * Since GRPH_PFLIP is not used, VUPDATE_NO_LOCK is the flip latch
+ * point. Deliver any pending pageflip completion event from here.
*
- * If any planes are enabled, use dm_pflip_high_irq() instead, to
- * avoid race conditions between flip programming and completion,
- * which could cause too early flip completion events.
+ * NOTE: This can deliver an event for a flip that was armed but not yet
+ * programmed into HW; that race is closed in a follow-up change by
+ * checking the programmed flip status.
*/
- if (adev->family >= AMDGPU_FAMILY_RV &&
- acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED &&
- acrtc->dm_irq_params.active_planes == 0) {
+ if (is_dcn && acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED) {
if (acrtc->event) {
drm_crtc_send_vblank_event(&acrtc->base, acrtc->event);
acrtc->event = NULL;
spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
}
+static void dm_vupdate_high_irq(void *interrupt_params)
+{
+ struct common_irq_params *irq_params = interrupt_params;
+ struct amdgpu_device *adev = irq_params->adev;
+ struct amdgpu_crtc *acrtc;
+ struct drm_device *drm_dev;
+ struct drm_vblank_crtc *vblank;
+ ktime_t frame_duration_ns, previous_timestamp;
+ unsigned long flags;
+ int vrr_active;
+
+ acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE);
+ if (!acrtc)
+ return;
+
+ vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
+ drm_dev = acrtc->base.dev;
+ vblank = drm_crtc_vblank_crtc(&acrtc->base);
+ previous_timestamp = atomic64_read(&irq_params->previous_timestamp);
+ frame_duration_ns = vblank->time - previous_timestamp;
+
+ if (frame_duration_ns > 0) {
+ trace_amdgpu_refresh_rate_track(acrtc->base.index,
+ frame_duration_ns,
+ ktime_divns(NSEC_PER_SEC, frame_duration_ns));
+ atomic64_set(&irq_params->previous_timestamp, vblank->time);
+ }
+
+ drm_dbg_vbl(drm_dev,
+ "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id,
+ vrr_active);
+
+ /*
+ * On DCN, VUPDATE_NO_LOCK is the single OTG interrupt used to deliver
+ * vblank and pageflip completion events; VSTARTUP and GRPH_PFLIP are
+ * not used. Run the full handler here.
+ */
+ if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0) {
+ dm_crtc_high_irq_handler(adev, acrtc);
+ return;
+ }
+
+ /* DCE only below. */
+
+ /* Core vblank handling is done here after end of front-porch in
+ * vrr mode, as vblank timestamping will give valid results
+ * while now done after front-porch. This will also deliver
+ * page-flip completion events that have been queued to us
+ * if a pageflip happened inside front-porch.
+ */
+ if (vrr_active && acrtc->dm_irq_params.stream) {
+ bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled;
+ bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled;
+ bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state
+ == VRR_STATE_ACTIVE_VARIABLE;
+
+ amdgpu_dm_crtc_handle_vblank(acrtc);
+
+ /* BTR processing for pre-DCE12 ASICs */
+ if (adev->family < AMDGPU_FAMILY_AI) {
+ spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags);
+ mod_freesync_handle_v_update(
+ adev->dm.freesync_module,
+ acrtc->dm_irq_params.stream,
+ &acrtc->dm_irq_params.vrr_params);
+
+ if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) {
+ schedule_dc_vmin_vmax(adev,
+ acrtc->dm_irq_params.stream,
+ &acrtc->dm_irq_params.vrr_params.adjust);
+ }
+ spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
+ }
+ }
+}
+
+/**
+ * dm_crtc_high_irq() - Handles CRTC interrupt
+ * @interrupt_params: used for determining the CRTC instance
+ *
+ * Handles the CRTC/VSYNC interrupt by notifying DRM's VBLANK event handler.
+ *
+ * Used on DCE (VLINE0, set to vblank start). On DCN the equivalent handling is
+ * driven by VUPDATE_NO_LOCK in dm_vupdate_high_irq().
+ */
+static void dm_crtc_high_irq(void *interrupt_params)
+{
+ struct common_irq_params *irq_params = interrupt_params;
+ struct amdgpu_device *adev = irq_params->adev;
+ struct amdgpu_crtc *acrtc;
+
+ acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK);
+ if (!acrtc)
+ return;
+
+ dm_crtc_high_irq_handler(adev, acrtc);
+}
+
#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
/**
* dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for
*/
if (!dc_interrupt_set(adev->dm.dc, irq_source, enable))
drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis");
+
+ } else if (acrtc && state->stream_status[i].plane_count != 0) {
+ /* DCN only needs to toggle VUPDATE_NO_LOCK */
+ rc = amdgpu_dm_crtc_set_vupdate_irq(&acrtc->base, enable);
+ if (rc)
+ drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n",
+ enable ? "en" : "dis");
}
}
* for acknowledging and handling.
*/
- /* Use VSTARTUP interrupt */
- for (i = DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP;
- i <= DCN_1_0__SRCID__DC_D1_OTG_VSTARTUP + adev->mode_info.num_crtc - 1;
- i++) {
- r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->crtc_irq);
-
- if (r) {
- drm_err(adev_to_drm(adev), "Failed to add crtc irq id!\n");
- return r;
- }
-
- int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
- int_params.irq_source =
- dc_interrupt_to_irq_source(dc, i, 0);
-
- if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
- int_params.irq_source < DC_IRQ_SOURCE_VBLANK1 ||
- int_params.irq_source > DC_IRQ_SOURCE_VBLANK6) {
- drm_err(adev_to_drm(adev), "Failed to register vblank irq!\n");
- return -EINVAL;
- }
-
- c_irq_params = &adev->dm.vblank_params[int_params.irq_source - DC_IRQ_SOURCE_VBLANK1];
-
- c_irq_params->adev = adev;
- c_irq_params->irq_src = int_params.irq_source;
-
- if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
- dm_crtc_high_irq, c_irq_params))
- return -ENOMEM;
- }
-
/* Use otg vertical line interrupt */
#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
for (i = 0; i <= adev->mode_info.num_crtc - 1; i++) {
return -ENOMEM;
}
- /* Use GRPH_PFLIP interrupt */
- for (i = DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT;
- i <= DCN_1_0__SRCID__HUBP0_FLIP_INTERRUPT + dc->caps.max_otg_num - 1;
- i++) {
- r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->pageflip_irq);
- if (r) {
- drm_err(adev_to_drm(adev), "Failed to add page flip irq id!\n");
- return r;
- }
-
- int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
- int_params.irq_source =
- dc_interrupt_to_irq_source(dc, i, 0);
-
- if (int_params.irq_source == DC_IRQ_SOURCE_INVALID ||
- int_params.irq_source < DC_IRQ_SOURCE_PFLIP_FIRST ||
- int_params.irq_source > DC_IRQ_SOURCE_PFLIP_LAST) {
- drm_err(adev_to_drm(adev), "Failed to register pflip irq!\n");
- return -EINVAL;
- }
-
- c_irq_params = &adev->dm.pflip_params[int_params.irq_source - DC_IRQ_SOURCE_PFLIP_FIRST];
-
- c_irq_params->adev = adev;
- c_irq_params->irq_src = int_params.irq_source;
-
- if (!amdgpu_dm_irq_register_interrupt(adev, &int_params,
- dm_pflip_high_irq, c_irq_params))
- return -ENOMEM;
- }
-
/* HPD */
r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
&adev->hpd_irq);
drm_crtc_vblank_on_config(&acrtc->base,
&config);
- /* Allow RX6xxx, RX7700, RX7800 GPUs to call amdgpu_irq_get.*/
+ /*
+ * Since pflip_high_irq is no longer registered for DCN, grab an
+ * extra reference to vupdate irq instead to workaround this
+ * issue:
+ * https://gitlab.freedesktop.org/drm/amd/-/work_items/3936
+ *
+ * The callbacks to drm_vblank_on/off should really take care of
+ * this though.
+ */
switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
case IP_VERSION(3, 0, 0):
case IP_VERSION(3, 0, 2):
case IP_VERSION(3, 0, 3):
case IP_VERSION(3, 2, 0):
- if (amdgpu_irq_get(adev, &adev->pageflip_irq, irq_type))
- drm_err(dev, "DM_IRQ: Cannot get pageflip irq!\n");
+ if (amdgpu_irq_get(adev, &adev->vupdate_irq, irq_type))
+ drm_err(dev, "DM_IRQ: Cannot get vupdate irq!\n");
#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
if (amdgpu_irq_get(adev, &adev->vline0_irq, irq_type))
drm_err(dev, "DM_IRQ: Cannot get vline0 irq!\n");
if (amdgpu_irq_put(adev, &adev->vline0_irq, irq_type))
drm_err(dev, "DM_IRQ: Cannot put vline0 irq!\n");
#endif
- if (amdgpu_irq_put(adev, &adev->pageflip_irq, irq_type))
- drm_err(dev, "DM_IRQ: Cannot put pageflip irq!\n");
+ if (amdgpu_irq_put(adev, &adev->vupdate_irq, irq_type))
+ drm_err(dev, "DM_IRQ: Cannot put vupdate irq!\n");
}
drm_crtc_vblank_off(&acrtc->base);
int irq_type =
amdgpu_display_crtc_idx_to_irq_type(adev, acrtc->crtc_id);
+ /* GRPH_PFLIP is not used on DCN; nothing to reapply. */
+ if (amdgpu_ip_version(adev, DCE_HWIP, 0) != 0)
+ return;
+
/**
* This reads the current state for the IRQ and force reapplies
* the setting to hardware.
struct dm_crtc_state *old_state,
struct dm_crtc_state *new_state)
{
+ struct amdgpu_device *adev = drm_to_adev(new_state->base.crtc->dev);
bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state);
bool new_vrr_active = amdgpu_dm_crtc_vrr_active(new_state);
+ /* Only DCE gates vupdate on VRR, keep it enabled for DCN */
+ bool vrr_gates_vupdate = amdgpu_ip_version(adev, DCE_HWIP, 0) == 0;
+
if (!old_vrr_active && new_vrr_active) {
/* Transition VRR inactive -> active:
* While VRR is active, we must not disable vblank irq, as a
* We also need vupdate irq for the actual core vblank handling
* at end of vblank.
*/
- WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0);
+ if (vrr_gates_vupdate)
+ WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, true) != 0);
WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n",
__func__, new_state->base.crtc->base.id);
/* Transition VRR active -> inactive:
* Allow vblank irq disable again for fixed refresh rate.
*/
- WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0);
+ if (vrr_gates_vupdate)
+ WARN_ON(amdgpu_dm_crtc_set_vupdate_irq(new_state->base.crtc, false) != 0);
drm_crtc_vblank_put(new_state->base.crtc);
drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n",
__func__, new_state->base.crtc->base.id);
bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state);
bool cursor_update = false;
bool pflip_present = false;
+ bool immediate_flip = false;
bool dirty_rects_changed = false;
bool updated_planes_and_streams = false;
struct {
acrtc_state->update_type == UPDATE_TYPE_FAST &&
get_mem_type(old_plane_state->fb) == get_mem_type(fb);
+ immediate_flip |= bundle->flip_addrs[planes_count].flip_immediate;
+
timestamp_ns = ktime_get_ns();
bundle->flip_addrs[planes_count].flip_timestamp_in_us = div_u64(timestamp_ns, 1000);
bundle->surface_updates[planes_count].flip_addr = &bundle->flip_addrs[planes_count];
acrtc_state->cursor_mode == DM_CURSOR_NATIVE_MODE)
amdgpu_dm_commit_cursors(state);
+ /*
+ * On DCN, flip completion is normally delivered from VUPDATE_NO_LOCK.
+ * However, an immediate (tearing / async) flip is latched by HW right
+ * away and does not wait for the next vupdate, so deliver its
+ * completion event here after programming.
+ *
+ * On DCE, GRPH_PFLIP already fires immediately for immediate flips, so
+ * this is DCN-only.
+ */
+ if (immediate_flip && amdgpu_ip_version(dm->adev, DCE_HWIP, 0) != 0) {
+ spin_lock_irqsave(&pcrtc->dev->event_lock, flags);
+ if (acrtc_attach->pflip_status == AMDGPU_FLIP_SUBMITTED &&
+ acrtc_attach->event) {
+ drm_crtc_accurate_vblank_count(&acrtc_attach->base);
+ drm_crtc_send_vblank_event(&acrtc_attach->base,
+ acrtc_attach->event);
+ acrtc_attach->event = NULL;
+ drm_crtc_vblank_put(&acrtc_attach->base);
+ acrtc_attach->pflip_status = AMDGPU_FLIP_NONE;
+ }
+ spin_unlock_irqrestore(&pcrtc->dev->event_lock, flags);
+ }
+
cleanup:
kfree(bundle);
}
projects / thirdparty / kernel / linux.git / commitdiff
