2 * Copyright © 2013 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 #include "intel_drv.h"
26 #include "i915_vgpu.h"
28 #include <asm/iosf_mbi.h>
29 #include <linux/pm_runtime.h>
31 #define FORCEWAKE_ACK_TIMEOUT_MS 50
32 #define GT_FIFO_TIMEOUT_MS 10
34 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
36 static const char * const forcewake_domain_names
[] = {
49 intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id
)
51 BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names
) != FW_DOMAIN_ID_COUNT
);
53 if (id
>= 0 && id
< FW_DOMAIN_ID_COUNT
)
54 return forcewake_domain_names
[id
];
62 fw_domain_reset(struct drm_i915_private
*i915
,
63 const struct intel_uncore_forcewake_domain
*d
)
66 * We don't really know if the powerwell for the forcewake domain we are
67 * trying to reset here does exist at this point (engines could be fused
68 * off in ICL+), so no waiting for acks
70 __raw_i915_write32(i915
, d
->reg_set
, i915
->uncore
.fw_reset
);
74 fw_domain_arm_timer(struct intel_uncore_forcewake_domain
*d
)
77 hrtimer_start_range_ns(&d
->timer
,
84 __wait_for_ack(const struct drm_i915_private
*i915
,
85 const struct intel_uncore_forcewake_domain
*d
,
89 return wait_for_atomic((__raw_i915_read32(i915
, d
->reg_ack
) & ack
) == value
,
90 FORCEWAKE_ACK_TIMEOUT_MS
);
94 wait_ack_clear(const struct drm_i915_private
*i915
,
95 const struct intel_uncore_forcewake_domain
*d
,
98 return __wait_for_ack(i915
, d
, ack
, 0);
102 wait_ack_set(const struct drm_i915_private
*i915
,
103 const struct intel_uncore_forcewake_domain
*d
,
106 return __wait_for_ack(i915
, d
, ack
, ack
);
110 fw_domain_wait_ack_clear(const struct drm_i915_private
*i915
,
111 const struct intel_uncore_forcewake_domain
*d
)
113 if (wait_ack_clear(i915
, d
, FORCEWAKE_KERNEL
))
114 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
115 intel_uncore_forcewake_domain_to_str(d
->id
));
124 fw_domain_wait_ack_with_fallback(const struct drm_i915_private
*i915
,
125 const struct intel_uncore_forcewake_domain
*d
,
126 const enum ack_type type
)
128 const u32 ack_bit
= FORCEWAKE_KERNEL
;
129 const u32 value
= type
== ACK_SET
? ack_bit
: 0;
134 * There is a possibility of driver's wake request colliding
135 * with hardware's own wake requests and that can cause
136 * hardware to not deliver the driver's ack message.
138 * Use a fallback bit toggle to kick the gpu state machine
139 * in the hope that the original ack will be delivered along with
142 * This workaround is described in HSDES #1604254524 and it's known as:
143 * WaRsForcewakeAddDelayForAck:skl,bxt,kbl,glk,cfl,cnl,icl
144 * although the name is a bit misleading.
149 wait_ack_clear(i915
, d
, FORCEWAKE_KERNEL_FALLBACK
);
151 __raw_i915_write32(i915
, d
->reg_set
,
152 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK
));
153 /* Give gt some time to relax before the polling frenzy */
155 wait_ack_set(i915
, d
, FORCEWAKE_KERNEL_FALLBACK
);
157 ack_detected
= (__raw_i915_read32(i915
, d
->reg_ack
) & ack_bit
) == value
;
159 __raw_i915_write32(i915
, d
->reg_set
,
160 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK
));
161 } while (!ack_detected
&& pass
++ < 10);
163 DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
164 intel_uncore_forcewake_domain_to_str(d
->id
),
165 type
== ACK_SET
? "set" : "clear",
166 __raw_i915_read32(i915
, d
->reg_ack
),
169 return ack_detected
? 0 : -ETIMEDOUT
;
173 fw_domain_wait_ack_clear_fallback(const struct drm_i915_private
*i915
,
174 const struct intel_uncore_forcewake_domain
*d
)
176 if (likely(!wait_ack_clear(i915
, d
, FORCEWAKE_KERNEL
)))
179 if (fw_domain_wait_ack_with_fallback(i915
, d
, ACK_CLEAR
))
180 fw_domain_wait_ack_clear(i915
, d
);
184 fw_domain_get(struct drm_i915_private
*i915
,
185 const struct intel_uncore_forcewake_domain
*d
)
187 __raw_i915_write32(i915
, d
->reg_set
, i915
->uncore
.fw_set
);
191 fw_domain_wait_ack_set(const struct drm_i915_private
*i915
,
192 const struct intel_uncore_forcewake_domain
*d
)
194 if (wait_ack_set(i915
, d
, FORCEWAKE_KERNEL
))
195 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
196 intel_uncore_forcewake_domain_to_str(d
->id
));
200 fw_domain_wait_ack_set_fallback(const struct drm_i915_private
*i915
,
201 const struct intel_uncore_forcewake_domain
*d
)
203 if (likely(!wait_ack_set(i915
, d
, FORCEWAKE_KERNEL
)))
206 if (fw_domain_wait_ack_with_fallback(i915
, d
, ACK_SET
))
207 fw_domain_wait_ack_set(i915
, d
);
211 fw_domain_put(const struct drm_i915_private
*i915
,
212 const struct intel_uncore_forcewake_domain
*d
)
214 __raw_i915_write32(i915
, d
->reg_set
, i915
->uncore
.fw_clear
);
218 fw_domains_get(struct drm_i915_private
*i915
, enum forcewake_domains fw_domains
)
220 struct intel_uncore_forcewake_domain
*d
;
223 GEM_BUG_ON(fw_domains
& ~i915
->uncore
.fw_domains
);
225 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
) {
226 fw_domain_wait_ack_clear(i915
, d
);
227 fw_domain_get(i915
, d
);
230 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
)
231 fw_domain_wait_ack_set(i915
, d
);
233 i915
->uncore
.fw_domains_active
|= fw_domains
;
237 fw_domains_get_with_fallback(struct drm_i915_private
*i915
,
238 enum forcewake_domains fw_domains
)
240 struct intel_uncore_forcewake_domain
*d
;
243 GEM_BUG_ON(fw_domains
& ~i915
->uncore
.fw_domains
);
245 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
) {
246 fw_domain_wait_ack_clear_fallback(i915
, d
);
247 fw_domain_get(i915
, d
);
250 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
)
251 fw_domain_wait_ack_set_fallback(i915
, d
);
253 i915
->uncore
.fw_domains_active
|= fw_domains
;
257 fw_domains_put(struct drm_i915_private
*i915
, enum forcewake_domains fw_domains
)
259 struct intel_uncore_forcewake_domain
*d
;
262 GEM_BUG_ON(fw_domains
& ~i915
->uncore
.fw_domains
);
264 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
)
265 fw_domain_put(i915
, d
);
267 i915
->uncore
.fw_domains_active
&= ~fw_domains
;
271 fw_domains_reset(struct drm_i915_private
*i915
,
272 enum forcewake_domains fw_domains
)
274 struct intel_uncore_forcewake_domain
*d
;
280 GEM_BUG_ON(fw_domains
& ~i915
->uncore
.fw_domains
);
282 for_each_fw_domain_masked(d
, fw_domains
, i915
, tmp
)
283 fw_domain_reset(i915
, d
);
286 static inline u32
gt_thread_status(struct drm_i915_private
*dev_priv
)
290 val
= __raw_i915_read32(dev_priv
, GEN6_GT_THREAD_STATUS_REG
);
291 val
&= GEN6_GT_THREAD_STATUS_CORE_MASK
;
296 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private
*dev_priv
)
299 * w/a for a sporadic read returning 0 by waiting for the GT
302 WARN_ONCE(wait_for_atomic_us(gt_thread_status(dev_priv
) == 0, 5000),
303 "GT thread status wait timed out\n");
306 static void fw_domains_get_with_thread_status(struct drm_i915_private
*dev_priv
,
307 enum forcewake_domains fw_domains
)
309 fw_domains_get(dev_priv
, fw_domains
);
311 /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
312 __gen6_gt_wait_for_thread_c0(dev_priv
);
315 static inline u32
fifo_free_entries(struct drm_i915_private
*dev_priv
)
317 u32 count
= __raw_i915_read32(dev_priv
, GTFIFOCTL
);
319 return count
& GT_FIFO_FREE_ENTRIES_MASK
;
322 static void __gen6_gt_wait_for_fifo(struct drm_i915_private
*dev_priv
)
326 /* On VLV, FIFO will be shared by both SW and HW.
327 * So, we need to read the FREE_ENTRIES everytime */
328 if (IS_VALLEYVIEW(dev_priv
))
329 n
= fifo_free_entries(dev_priv
);
331 n
= dev_priv
->uncore
.fifo_count
;
333 if (n
<= GT_FIFO_NUM_RESERVED_ENTRIES
) {
334 if (wait_for_atomic((n
= fifo_free_entries(dev_priv
)) >
335 GT_FIFO_NUM_RESERVED_ENTRIES
,
336 GT_FIFO_TIMEOUT_MS
)) {
337 DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n
);
342 dev_priv
->uncore
.fifo_count
= n
- 1;
345 static enum hrtimer_restart
346 intel_uncore_fw_release_timer(struct hrtimer
*timer
)
348 struct intel_uncore_forcewake_domain
*domain
=
349 container_of(timer
, struct intel_uncore_forcewake_domain
, timer
);
350 struct drm_i915_private
*dev_priv
=
351 container_of(domain
, struct drm_i915_private
, uncore
.fw_domain
[domain
->id
]);
352 unsigned long irqflags
;
354 assert_rpm_device_not_suspended(dev_priv
);
356 if (xchg(&domain
->active
, false))
357 return HRTIMER_RESTART
;
359 spin_lock_irqsave(&dev_priv
->uncore
.lock
, irqflags
);
360 if (WARN_ON(domain
->wake_count
== 0))
361 domain
->wake_count
++;
363 if (--domain
->wake_count
== 0)
364 dev_priv
->uncore
.funcs
.force_wake_put(dev_priv
, domain
->mask
);
366 spin_unlock_irqrestore(&dev_priv
->uncore
.lock
, irqflags
);
368 return HRTIMER_NORESTART
;
371 /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
373 intel_uncore_forcewake_reset(struct drm_i915_private
*dev_priv
)
375 unsigned long irqflags
;
376 struct intel_uncore_forcewake_domain
*domain
;
377 int retry_count
= 100;
378 enum forcewake_domains fw
, active_domains
;
380 iosf_mbi_assert_punit_acquired();
382 /* Hold uncore.lock across reset to prevent any register access
383 * with forcewake not set correctly. Wait until all pending
384 * timers are run before holding.
391 for_each_fw_domain(domain
, dev_priv
, tmp
) {
392 smp_store_mb(domain
->active
, false);
393 if (hrtimer_cancel(&domain
->timer
) == 0)
396 intel_uncore_fw_release_timer(&domain
->timer
);
399 spin_lock_irqsave(&dev_priv
->uncore
.lock
, irqflags
);
401 for_each_fw_domain(domain
, dev_priv
, tmp
) {
402 if (hrtimer_active(&domain
->timer
))
403 active_domains
|= domain
->mask
;
406 if (active_domains
== 0)
409 if (--retry_count
== 0) {
410 DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
414 spin_unlock_irqrestore(&dev_priv
->uncore
.lock
, irqflags
);
418 WARN_ON(active_domains
);
420 fw
= dev_priv
->uncore
.fw_domains_active
;
422 dev_priv
->uncore
.funcs
.force_wake_put(dev_priv
, fw
);
424 fw_domains_reset(dev_priv
, dev_priv
->uncore
.fw_domains
);
425 assert_forcewakes_inactive(dev_priv
);
427 spin_unlock_irqrestore(&dev_priv
->uncore
.lock
, irqflags
);
429 return fw
; /* track the lost user forcewake domains */
432 static u64
gen9_edram_size(struct drm_i915_private
*dev_priv
)
434 const unsigned int ways
[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
435 const unsigned int sets
[4] = { 1, 1, 2, 2 };
436 const u32 cap
= dev_priv
->edram_cap
;
438 return EDRAM_NUM_BANKS(cap
) *
439 ways
[EDRAM_WAYS_IDX(cap
)] *
440 sets
[EDRAM_SETS_IDX(cap
)] *
444 u64
intel_uncore_edram_size(struct drm_i915_private
*dev_priv
)
446 if (!HAS_EDRAM(dev_priv
))
449 /* The needed capability bits for size calculation
450 * are not there with pre gen9 so return 128MB always.
452 if (INTEL_GEN(dev_priv
) < 9)
453 return 128 * 1024 * 1024;
455 return gen9_edram_size(dev_priv
);
458 static void intel_uncore_edram_detect(struct drm_i915_private
*dev_priv
)
460 if (IS_HASWELL(dev_priv
) ||
461 IS_BROADWELL(dev_priv
) ||
462 INTEL_GEN(dev_priv
) >= 9) {
463 dev_priv
->edram_cap
= __raw_i915_read32(dev_priv
,
466 /* NB: We can't write IDICR yet because we do not have gt funcs
469 dev_priv
->edram_cap
= 0;
472 if (HAS_EDRAM(dev_priv
))
473 DRM_INFO("Found %lluMB of eDRAM\n",
474 intel_uncore_edram_size(dev_priv
) / (1024 * 1024));
478 fpga_check_for_unclaimed_mmio(struct drm_i915_private
*dev_priv
)
482 dbg
= __raw_i915_read32(dev_priv
, FPGA_DBG
);
483 if (likely(!(dbg
& FPGA_DBG_RM_NOCLAIM
)))
486 __raw_i915_write32(dev_priv
, FPGA_DBG
, FPGA_DBG_RM_NOCLAIM
);
492 vlv_check_for_unclaimed_mmio(struct drm_i915_private
*dev_priv
)
496 cer
= __raw_i915_read32(dev_priv
, CLAIM_ER
);
497 if (likely(!(cer
& (CLAIM_ER_OVERFLOW
| CLAIM_ER_CTR_MASK
))))
500 __raw_i915_write32(dev_priv
, CLAIM_ER
, CLAIM_ER_CLR
);
506 gen6_check_for_fifo_debug(struct drm_i915_private
*dev_priv
)
510 fifodbg
= __raw_i915_read32(dev_priv
, GTFIFODBG
);
512 if (unlikely(fifodbg
)) {
513 DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg
);
514 __raw_i915_write32(dev_priv
, GTFIFODBG
, fifodbg
);
521 check_for_unclaimed_mmio(struct drm_i915_private
*dev_priv
)
525 if (HAS_FPGA_DBG_UNCLAIMED(dev_priv
))
526 ret
|= fpga_check_for_unclaimed_mmio(dev_priv
);
528 if (IS_VALLEYVIEW(dev_priv
) || IS_CHERRYVIEW(dev_priv
))
529 ret
|= vlv_check_for_unclaimed_mmio(dev_priv
);
531 if (IS_GEN6(dev_priv
) || IS_GEN7(dev_priv
))
532 ret
|= gen6_check_for_fifo_debug(dev_priv
);
537 static void __intel_uncore_early_sanitize(struct drm_i915_private
*dev_priv
,
538 unsigned int restore_forcewake
)
540 /* clear out unclaimed reg detection bit */
541 if (check_for_unclaimed_mmio(dev_priv
))
542 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
544 /* WaDisableShadowRegForCpd:chv */
545 if (IS_CHERRYVIEW(dev_priv
)) {
546 __raw_i915_write32(dev_priv
, GTFIFOCTL
,
547 __raw_i915_read32(dev_priv
, GTFIFOCTL
) |
548 GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL
|
549 GT_FIFO_CTL_RC6_POLICY_STALL
);
552 iosf_mbi_punit_acquire();
553 intel_uncore_forcewake_reset(dev_priv
);
554 if (restore_forcewake
) {
555 spin_lock_irq(&dev_priv
->uncore
.lock
);
556 dev_priv
->uncore
.funcs
.force_wake_get(dev_priv
,
559 if (IS_GEN6(dev_priv
) || IS_GEN7(dev_priv
))
560 dev_priv
->uncore
.fifo_count
=
561 fifo_free_entries(dev_priv
);
562 spin_unlock_irq(&dev_priv
->uncore
.lock
);
564 iosf_mbi_punit_release();
567 void intel_uncore_suspend(struct drm_i915_private
*dev_priv
)
569 iosf_mbi_punit_acquire();
570 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
571 &dev_priv
->uncore
.pmic_bus_access_nb
);
572 dev_priv
->uncore
.fw_domains_saved
=
573 intel_uncore_forcewake_reset(dev_priv
);
574 iosf_mbi_punit_release();
577 void intel_uncore_resume_early(struct drm_i915_private
*dev_priv
)
579 unsigned int restore_forcewake
;
581 restore_forcewake
= fetch_and_zero(&dev_priv
->uncore
.fw_domains_saved
);
582 __intel_uncore_early_sanitize(dev_priv
, restore_forcewake
);
584 iosf_mbi_register_pmic_bus_access_notifier(
585 &dev_priv
->uncore
.pmic_bus_access_nb
);
586 i915_check_and_clear_faults(dev_priv
);
589 void intel_uncore_runtime_resume(struct drm_i915_private
*dev_priv
)
591 iosf_mbi_register_pmic_bus_access_notifier(
592 &dev_priv
->uncore
.pmic_bus_access_nb
);
595 void intel_uncore_sanitize(struct drm_i915_private
*dev_priv
)
597 /* BIOS often leaves RC6 enabled, but disable it for hw init */
598 intel_sanitize_gt_powersave(dev_priv
);
601 static void __intel_uncore_forcewake_get(struct drm_i915_private
*dev_priv
,
602 enum forcewake_domains fw_domains
)
604 struct intel_uncore_forcewake_domain
*domain
;
607 fw_domains
&= dev_priv
->uncore
.fw_domains
;
609 for_each_fw_domain_masked(domain
, fw_domains
, dev_priv
, tmp
) {
610 if (domain
->wake_count
++) {
611 fw_domains
&= ~domain
->mask
;
612 domain
->active
= true;
617 dev_priv
->uncore
.funcs
.force_wake_get(dev_priv
, fw_domains
);
621 * intel_uncore_forcewake_get - grab forcewake domain references
622 * @dev_priv: i915 device instance
623 * @fw_domains: forcewake domains to get reference on
625 * This function can be used get GT's forcewake domain references.
626 * Normal register access will handle the forcewake domains automatically.
627 * However if some sequence requires the GT to not power down a particular
628 * forcewake domains this function should be called at the beginning of the
629 * sequence. And subsequently the reference should be dropped by symmetric
630 * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
631 * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
633 void intel_uncore_forcewake_get(struct drm_i915_private
*dev_priv
,
634 enum forcewake_domains fw_domains
)
636 unsigned long irqflags
;
638 if (!dev_priv
->uncore
.funcs
.force_wake_get
)
641 assert_rpm_wakelock_held(dev_priv
);
643 spin_lock_irqsave(&dev_priv
->uncore
.lock
, irqflags
);
644 __intel_uncore_forcewake_get(dev_priv
, fw_domains
);
645 spin_unlock_irqrestore(&dev_priv
->uncore
.lock
, irqflags
);
649 * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
650 * @dev_priv: i915 device instance
652 * This function is a wrapper around intel_uncore_forcewake_get() to acquire
653 * the GT powerwell and in the process disable our debugging for the
654 * duration of userspace's bypass.
656 void intel_uncore_forcewake_user_get(struct drm_i915_private
*dev_priv
)
658 spin_lock_irq(&dev_priv
->uncore
.lock
);
659 if (!dev_priv
->uncore
.user_forcewake
.count
++) {
660 intel_uncore_forcewake_get__locked(dev_priv
, FORCEWAKE_ALL
);
662 /* Save and disable mmio debugging for the user bypass */
663 dev_priv
->uncore
.user_forcewake
.saved_mmio_check
=
664 dev_priv
->uncore
.unclaimed_mmio_check
;
665 dev_priv
->uncore
.user_forcewake
.saved_mmio_debug
=
666 i915_modparams
.mmio_debug
;
668 dev_priv
->uncore
.unclaimed_mmio_check
= 0;
669 i915_modparams
.mmio_debug
= 0;
671 spin_unlock_irq(&dev_priv
->uncore
.lock
);
675 * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
676 * @dev_priv: i915 device instance
678 * This function complements intel_uncore_forcewake_user_get() and releases
679 * the GT powerwell taken on behalf of the userspace bypass.
681 void intel_uncore_forcewake_user_put(struct drm_i915_private
*dev_priv
)
683 spin_lock_irq(&dev_priv
->uncore
.lock
);
684 if (!--dev_priv
->uncore
.user_forcewake
.count
) {
685 if (intel_uncore_unclaimed_mmio(dev_priv
))
686 dev_info(dev_priv
->drm
.dev
,
687 "Invalid mmio detected during user access\n");
689 dev_priv
->uncore
.unclaimed_mmio_check
=
690 dev_priv
->uncore
.user_forcewake
.saved_mmio_check
;
691 i915_modparams
.mmio_debug
=
692 dev_priv
->uncore
.user_forcewake
.saved_mmio_debug
;
694 intel_uncore_forcewake_put__locked(dev_priv
, FORCEWAKE_ALL
);
696 spin_unlock_irq(&dev_priv
->uncore
.lock
);
700 * intel_uncore_forcewake_get__locked - grab forcewake domain references
701 * @dev_priv: i915 device instance
702 * @fw_domains: forcewake domains to get reference on
704 * See intel_uncore_forcewake_get(). This variant places the onus
705 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
707 void intel_uncore_forcewake_get__locked(struct drm_i915_private
*dev_priv
,
708 enum forcewake_domains fw_domains
)
710 lockdep_assert_held(&dev_priv
->uncore
.lock
);
712 if (!dev_priv
->uncore
.funcs
.force_wake_get
)
715 __intel_uncore_forcewake_get(dev_priv
, fw_domains
);
718 static void __intel_uncore_forcewake_put(struct drm_i915_private
*dev_priv
,
719 enum forcewake_domains fw_domains
)
721 struct intel_uncore_forcewake_domain
*domain
;
724 fw_domains
&= dev_priv
->uncore
.fw_domains
;
726 for_each_fw_domain_masked(domain
, fw_domains
, dev_priv
, tmp
) {
727 if (WARN_ON(domain
->wake_count
== 0))
730 if (--domain
->wake_count
) {
731 domain
->active
= true;
735 fw_domain_arm_timer(domain
);
740 * intel_uncore_forcewake_put - release a forcewake domain reference
741 * @dev_priv: i915 device instance
742 * @fw_domains: forcewake domains to put references
744 * This function drops the device-level forcewakes for specified
745 * domains obtained by intel_uncore_forcewake_get().
747 void intel_uncore_forcewake_put(struct drm_i915_private
*dev_priv
,
748 enum forcewake_domains fw_domains
)
750 unsigned long irqflags
;
752 if (!dev_priv
->uncore
.funcs
.force_wake_put
)
755 spin_lock_irqsave(&dev_priv
->uncore
.lock
, irqflags
);
756 __intel_uncore_forcewake_put(dev_priv
, fw_domains
);
757 spin_unlock_irqrestore(&dev_priv
->uncore
.lock
, irqflags
);
761 * intel_uncore_forcewake_put__locked - grab forcewake domain references
762 * @dev_priv: i915 device instance
763 * @fw_domains: forcewake domains to get reference on
765 * See intel_uncore_forcewake_put(). This variant places the onus
766 * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
768 void intel_uncore_forcewake_put__locked(struct drm_i915_private
*dev_priv
,
769 enum forcewake_domains fw_domains
)
771 lockdep_assert_held(&dev_priv
->uncore
.lock
);
773 if (!dev_priv
->uncore
.funcs
.force_wake_put
)
776 __intel_uncore_forcewake_put(dev_priv
, fw_domains
);
779 void assert_forcewakes_inactive(struct drm_i915_private
*dev_priv
)
781 if (!dev_priv
->uncore
.funcs
.force_wake_get
)
784 WARN(dev_priv
->uncore
.fw_domains_active
,
785 "Expected all fw_domains to be inactive, but %08x are still on\n",
786 dev_priv
->uncore
.fw_domains_active
);
789 void assert_forcewakes_active(struct drm_i915_private
*dev_priv
,
790 enum forcewake_domains fw_domains
)
792 if (!dev_priv
->uncore
.funcs
.force_wake_get
)
795 assert_rpm_wakelock_held(dev_priv
);
797 fw_domains
&= dev_priv
->uncore
.fw_domains
;
798 WARN(fw_domains
& ~dev_priv
->uncore
.fw_domains_active
,
799 "Expected %08x fw_domains to be active, but %08x are off\n",
800 fw_domains
, fw_domains
& ~dev_priv
->uncore
.fw_domains_active
);
803 /* We give fast paths for the really cool registers */
804 #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
806 #define GEN11_NEEDS_FORCE_WAKE(reg) \
807 ((reg) < 0x40000 || ((reg) >= 0x1c0000 && (reg) < 0x1dc000))
809 #define __gen6_reg_read_fw_domains(offset) \
811 enum forcewake_domains __fwd; \
812 if (NEEDS_FORCE_WAKE(offset)) \
813 __fwd = FORCEWAKE_RENDER; \
819 static int fw_range_cmp(u32 offset
, const struct intel_forcewake_range
*entry
)
821 if (offset
< entry
->start
)
823 else if (offset
> entry
->end
)
829 /* Copied and "macroized" from lib/bsearch.c */
830 #define BSEARCH(key, base, num, cmp) ({ \
831 unsigned int start__ = 0, end__ = (num); \
832 typeof(base) result__ = NULL; \
833 while (start__ < end__) { \
834 unsigned int mid__ = start__ + (end__ - start__) / 2; \
835 int ret__ = (cmp)((key), (base) + mid__); \
838 } else if (ret__ > 0) { \
839 start__ = mid__ + 1; \
841 result__ = (base) + mid__; \
848 static enum forcewake_domains
849 find_fw_domain(struct drm_i915_private
*dev_priv
, u32 offset
)
851 const struct intel_forcewake_range
*entry
;
853 entry
= BSEARCH(offset
,
854 dev_priv
->uncore
.fw_domains_table
,
855 dev_priv
->uncore
.fw_domains_table_entries
,
862 * The list of FW domains depends on the SKU in gen11+ so we
863 * can't determine it statically. We use FORCEWAKE_ALL and
864 * translate it here to the list of available domains.
866 if (entry
->domains
== FORCEWAKE_ALL
)
867 return dev_priv
->uncore
.fw_domains
;
869 WARN(entry
->domains
& ~dev_priv
->uncore
.fw_domains
,
870 "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
871 entry
->domains
& ~dev_priv
->uncore
.fw_domains
, offset
);
873 return entry
->domains
;
876 #define GEN_FW_RANGE(s, e, d) \
877 { .start = (s), .end = (e), .domains = (d) }
879 #define HAS_FWTABLE(dev_priv) \
880 (INTEL_GEN(dev_priv) >= 9 || \
881 IS_CHERRYVIEW(dev_priv) || \
882 IS_VALLEYVIEW(dev_priv))
884 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
885 static const struct intel_forcewake_range __vlv_fw_ranges
[] = {
886 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER
),
887 GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER
),
888 GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER
),
889 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA
),
890 GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA
),
891 GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER
),
892 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA
),
895 #define __fwtable_reg_read_fw_domains(offset) \
897 enum forcewake_domains __fwd = 0; \
898 if (NEEDS_FORCE_WAKE((offset))) \
899 __fwd = find_fw_domain(dev_priv, offset); \
903 #define __gen11_fwtable_reg_read_fw_domains(offset) \
905 enum forcewake_domains __fwd = 0; \
906 if (GEN11_NEEDS_FORCE_WAKE((offset))) \
907 __fwd = find_fw_domain(dev_priv, offset); \
911 /* *Must* be sorted by offset! See intel_shadow_table_check(). */
912 static const i915_reg_t gen8_shadowed_regs
[] = {
913 RING_TAIL(RENDER_RING_BASE
), /* 0x2000 (base) */
914 GEN6_RPNSWREQ
, /* 0xA008 */
915 GEN6_RC_VIDEO_FREQ
, /* 0xA00C */
916 RING_TAIL(GEN6_BSD_RING_BASE
), /* 0x12000 (base) */
917 RING_TAIL(VEBOX_RING_BASE
), /* 0x1a000 (base) */
918 RING_TAIL(BLT_RING_BASE
), /* 0x22000 (base) */
919 /* TODO: Other registers are not yet used */
922 static const i915_reg_t gen11_shadowed_regs
[] = {
923 RING_TAIL(RENDER_RING_BASE
), /* 0x2000 (base) */
924 GEN6_RPNSWREQ
, /* 0xA008 */
925 GEN6_RC_VIDEO_FREQ
, /* 0xA00C */
926 RING_TAIL(BLT_RING_BASE
), /* 0x22000 (base) */
927 RING_TAIL(GEN11_BSD_RING_BASE
), /* 0x1C0000 (base) */
928 RING_TAIL(GEN11_BSD2_RING_BASE
), /* 0x1C4000 (base) */
929 RING_TAIL(GEN11_VEBOX_RING_BASE
), /* 0x1C8000 (base) */
930 RING_TAIL(GEN11_BSD3_RING_BASE
), /* 0x1D0000 (base) */
931 RING_TAIL(GEN11_BSD4_RING_BASE
), /* 0x1D4000 (base) */
932 RING_TAIL(GEN11_VEBOX2_RING_BASE
), /* 0x1D8000 (base) */
933 /* TODO: Other registers are not yet used */
936 static int mmio_reg_cmp(u32 key
, const i915_reg_t
*reg
)
938 u32 offset
= i915_mmio_reg_offset(*reg
);
942 else if (key
> offset
)
948 #define __is_genX_shadowed(x) \
949 static bool is_gen##x##_shadowed(u32 offset) \
951 const i915_reg_t *regs = gen##x##_shadowed_regs; \
952 return BSEARCH(offset, regs, ARRAY_SIZE(gen##x##_shadowed_regs), \
956 __is_genX_shadowed(8)
957 __is_genX_shadowed(11)
959 #define __gen8_reg_write_fw_domains(offset) \
961 enum forcewake_domains __fwd; \
962 if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
963 __fwd = FORCEWAKE_RENDER; \
969 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
970 static const struct intel_forcewake_range __chv_fw_ranges
[] = {
971 GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER
),
972 GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
973 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER
),
974 GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
975 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER
),
976 GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
977 GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA
),
978 GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
979 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER
),
980 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA
),
981 GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER
),
982 GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
983 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA
),
984 GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA
),
985 GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA
),
986 GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA
),
989 #define __fwtable_reg_write_fw_domains(offset) \
991 enum forcewake_domains __fwd = 0; \
992 if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
993 __fwd = find_fw_domain(dev_priv, offset); \
997 #define __gen11_fwtable_reg_write_fw_domains(offset) \
999 enum forcewake_domains __fwd = 0; \
1000 if (GEN11_NEEDS_FORCE_WAKE((offset)) && !is_gen11_shadowed(offset)) \
1001 __fwd = find_fw_domain(dev_priv, offset); \
1005 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
1006 static const struct intel_forcewake_range __gen9_fw_ranges
[] = {
1007 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER
),
1008 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1009 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER
),
1010 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER
),
1011 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER
),
1012 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER
),
1013 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER
),
1014 GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER
),
1015 GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA
),
1016 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER
),
1017 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER
),
1018 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER
),
1019 GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER
),
1020 GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA
),
1021 GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER
),
1022 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER
),
1023 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER
),
1024 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER
| FORCEWAKE_MEDIA
),
1025 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER
),
1026 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER
),
1027 GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER
),
1028 GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA
),
1029 GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER
),
1030 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER
),
1031 GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER
),
1032 GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA
),
1033 GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER
),
1034 GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA
),
1035 GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER
),
1036 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER
),
1037 GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER
),
1038 GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA
),
1041 /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
1042 static const struct intel_forcewake_range __gen11_fw_ranges
[] = {
1043 GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER
),
1044 GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
1045 GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER
),
1046 GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER
),
1047 GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER
),
1048 GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER
),
1049 GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER
),
1050 GEN_FW_RANGE(0x8000, 0x813f, FORCEWAKE_BLITTER
),
1051 GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER
),
1052 GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER
),
1053 GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER
),
1054 GEN_FW_RANGE(0x8500, 0x8bff, FORCEWAKE_BLITTER
),
1055 GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER
),
1056 GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER
),
1057 GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_ALL
),
1058 GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER
),
1059 GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER
),
1060 GEN_FW_RANGE(0xb480, 0xdfff, FORCEWAKE_BLITTER
),
1061 GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER
),
1062 GEN_FW_RANGE(0xe900, 0x243ff, FORCEWAKE_BLITTER
),
1063 GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER
),
1064 GEN_FW_RANGE(0x24800, 0x3ffff, FORCEWAKE_BLITTER
),
1065 GEN_FW_RANGE(0x40000, 0x1bffff, 0),
1066 GEN_FW_RANGE(0x1c0000, 0x1c3fff, FORCEWAKE_MEDIA_VDBOX0
),
1067 GEN_FW_RANGE(0x1c4000, 0x1c7fff, FORCEWAKE_MEDIA_VDBOX1
),
1068 GEN_FW_RANGE(0x1c8000, 0x1cbfff, FORCEWAKE_MEDIA_VEBOX0
),
1069 GEN_FW_RANGE(0x1cc000, 0x1cffff, FORCEWAKE_BLITTER
),
1070 GEN_FW_RANGE(0x1d0000, 0x1d3fff, FORCEWAKE_MEDIA_VDBOX2
),
1071 GEN_FW_RANGE(0x1d4000, 0x1d7fff, FORCEWAKE_MEDIA_VDBOX3
),
1072 GEN_FW_RANGE(0x1d8000, 0x1dbfff, FORCEWAKE_MEDIA_VEBOX1
)
1076 ilk_dummy_write(struct drm_i915_private
*dev_priv
)
1078 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
1079 * the chip from rc6 before touching it for real. MI_MODE is masked,
1080 * hence harmless to write 0 into. */
1081 __raw_i915_write32(dev_priv
, MI_MODE
, 0);
1085 __unclaimed_reg_debug(struct drm_i915_private
*dev_priv
,
1086 const i915_reg_t reg
,
1090 if (WARN(check_for_unclaimed_mmio(dev_priv
) && !before
,
1091 "Unclaimed %s register 0x%x\n",
1092 read
? "read from" : "write to",
1093 i915_mmio_reg_offset(reg
)))
1094 /* Only report the first N failures */
1095 i915_modparams
.mmio_debug
--;
1099 unclaimed_reg_debug(struct drm_i915_private
*dev_priv
,
1100 const i915_reg_t reg
,
1104 if (likely(!i915_modparams
.mmio_debug
))
1107 __unclaimed_reg_debug(dev_priv
, reg
, read
, before
);
1110 #define GEN2_READ_HEADER(x) \
1112 assert_rpm_wakelock_held(dev_priv);
1114 #define GEN2_READ_FOOTER \
1115 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1118 #define __gen2_read(x) \
1120 gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1121 GEN2_READ_HEADER(x); \
1122 val = __raw_i915_read##x(dev_priv, reg); \
1126 #define __gen5_read(x) \
1128 gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1129 GEN2_READ_HEADER(x); \
1130 ilk_dummy_write(dev_priv); \
1131 val = __raw_i915_read##x(dev_priv, reg); \
1147 #undef GEN2_READ_FOOTER
1148 #undef GEN2_READ_HEADER
1150 #define GEN6_READ_HEADER(x) \
1151 u32 offset = i915_mmio_reg_offset(reg); \
1152 unsigned long irqflags; \
1154 assert_rpm_wakelock_held(dev_priv); \
1155 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1156 unclaimed_reg_debug(dev_priv, reg, true, true)
1158 #define GEN6_READ_FOOTER \
1159 unclaimed_reg_debug(dev_priv, reg, true, false); \
1160 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
1161 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
1164 static noinline
void ___force_wake_auto(struct drm_i915_private
*dev_priv
,
1165 enum forcewake_domains fw_domains
)
1167 struct intel_uncore_forcewake_domain
*domain
;
1170 GEM_BUG_ON(fw_domains
& ~dev_priv
->uncore
.fw_domains
);
1172 for_each_fw_domain_masked(domain
, fw_domains
, dev_priv
, tmp
)
1173 fw_domain_arm_timer(domain
);
1175 dev_priv
->uncore
.funcs
.force_wake_get(dev_priv
, fw_domains
);
1178 static inline void __force_wake_auto(struct drm_i915_private
*dev_priv
,
1179 enum forcewake_domains fw_domains
)
1181 if (WARN_ON(!fw_domains
))
1184 /* Turn on all requested but inactive supported forcewake domains. */
1185 fw_domains
&= dev_priv
->uncore
.fw_domains
;
1186 fw_domains
&= ~dev_priv
->uncore
.fw_domains_active
;
1189 ___force_wake_auto(dev_priv
, fw_domains
);
1192 #define __gen_read(func, x) \
1194 func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
1195 enum forcewake_domains fw_engine; \
1196 GEN6_READ_HEADER(x); \
1197 fw_engine = __##func##_reg_read_fw_domains(offset); \
1199 __force_wake_auto(dev_priv, fw_engine); \
1200 val = __raw_i915_read##x(dev_priv, reg); \
1203 #define __gen6_read(x) __gen_read(gen6, x)
1204 #define __fwtable_read(x) __gen_read(fwtable, x)
1205 #define __gen11_fwtable_read(x) __gen_read(gen11_fwtable, x)
1207 __gen11_fwtable_read(8)
1208 __gen11_fwtable_read(16)
1209 __gen11_fwtable_read(32)
1210 __gen11_fwtable_read(64)
1220 #undef __gen11_fwtable_read
1221 #undef __fwtable_read
1223 #undef GEN6_READ_FOOTER
1224 #undef GEN6_READ_HEADER
1226 #define GEN2_WRITE_HEADER \
1227 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1228 assert_rpm_wakelock_held(dev_priv); \
1230 #define GEN2_WRITE_FOOTER
1232 #define __gen2_write(x) \
1234 gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1235 GEN2_WRITE_HEADER; \
1236 __raw_i915_write##x(dev_priv, reg, val); \
1237 GEN2_WRITE_FOOTER; \
1240 #define __gen5_write(x) \
1242 gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1243 GEN2_WRITE_HEADER; \
1244 ilk_dummy_write(dev_priv); \
1245 __raw_i915_write##x(dev_priv, reg, val); \
1246 GEN2_WRITE_FOOTER; \
1259 #undef GEN2_WRITE_FOOTER
1260 #undef GEN2_WRITE_HEADER
1262 #define GEN6_WRITE_HEADER \
1263 u32 offset = i915_mmio_reg_offset(reg); \
1264 unsigned long irqflags; \
1265 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1266 assert_rpm_wakelock_held(dev_priv); \
1267 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1268 unclaimed_reg_debug(dev_priv, reg, false, true)
1270 #define GEN6_WRITE_FOOTER \
1271 unclaimed_reg_debug(dev_priv, reg, false, false); \
1272 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
1274 #define __gen6_write(x) \
1276 gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1277 GEN6_WRITE_HEADER; \
1278 if (NEEDS_FORCE_WAKE(offset)) \
1279 __gen6_gt_wait_for_fifo(dev_priv); \
1280 __raw_i915_write##x(dev_priv, reg, val); \
1281 GEN6_WRITE_FOOTER; \
1284 #define __gen_write(func, x) \
1286 func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
1287 enum forcewake_domains fw_engine; \
1288 GEN6_WRITE_HEADER; \
1289 fw_engine = __##func##_reg_write_fw_domains(offset); \
1291 __force_wake_auto(dev_priv, fw_engine); \
1292 __raw_i915_write##x(dev_priv, reg, val); \
1293 GEN6_WRITE_FOOTER; \
1295 #define __gen8_write(x) __gen_write(gen8, x)
1296 #define __fwtable_write(x) __gen_write(fwtable, x)
1297 #define __gen11_fwtable_write(x) __gen_write(gen11_fwtable, x)
1299 __gen11_fwtable_write(8)
1300 __gen11_fwtable_write(16)
1301 __gen11_fwtable_write(32)
1312 #undef __gen11_fwtable_write
1313 #undef __fwtable_write
1316 #undef GEN6_WRITE_FOOTER
1317 #undef GEN6_WRITE_HEADER
1319 #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
1321 (i915)->uncore.funcs.mmio_writeb = x##_write8; \
1322 (i915)->uncore.funcs.mmio_writew = x##_write16; \
1323 (i915)->uncore.funcs.mmio_writel = x##_write32; \
1326 #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
1328 (i915)->uncore.funcs.mmio_readb = x##_read8; \
1329 (i915)->uncore.funcs.mmio_readw = x##_read16; \
1330 (i915)->uncore.funcs.mmio_readl = x##_read32; \
1331 (i915)->uncore.funcs.mmio_readq = x##_read64; \
1335 static void fw_domain_init(struct drm_i915_private
*dev_priv
,
1336 enum forcewake_domain_id domain_id
,
1340 struct intel_uncore_forcewake_domain
*d
;
1342 if (WARN_ON(domain_id
>= FW_DOMAIN_ID_COUNT
))
1345 d
= &dev_priv
->uncore
.fw_domain
[domain_id
];
1347 WARN_ON(d
->wake_count
);
1349 WARN_ON(!i915_mmio_reg_valid(reg_set
));
1350 WARN_ON(!i915_mmio_reg_valid(reg_ack
));
1353 d
->reg_set
= reg_set
;
1354 d
->reg_ack
= reg_ack
;
1358 BUILD_BUG_ON(FORCEWAKE_RENDER
!= (1 << FW_DOMAIN_ID_RENDER
));
1359 BUILD_BUG_ON(FORCEWAKE_BLITTER
!= (1 << FW_DOMAIN_ID_BLITTER
));
1360 BUILD_BUG_ON(FORCEWAKE_MEDIA
!= (1 << FW_DOMAIN_ID_MEDIA
));
1361 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX0
!= (1 << FW_DOMAIN_ID_MEDIA_VDBOX0
));
1362 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX1
!= (1 << FW_DOMAIN_ID_MEDIA_VDBOX1
));
1363 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX2
!= (1 << FW_DOMAIN_ID_MEDIA_VDBOX2
));
1364 BUILD_BUG_ON(FORCEWAKE_MEDIA_VDBOX3
!= (1 << FW_DOMAIN_ID_MEDIA_VDBOX3
));
1365 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX0
!= (1 << FW_DOMAIN_ID_MEDIA_VEBOX0
));
1366 BUILD_BUG_ON(FORCEWAKE_MEDIA_VEBOX1
!= (1 << FW_DOMAIN_ID_MEDIA_VEBOX1
));
1369 d
->mask
= BIT(domain_id
);
1371 hrtimer_init(&d
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
1372 d
->timer
.function
= intel_uncore_fw_release_timer
;
1374 dev_priv
->uncore
.fw_domains
|= BIT(domain_id
);
1376 fw_domain_reset(dev_priv
, d
);
1379 static void fw_domain_fini(struct drm_i915_private
*dev_priv
,
1380 enum forcewake_domain_id domain_id
)
1382 struct intel_uncore_forcewake_domain
*d
;
1384 if (WARN_ON(domain_id
>= FW_DOMAIN_ID_COUNT
))
1387 d
= &dev_priv
->uncore
.fw_domain
[domain_id
];
1389 WARN_ON(d
->wake_count
);
1390 WARN_ON(hrtimer_cancel(&d
->timer
));
1391 memset(d
, 0, sizeof(*d
));
1393 dev_priv
->uncore
.fw_domains
&= ~BIT(domain_id
);
1396 static void intel_uncore_fw_domains_init(struct drm_i915_private
*dev_priv
)
1398 if (INTEL_GEN(dev_priv
) <= 5 || intel_vgpu_active(dev_priv
))
1401 if (IS_GEN6(dev_priv
)) {
1402 dev_priv
->uncore
.fw_reset
= 0;
1403 dev_priv
->uncore
.fw_set
= FORCEWAKE_KERNEL
;
1404 dev_priv
->uncore
.fw_clear
= 0;
1406 /* WaRsClearFWBitsAtReset:bdw,skl */
1407 dev_priv
->uncore
.fw_reset
= _MASKED_BIT_DISABLE(0xffff);
1408 dev_priv
->uncore
.fw_set
= _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL
);
1409 dev_priv
->uncore
.fw_clear
= _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL
);
1412 if (INTEL_GEN(dev_priv
) >= 11) {
1415 dev_priv
->uncore
.funcs
.force_wake_get
=
1416 fw_domains_get_with_fallback
;
1417 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1418 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1419 FORCEWAKE_RENDER_GEN9
,
1420 FORCEWAKE_ACK_RENDER_GEN9
);
1421 fw_domain_init(dev_priv
, FW_DOMAIN_ID_BLITTER
,
1422 FORCEWAKE_BLITTER_GEN9
,
1423 FORCEWAKE_ACK_BLITTER_GEN9
);
1424 for (i
= 0; i
< I915_MAX_VCS
; i
++) {
1425 if (!HAS_ENGINE(dev_priv
, _VCS(i
)))
1428 fw_domain_init(dev_priv
, FW_DOMAIN_ID_MEDIA_VDBOX0
+ i
,
1429 FORCEWAKE_MEDIA_VDBOX_GEN11(i
),
1430 FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(i
));
1432 for (i
= 0; i
< I915_MAX_VECS
; i
++) {
1433 if (!HAS_ENGINE(dev_priv
, _VECS(i
)))
1436 fw_domain_init(dev_priv
, FW_DOMAIN_ID_MEDIA_VEBOX0
+ i
,
1437 FORCEWAKE_MEDIA_VEBOX_GEN11(i
),
1438 FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(i
));
1440 } else if (IS_GEN10(dev_priv
) || IS_GEN9(dev_priv
)) {
1441 dev_priv
->uncore
.funcs
.force_wake_get
=
1442 fw_domains_get_with_fallback
;
1443 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1444 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1445 FORCEWAKE_RENDER_GEN9
,
1446 FORCEWAKE_ACK_RENDER_GEN9
);
1447 fw_domain_init(dev_priv
, FW_DOMAIN_ID_BLITTER
,
1448 FORCEWAKE_BLITTER_GEN9
,
1449 FORCEWAKE_ACK_BLITTER_GEN9
);
1450 fw_domain_init(dev_priv
, FW_DOMAIN_ID_MEDIA
,
1451 FORCEWAKE_MEDIA_GEN9
, FORCEWAKE_ACK_MEDIA_GEN9
);
1452 } else if (IS_VALLEYVIEW(dev_priv
) || IS_CHERRYVIEW(dev_priv
)) {
1453 dev_priv
->uncore
.funcs
.force_wake_get
= fw_domains_get
;
1454 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1455 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1456 FORCEWAKE_VLV
, FORCEWAKE_ACK_VLV
);
1457 fw_domain_init(dev_priv
, FW_DOMAIN_ID_MEDIA
,
1458 FORCEWAKE_MEDIA_VLV
, FORCEWAKE_ACK_MEDIA_VLV
);
1459 } else if (IS_HASWELL(dev_priv
) || IS_BROADWELL(dev_priv
)) {
1460 dev_priv
->uncore
.funcs
.force_wake_get
=
1461 fw_domains_get_with_thread_status
;
1462 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1463 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1464 FORCEWAKE_MT
, FORCEWAKE_ACK_HSW
);
1465 } else if (IS_IVYBRIDGE(dev_priv
)) {
1468 /* IVB configs may use multi-threaded forcewake */
1470 /* A small trick here - if the bios hasn't configured
1471 * MT forcewake, and if the device is in RC6, then
1472 * force_wake_mt_get will not wake the device and the
1473 * ECOBUS read will return zero. Which will be
1474 * (correctly) interpreted by the test below as MT
1475 * forcewake being disabled.
1477 dev_priv
->uncore
.funcs
.force_wake_get
=
1478 fw_domains_get_with_thread_status
;
1479 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1481 /* We need to init first for ECOBUS access and then
1482 * determine later if we want to reinit, in case of MT access is
1483 * not working. In this stage we don't know which flavour this
1484 * ivb is, so it is better to reset also the gen6 fw registers
1485 * before the ecobus check.
1488 __raw_i915_write32(dev_priv
, FORCEWAKE
, 0);
1489 __raw_posting_read(dev_priv
, ECOBUS
);
1491 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1492 FORCEWAKE_MT
, FORCEWAKE_MT_ACK
);
1494 spin_lock_irq(&dev_priv
->uncore
.lock
);
1495 fw_domains_get_with_thread_status(dev_priv
, FORCEWAKE_RENDER
);
1496 ecobus
= __raw_i915_read32(dev_priv
, ECOBUS
);
1497 fw_domains_put(dev_priv
, FORCEWAKE_RENDER
);
1498 spin_unlock_irq(&dev_priv
->uncore
.lock
);
1500 if (!(ecobus
& FORCEWAKE_MT_ENABLE
)) {
1501 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
1502 DRM_INFO("when using vblank-synced partial screen updates.\n");
1503 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1504 FORCEWAKE
, FORCEWAKE_ACK
);
1506 } else if (IS_GEN6(dev_priv
)) {
1507 dev_priv
->uncore
.funcs
.force_wake_get
=
1508 fw_domains_get_with_thread_status
;
1509 dev_priv
->uncore
.funcs
.force_wake_put
= fw_domains_put
;
1510 fw_domain_init(dev_priv
, FW_DOMAIN_ID_RENDER
,
1511 FORCEWAKE
, FORCEWAKE_ACK
);
1514 /* All future platforms are expected to require complex power gating */
1515 WARN_ON(dev_priv
->uncore
.fw_domains
== 0);
1518 #define ASSIGN_FW_DOMAINS_TABLE(d) \
1520 dev_priv->uncore.fw_domains_table = \
1521 (struct intel_forcewake_range *)(d); \
1522 dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
1525 static int i915_pmic_bus_access_notifier(struct notifier_block
*nb
,
1526 unsigned long action
, void *data
)
1528 struct drm_i915_private
*dev_priv
= container_of(nb
,
1529 struct drm_i915_private
, uncore
.pmic_bus_access_nb
);
1532 case MBI_PMIC_BUS_ACCESS_BEGIN
:
1534 * forcewake all now to make sure that we don't need to do a
1535 * forcewake later which on systems where this notifier gets
1536 * called requires the punit to access to the shared pmic i2c
1537 * bus, which will be busy after this notification, leading to:
1538 * "render: timed out waiting for forcewake ack request."
1541 * The notifier is unregistered during intel_runtime_suspend(),
1542 * so it's ok to access the HW here without holding a RPM
1543 * wake reference -> disable wakeref asserts for the time of
1546 disable_rpm_wakeref_asserts(dev_priv
);
1547 intel_uncore_forcewake_get(dev_priv
, FORCEWAKE_ALL
);
1548 enable_rpm_wakeref_asserts(dev_priv
);
1550 case MBI_PMIC_BUS_ACCESS_END
:
1551 intel_uncore_forcewake_put(dev_priv
, FORCEWAKE_ALL
);
1558 void intel_uncore_init(struct drm_i915_private
*dev_priv
)
1560 i915_check_vgpu(dev_priv
);
1562 intel_uncore_edram_detect(dev_priv
);
1563 intel_uncore_fw_domains_init(dev_priv
);
1564 __intel_uncore_early_sanitize(dev_priv
, 0);
1566 dev_priv
->uncore
.unclaimed_mmio_check
= 1;
1567 dev_priv
->uncore
.pmic_bus_access_nb
.notifier_call
=
1568 i915_pmic_bus_access_notifier
;
1570 if (IS_GEN(dev_priv
, 2, 4) || intel_vgpu_active(dev_priv
)) {
1571 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, gen2
);
1572 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, gen2
);
1573 } else if (IS_GEN5(dev_priv
)) {
1574 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, gen5
);
1575 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, gen5
);
1576 } else if (IS_GEN(dev_priv
, 6, 7)) {
1577 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, gen6
);
1579 if (IS_VALLEYVIEW(dev_priv
)) {
1580 ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges
);
1581 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, fwtable
);
1583 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, gen6
);
1585 } else if (IS_GEN8(dev_priv
)) {
1586 if (IS_CHERRYVIEW(dev_priv
)) {
1587 ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges
);
1588 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, fwtable
);
1589 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, fwtable
);
1592 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, gen8
);
1593 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, gen6
);
1595 } else if (IS_GEN(dev_priv
, 9, 10)) {
1596 ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges
);
1597 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, fwtable
);
1598 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, fwtable
);
1600 ASSIGN_FW_DOMAINS_TABLE(__gen11_fw_ranges
);
1601 ASSIGN_WRITE_MMIO_VFUNCS(dev_priv
, gen11_fwtable
);
1602 ASSIGN_READ_MMIO_VFUNCS(dev_priv
, gen11_fwtable
);
1605 iosf_mbi_register_pmic_bus_access_notifier(
1606 &dev_priv
->uncore
.pmic_bus_access_nb
);
1610 * We might have detected that some engines are fused off after we initialized
1611 * the forcewake domains. Prune them, to make sure they only reference existing
1614 void intel_uncore_prune(struct drm_i915_private
*dev_priv
)
1616 if (INTEL_GEN(dev_priv
) >= 11) {
1617 enum forcewake_domains fw_domains
= dev_priv
->uncore
.fw_domains
;
1618 enum forcewake_domain_id domain_id
;
1621 for (i
= 0; i
< I915_MAX_VCS
; i
++) {
1622 domain_id
= FW_DOMAIN_ID_MEDIA_VDBOX0
+ i
;
1624 if (HAS_ENGINE(dev_priv
, _VCS(i
)))
1627 if (fw_domains
& BIT(domain_id
))
1628 fw_domain_fini(dev_priv
, domain_id
);
1631 for (i
= 0; i
< I915_MAX_VECS
; i
++) {
1632 domain_id
= FW_DOMAIN_ID_MEDIA_VEBOX0
+ i
;
1634 if (HAS_ENGINE(dev_priv
, _VECS(i
)))
1637 if (fw_domains
& BIT(domain_id
))
1638 fw_domain_fini(dev_priv
, domain_id
);
1643 void intel_uncore_fini(struct drm_i915_private
*dev_priv
)
1645 /* Paranoia: make sure we have disabled everything before we exit. */
1646 intel_uncore_sanitize(dev_priv
);
1648 iosf_mbi_punit_acquire();
1649 iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
1650 &dev_priv
->uncore
.pmic_bus_access_nb
);
1651 intel_uncore_forcewake_reset(dev_priv
);
1652 iosf_mbi_punit_release();
1655 static const struct reg_whitelist
{
1656 i915_reg_t offset_ldw
;
1657 i915_reg_t offset_udw
;
1660 } reg_read_whitelist
[] = { {
1661 .offset_ldw
= RING_TIMESTAMP(RENDER_RING_BASE
),
1662 .offset_udw
= RING_TIMESTAMP_UDW(RENDER_RING_BASE
),
1663 .gen_mask
= INTEL_GEN_MASK(4, 11),
1667 int i915_reg_read_ioctl(struct drm_device
*dev
,
1668 void *data
, struct drm_file
*file
)
1670 struct drm_i915_private
*dev_priv
= to_i915(dev
);
1671 struct drm_i915_reg_read
*reg
= data
;
1672 struct reg_whitelist
const *entry
;
1677 entry
= reg_read_whitelist
;
1678 remain
= ARRAY_SIZE(reg_read_whitelist
);
1680 u32 entry_offset
= i915_mmio_reg_offset(entry
->offset_ldw
);
1682 GEM_BUG_ON(!is_power_of_2(entry
->size
));
1683 GEM_BUG_ON(entry
->size
> 8);
1684 GEM_BUG_ON(entry_offset
& (entry
->size
- 1));
1686 if (INTEL_INFO(dev_priv
)->gen_mask
& entry
->gen_mask
&&
1687 entry_offset
== (reg
->offset
& -entry
->size
))
1696 flags
= reg
->offset
& (entry
->size
- 1);
1698 intel_runtime_pm_get(dev_priv
);
1699 if (entry
->size
== 8 && flags
== I915_REG_READ_8B_WA
)
1700 reg
->val
= I915_READ64_2x32(entry
->offset_ldw
,
1702 else if (entry
->size
== 8 && flags
== 0)
1703 reg
->val
= I915_READ64(entry
->offset_ldw
);
1704 else if (entry
->size
== 4 && flags
== 0)
1705 reg
->val
= I915_READ(entry
->offset_ldw
);
1706 else if (entry
->size
== 2 && flags
== 0)
1707 reg
->val
= I915_READ16(entry
->offset_ldw
);
1708 else if (entry
->size
== 1 && flags
== 0)
1709 reg
->val
= I915_READ8(entry
->offset_ldw
);
1712 intel_runtime_pm_put(dev_priv
);
1717 static void gen3_stop_engine(struct intel_engine_cs
*engine
)
1719 struct drm_i915_private
*dev_priv
= engine
->i915
;
1720 const u32 base
= engine
->mmio_base
;
1722 if (intel_engine_stop_cs(engine
))
1723 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n", engine
->name
);
1725 I915_WRITE_FW(RING_HEAD(base
), I915_READ_FW(RING_TAIL(base
)));
1726 POSTING_READ_FW(RING_HEAD(base
)); /* paranoia */
1728 I915_WRITE_FW(RING_HEAD(base
), 0);
1729 I915_WRITE_FW(RING_TAIL(base
), 0);
1730 POSTING_READ_FW(RING_TAIL(base
));
1732 /* The ring must be empty before it is disabled */
1733 I915_WRITE_FW(RING_CTL(base
), 0);
1735 /* Check acts as a post */
1736 if (I915_READ_FW(RING_HEAD(base
)) != 0)
1737 DRM_DEBUG_DRIVER("%s: ring head not parked\n",
1741 static void i915_stop_engines(struct drm_i915_private
*dev_priv
,
1742 unsigned int engine_mask
)
1744 struct intel_engine_cs
*engine
;
1745 enum intel_engine_id id
;
1747 if (INTEL_GEN(dev_priv
) < 3)
1750 for_each_engine_masked(engine
, dev_priv
, engine_mask
, id
)
1751 gen3_stop_engine(engine
);
1754 static bool i915_in_reset(struct pci_dev
*pdev
)
1758 pci_read_config_byte(pdev
, I915_GDRST
, &gdrst
);
1759 return gdrst
& GRDOM_RESET_STATUS
;
1762 static int i915_do_reset(struct drm_i915_private
*dev_priv
,
1763 unsigned int engine_mask
,
1766 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
1769 /* Assert reset for at least 20 usec, and wait for acknowledgement. */
1770 pci_write_config_byte(pdev
, I915_GDRST
, GRDOM_RESET_ENABLE
);
1771 usleep_range(50, 200);
1772 err
= wait_for(i915_in_reset(pdev
), 500);
1774 /* Clear the reset request. */
1775 pci_write_config_byte(pdev
, I915_GDRST
, 0);
1776 usleep_range(50, 200);
1778 err
= wait_for(!i915_in_reset(pdev
), 500);
1783 static bool g4x_reset_complete(struct pci_dev
*pdev
)
1787 pci_read_config_byte(pdev
, I915_GDRST
, &gdrst
);
1788 return (gdrst
& GRDOM_RESET_ENABLE
) == 0;
1791 static int g33_do_reset(struct drm_i915_private
*dev_priv
,
1792 unsigned int engine_mask
,
1795 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
1797 pci_write_config_byte(pdev
, I915_GDRST
, GRDOM_RESET_ENABLE
);
1798 return wait_for(g4x_reset_complete(pdev
), 500);
1801 static int g4x_do_reset(struct drm_i915_private
*dev_priv
,
1802 unsigned int engine_mask
,
1805 struct pci_dev
*pdev
= dev_priv
->drm
.pdev
;
1808 /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1809 I915_WRITE(VDECCLK_GATE_D
,
1810 I915_READ(VDECCLK_GATE_D
) | VCP_UNIT_CLOCK_GATE_DISABLE
);
1811 POSTING_READ(VDECCLK_GATE_D
);
1813 pci_write_config_byte(pdev
, I915_GDRST
,
1814 GRDOM_MEDIA
| GRDOM_RESET_ENABLE
);
1815 ret
= wait_for(g4x_reset_complete(pdev
), 500);
1817 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1821 pci_write_config_byte(pdev
, I915_GDRST
,
1822 GRDOM_RENDER
| GRDOM_RESET_ENABLE
);
1823 ret
= wait_for(g4x_reset_complete(pdev
), 500);
1825 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1830 pci_write_config_byte(pdev
, I915_GDRST
, 0);
1832 I915_WRITE(VDECCLK_GATE_D
,
1833 I915_READ(VDECCLK_GATE_D
) & ~VCP_UNIT_CLOCK_GATE_DISABLE
);
1834 POSTING_READ(VDECCLK_GATE_D
);
1839 static int ironlake_do_reset(struct drm_i915_private
*dev_priv
,
1840 unsigned int engine_mask
,
1845 I915_WRITE(ILK_GDSR
, ILK_GRDOM_RENDER
| ILK_GRDOM_RESET_ENABLE
);
1846 ret
= intel_wait_for_register(dev_priv
,
1847 ILK_GDSR
, ILK_GRDOM_RESET_ENABLE
, 0,
1850 DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1854 I915_WRITE(ILK_GDSR
, ILK_GRDOM_MEDIA
| ILK_GRDOM_RESET_ENABLE
);
1855 ret
= intel_wait_for_register(dev_priv
,
1856 ILK_GDSR
, ILK_GRDOM_RESET_ENABLE
, 0,
1859 DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1864 I915_WRITE(ILK_GDSR
, 0);
1865 POSTING_READ(ILK_GDSR
);
1869 /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
1870 static int gen6_hw_domain_reset(struct drm_i915_private
*dev_priv
,
1875 /* GEN6_GDRST is not in the gt power well, no need to check
1876 * for fifo space for the write or forcewake the chip for
1879 __raw_i915_write32(dev_priv
, GEN6_GDRST
, hw_domain_mask
);
1881 /* Wait for the device to ack the reset requests */
1882 err
= __intel_wait_for_register_fw(dev_priv
,
1883 GEN6_GDRST
, hw_domain_mask
, 0,
1887 DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
1894 * gen6_reset_engines - reset individual engines
1895 * @dev_priv: i915 device
1896 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
1897 * @retry: the count of of previous attempts to reset.
1899 * This function will reset the individual engines that are set in engine_mask.
1900 * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
1902 * Note: It is responsibility of the caller to handle the difference between
1903 * asking full domain reset versus reset for all available individual engines.
1905 * Returns 0 on success, nonzero on error.
1907 static int gen6_reset_engines(struct drm_i915_private
*dev_priv
,
1908 unsigned int engine_mask
,
1911 struct intel_engine_cs
*engine
;
1912 const u32 hw_engine_mask
[I915_NUM_ENGINES
] = {
1913 [RCS
] = GEN6_GRDOM_RENDER
,
1914 [BCS
] = GEN6_GRDOM_BLT
,
1915 [VCS
] = GEN6_GRDOM_MEDIA
,
1916 [VCS2
] = GEN8_GRDOM_MEDIA2
,
1917 [VECS
] = GEN6_GRDOM_VECS
,
1921 if (engine_mask
== ALL_ENGINES
) {
1922 hw_mask
= GEN6_GRDOM_FULL
;
1927 for_each_engine_masked(engine
, dev_priv
, engine_mask
, tmp
)
1928 hw_mask
|= hw_engine_mask
[engine
->id
];
1931 return gen6_hw_domain_reset(dev_priv
, hw_mask
);
1935 * gen11_reset_engines - reset individual engines
1936 * @dev_priv: i915 device
1937 * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
1939 * This function will reset the individual engines that are set in engine_mask.
1940 * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
1942 * Note: It is responsibility of the caller to handle the difference between
1943 * asking full domain reset versus reset for all available individual engines.
1945 * Returns 0 on success, nonzero on error.
1947 static int gen11_reset_engines(struct drm_i915_private
*dev_priv
,
1948 unsigned int engine_mask
)
1950 struct intel_engine_cs
*engine
;
1951 const u32 hw_engine_mask
[I915_NUM_ENGINES
] = {
1952 [RCS
] = GEN11_GRDOM_RENDER
,
1953 [BCS
] = GEN11_GRDOM_BLT
,
1954 [VCS
] = GEN11_GRDOM_MEDIA
,
1955 [VCS2
] = GEN11_GRDOM_MEDIA2
,
1956 [VCS3
] = GEN11_GRDOM_MEDIA3
,
1957 [VCS4
] = GEN11_GRDOM_MEDIA4
,
1958 [VECS
] = GEN11_GRDOM_VECS
,
1959 [VECS2
] = GEN11_GRDOM_VECS2
,
1963 BUILD_BUG_ON(VECS2
+ 1 != I915_NUM_ENGINES
);
1965 if (engine_mask
== ALL_ENGINES
) {
1966 hw_mask
= GEN11_GRDOM_FULL
;
1971 for_each_engine_masked(engine
, dev_priv
, engine_mask
, tmp
)
1972 hw_mask
|= hw_engine_mask
[engine
->id
];
1975 return gen6_hw_domain_reset(dev_priv
, hw_mask
);
1979 * __intel_wait_for_register_fw - wait until register matches expected state
1980 * @dev_priv: the i915 device
1981 * @reg: the register to read
1982 * @mask: mask to apply to register value
1983 * @value: expected value
1984 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1985 * @slow_timeout_ms: slow timeout in millisecond
1986 * @out_value: optional placeholder to hold registry value
1988 * This routine waits until the target register @reg contains the expected
1989 * @value after applying the @mask, i.e. it waits until ::
1991 * (I915_READ_FW(reg) & mask) == value
1993 * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
1994 * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
1995 * must be not larger than 20,0000 microseconds.
1997 * Note that this routine assumes the caller holds forcewake asserted, it is
1998 * not suitable for very long waits. See intel_wait_for_register() if you
1999 * wish to wait without holding forcewake for the duration (i.e. you expect
2000 * the wait to be slow).
2002 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
2004 int __intel_wait_for_register_fw(struct drm_i915_private
*dev_priv
,
2008 unsigned int fast_timeout_us
,
2009 unsigned int slow_timeout_ms
,
2012 u32
uninitialized_var(reg_value
);
2013 #define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
2016 /* Catch any overuse of this function */
2017 might_sleep_if(slow_timeout_ms
);
2018 GEM_BUG_ON(fast_timeout_us
> 20000);
2021 if (fast_timeout_us
&& fast_timeout_us
<= 20000)
2022 ret
= _wait_for_atomic(done
, fast_timeout_us
, 0);
2023 if (ret
&& slow_timeout_ms
)
2024 ret
= wait_for(done
, slow_timeout_ms
);
2027 *out_value
= reg_value
;
2034 * __intel_wait_for_register - wait until register matches expected state
2035 * @dev_priv: the i915 device
2036 * @reg: the register to read
2037 * @mask: mask to apply to register value
2038 * @value: expected value
2039 * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
2040 * @slow_timeout_ms: slow timeout in millisecond
2041 * @out_value: optional placeholder to hold registry value
2043 * This routine waits until the target register @reg contains the expected
2044 * @value after applying the @mask, i.e. it waits until ::
2046 * (I915_READ(reg) & mask) == value
2048 * Otherwise, the wait will timeout after @timeout_ms milliseconds.
2050 * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
2052 int __intel_wait_for_register(struct drm_i915_private
*dev_priv
,
2056 unsigned int fast_timeout_us
,
2057 unsigned int slow_timeout_ms
,
2061 intel_uncore_forcewake_for_reg(dev_priv
, reg
, FW_REG_READ
);
2065 might_sleep_if(slow_timeout_ms
);
2067 spin_lock_irq(&dev_priv
->uncore
.lock
);
2068 intel_uncore_forcewake_get__locked(dev_priv
, fw
);
2070 ret
= __intel_wait_for_register_fw(dev_priv
,
2072 fast_timeout_us
, 0, ®_value
);
2074 intel_uncore_forcewake_put__locked(dev_priv
, fw
);
2075 spin_unlock_irq(&dev_priv
->uncore
.lock
);
2077 if (ret
&& slow_timeout_ms
)
2078 ret
= __wait_for(reg_value
= I915_READ_NOTRACE(reg
),
2079 (reg_value
& mask
) == value
,
2080 slow_timeout_ms
* 1000, 10, 1000);
2083 *out_value
= reg_value
;
2088 static int gen8_engine_reset_prepare(struct intel_engine_cs
*engine
)
2090 struct drm_i915_private
*dev_priv
= engine
->i915
;
2093 I915_WRITE_FW(RING_RESET_CTL(engine
->mmio_base
),
2094 _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET
));
2096 ret
= __intel_wait_for_register_fw(dev_priv
,
2097 RING_RESET_CTL(engine
->mmio_base
),
2098 RESET_CTL_READY_TO_RESET
,
2099 RESET_CTL_READY_TO_RESET
,
2103 DRM_ERROR("%s: reset request timeout\n", engine
->name
);
2108 static void gen8_engine_reset_cancel(struct intel_engine_cs
*engine
)
2110 struct drm_i915_private
*dev_priv
= engine
->i915
;
2112 I915_WRITE_FW(RING_RESET_CTL(engine
->mmio_base
),
2113 _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET
));
2116 static int reset_engines(struct drm_i915_private
*i915
,
2117 unsigned int engine_mask
,
2120 if (INTEL_GEN(i915
) >= 11)
2121 return gen11_reset_engines(i915
, engine_mask
);
2123 return gen6_reset_engines(i915
, engine_mask
, retry
);
2126 static int gen8_reset_engines(struct drm_i915_private
*dev_priv
,
2127 unsigned int engine_mask
,
2130 struct intel_engine_cs
*engine
;
2131 const bool reset_non_ready
= retry
>= 1;
2135 for_each_engine_masked(engine
, dev_priv
, engine_mask
, tmp
) {
2136 ret
= gen8_engine_reset_prepare(engine
);
2137 if (ret
&& !reset_non_ready
)
2141 * If this is not the first failed attempt to prepare,
2142 * we decide to proceed anyway.
2144 * By doing so we risk context corruption and with
2145 * some gens (kbl), possible system hang if reset
2146 * happens during active bb execution.
2148 * We rather take context corruption instead of
2149 * failed reset with a wedged driver/gpu. And
2150 * active bb execution case should be covered by
2151 * i915_stop_engines we have before the reset.
2155 ret
= reset_engines(dev_priv
, engine_mask
, retry
);
2158 for_each_engine_masked(engine
, dev_priv
, engine_mask
, tmp
)
2159 gen8_engine_reset_cancel(engine
);
2164 typedef int (*reset_func
)(struct drm_i915_private
*,
2165 unsigned int engine_mask
, unsigned int retry
);
2167 static reset_func
intel_get_gpu_reset(struct drm_i915_private
*dev_priv
)
2169 if (!i915_modparams
.reset
)
2172 if (INTEL_GEN(dev_priv
) >= 8)
2173 return gen8_reset_engines
;
2174 else if (INTEL_GEN(dev_priv
) >= 6)
2175 return gen6_reset_engines
;
2176 else if (IS_GEN5(dev_priv
))
2177 return ironlake_do_reset
;
2178 else if (IS_G4X(dev_priv
))
2179 return g4x_do_reset
;
2180 else if (IS_G33(dev_priv
) || IS_PINEVIEW(dev_priv
))
2181 return g33_do_reset
;
2182 else if (INTEL_GEN(dev_priv
) >= 3)
2183 return i915_do_reset
;
2188 int intel_gpu_reset(struct drm_i915_private
*dev_priv
,
2189 const unsigned int engine_mask
)
2191 reset_func reset
= intel_get_gpu_reset(dev_priv
);
2195 GEM_BUG_ON(!engine_mask
);
2198 * We want to perform per-engine reset from atomic context (e.g.
2199 * softirq), which imposes the constraint that we cannot sleep.
2200 * However, experience suggests that spending a bit of time waiting
2201 * for a reset helps in various cases, so for a full-device reset
2202 * we apply the opposite rule and wait if we want to. As we should
2203 * always follow up a failed per-engine reset with a full device reset,
2204 * being a little faster, stricter and more error prone for the
2205 * atomic case seems an acceptable compromise.
2207 * Unfortunately this leads to a bimodal routine, when the goal was
2208 * to have a single reset function that worked for resetting any
2209 * number of engines simultaneously.
2211 might_sleep_if(engine_mask
== ALL_ENGINES
);
2214 * If the power well sleeps during the reset, the reset
2215 * request may be dropped and never completes (causing -EIO).
2217 intel_uncore_forcewake_get(dev_priv
, FORCEWAKE_ALL
);
2218 for (retry
= 0; retry
< 3; retry
++) {
2221 * We stop engines, otherwise we might get failed reset and a
2222 * dead gpu (on elk). Also as modern gpu as kbl can suffer
2223 * from system hang if batchbuffer is progressing when
2224 * the reset is issued, regardless of READY_TO_RESET ack.
2225 * Thus assume it is best to stop engines on all gens
2226 * where we have a gpu reset.
2228 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES)
2230 * WaMediaResetMainRingCleanup:ctg,elk (presumably)
2232 * FIXME: Wa for more modern gens needs to be validated
2234 i915_stop_engines(dev_priv
, engine_mask
);
2238 ret
= reset(dev_priv
, engine_mask
, retry
);
2239 GEM_TRACE("engine_mask=%x, ret=%d, retry=%d\n",
2240 engine_mask
, ret
, retry
);
2242 if (ret
!= -ETIMEDOUT
|| engine_mask
!= ALL_ENGINES
)
2247 intel_uncore_forcewake_put(dev_priv
, FORCEWAKE_ALL
);
2252 bool intel_has_gpu_reset(struct drm_i915_private
*dev_priv
)
2254 return intel_get_gpu_reset(dev_priv
) != NULL
;
2257 bool intel_has_reset_engine(struct drm_i915_private
*dev_priv
)
2259 return (dev_priv
->info
.has_reset_engine
&&
2260 i915_modparams
.reset
>= 2);
2263 int intel_reset_guc(struct drm_i915_private
*dev_priv
)
2265 u32 guc_domain
= INTEL_GEN(dev_priv
) >= 11 ? GEN11_GRDOM_GUC
:
2269 GEM_BUG_ON(!HAS_GUC(dev_priv
));
2271 intel_uncore_forcewake_get(dev_priv
, FORCEWAKE_ALL
);
2272 ret
= gen6_hw_domain_reset(dev_priv
, guc_domain
);
2273 intel_uncore_forcewake_put(dev_priv
, FORCEWAKE_ALL
);
2278 bool intel_uncore_unclaimed_mmio(struct drm_i915_private
*dev_priv
)
2280 return check_for_unclaimed_mmio(dev_priv
);
2284 intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private
*dev_priv
)
2288 spin_lock_irq(&dev_priv
->uncore
.lock
);
2290 if (unlikely(dev_priv
->uncore
.unclaimed_mmio_check
<= 0))
2293 if (unlikely(intel_uncore_unclaimed_mmio(dev_priv
))) {
2294 if (!i915_modparams
.mmio_debug
) {
2295 DRM_DEBUG("Unclaimed register detected, "
2296 "enabling oneshot unclaimed register reporting. "
2297 "Please use i915.mmio_debug=N for more information.\n");
2298 i915_modparams
.mmio_debug
++;
2300 dev_priv
->uncore
.unclaimed_mmio_check
--;
2305 spin_unlock_irq(&dev_priv
->uncore
.lock
);
2310 static enum forcewake_domains
2311 intel_uncore_forcewake_for_read(struct drm_i915_private
*dev_priv
,
2314 u32 offset
= i915_mmio_reg_offset(reg
);
2315 enum forcewake_domains fw_domains
;
2317 if (INTEL_GEN(dev_priv
) >= 11) {
2318 fw_domains
= __gen11_fwtable_reg_read_fw_domains(offset
);
2319 } else if (HAS_FWTABLE(dev_priv
)) {
2320 fw_domains
= __fwtable_reg_read_fw_domains(offset
);
2321 } else if (INTEL_GEN(dev_priv
) >= 6) {
2322 fw_domains
= __gen6_reg_read_fw_domains(offset
);
2324 WARN_ON(!IS_GEN(dev_priv
, 2, 5));
2328 WARN_ON(fw_domains
& ~dev_priv
->uncore
.fw_domains
);
2333 static enum forcewake_domains
2334 intel_uncore_forcewake_for_write(struct drm_i915_private
*dev_priv
,
2337 u32 offset
= i915_mmio_reg_offset(reg
);
2338 enum forcewake_domains fw_domains
;
2340 if (INTEL_GEN(dev_priv
) >= 11) {
2341 fw_domains
= __gen11_fwtable_reg_write_fw_domains(offset
);
2342 } else if (HAS_FWTABLE(dev_priv
) && !IS_VALLEYVIEW(dev_priv
)) {
2343 fw_domains
= __fwtable_reg_write_fw_domains(offset
);
2344 } else if (IS_GEN8(dev_priv
)) {
2345 fw_domains
= __gen8_reg_write_fw_domains(offset
);
2346 } else if (IS_GEN(dev_priv
, 6, 7)) {
2347 fw_domains
= FORCEWAKE_RENDER
;
2349 WARN_ON(!IS_GEN(dev_priv
, 2, 5));
2353 WARN_ON(fw_domains
& ~dev_priv
->uncore
.fw_domains
);
2359 * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
2361 * @dev_priv: pointer to struct drm_i915_private
2362 * @reg: register in question
2363 * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
2365 * Returns a set of forcewake domains required to be taken with for example
2366 * intel_uncore_forcewake_get for the specified register to be accessible in the
2367 * specified mode (read, write or read/write) with raw mmio accessors.
2369 * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
2370 * callers to do FIFO management on their own or risk losing writes.
2372 enum forcewake_domains
2373 intel_uncore_forcewake_for_reg(struct drm_i915_private
*dev_priv
,
2374 i915_reg_t reg
, unsigned int op
)
2376 enum forcewake_domains fw_domains
= 0;
2380 if (intel_vgpu_active(dev_priv
))
2383 if (op
& FW_REG_READ
)
2384 fw_domains
= intel_uncore_forcewake_for_read(dev_priv
, reg
);
2386 if (op
& FW_REG_WRITE
)
2387 fw_domains
|= intel_uncore_forcewake_for_write(dev_priv
, reg
);
2392 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2393 #include "selftests/mock_uncore.c"
2394 #include "selftests/intel_uncore.c"