1 /* SPDX-License-Identifier: MIT */
3 * Copyright © 2014-2019 Intel Corporation
9 #include <linux/delay.h>
10 #include <linux/iosys-map.h>
11 #include <linux/xarray.h>
13 #include "intel_guc_ct.h"
14 #include "intel_guc_fw.h"
15 #include "intel_guc_fwif.h"
16 #include "intel_guc_log.h"
17 #include "intel_guc_reg.h"
18 #include "intel_guc_slpc_types.h"
19 #include "intel_uc_fw.h"
20 #include "intel_uncore.h"
21 #include "i915_utils.h"
24 struct __guc_ads_blob
;
25 struct intel_guc_state_capture
;
28 * struct intel_guc - Top level structure of GuC.
30 * It handles firmware loading and manages client pool. intel_guc owns an
31 * i915_sched_engine for submission.
34 /** @fw: the GuC firmware */
35 struct intel_uc_fw fw
;
36 /** @log: sub-structure containing GuC log related data and objects */
37 struct intel_guc_log log
;
38 /** @ct: the command transport communication channel */
39 struct intel_guc_ct ct
;
40 /** @slpc: sub-structure containing SLPC related data and objects */
41 struct intel_guc_slpc slpc
;
42 /** @capture: the error-state-capture module's data and objects */
43 struct intel_guc_state_capture
*capture
;
45 /** @dbgfs_node: debugfs node */
46 struct dentry
*dbgfs_node
;
48 /** @sched_engine: Global engine used to submit requests to GuC */
49 struct i915_sched_engine
*sched_engine
;
51 * @stalled_request: if GuC can't process a request for any reason, we
52 * save it until GuC restarts processing. No other request can be
53 * submitted until the stalled request is processed.
55 struct i915_request
*stalled_request
;
57 * @submission_stall_reason: reason why submission is stalled
61 STALL_REGISTER_CONTEXT
,
64 } submission_stall_reason
;
66 /* intel_guc_recv interrupt related state */
67 /** @irq_lock: protects GuC irq state */
70 * @msg_enabled_mask: mask of events that are processed when receiving
71 * an INTEL_GUC_ACTION_DEFAULT G2H message.
73 unsigned int msg_enabled_mask
;
76 * @outstanding_submission_g2h: number of outstanding GuC to Host
77 * responses related to GuC submission, used to determine if the GT is
80 atomic_t outstanding_submission_g2h
;
82 /** @tlb_lookup: xarray to store all pending TLB invalidation requests */
83 struct xarray tlb_lookup
;
86 * @serial_slot: id to the initial waiter created in tlb_lookup,
87 * which is used only when failed to allocate new waiter.
91 /** @next_seqno: the next id (sequence number) to allocate. */
94 /** @interrupts: pointers to GuC interrupt-managing functions. */
97 void (*reset
)(struct intel_guc
*guc
);
98 void (*enable
)(struct intel_guc
*guc
);
99 void (*disable
)(struct intel_guc
*guc
);
103 * @submission_state: sub-structure for submission state protected by
108 * @submission_state.lock: protects everything in
109 * submission_state, ce->guc_id.id, and ce->guc_id.ref
110 * when transitioning in and out of zero
114 * @submission_state.guc_ids: used to allocate new
115 * guc_ids, single-lrc
119 * @submission_state.num_guc_ids: Number of guc_ids, selftest
120 * feature to be able to reduce this number while testing.
124 * @submission_state.guc_ids_bitmap: used to allocate
125 * new guc_ids, multi-lrc
127 unsigned long *guc_ids_bitmap
;
129 * @submission_state.guc_id_list: list of intel_context
130 * with valid guc_ids but no refs
132 struct list_head guc_id_list
;
134 * @submission_state.guc_ids_in_use: Number single-lrc
137 unsigned int guc_ids_in_use
;
139 * @submission_state.destroyed_contexts: list of contexts
140 * waiting to be destroyed (deregistered with the GuC)
142 struct list_head destroyed_contexts
;
144 * @submission_state.destroyed_worker: worker to deregister
145 * contexts, need as we need to take a GT PM reference and
146 * can't from destroy function as it might be in an atomic
147 * context (no sleeping)
149 struct work_struct destroyed_worker
;
151 * @submission_state.reset_fail_worker: worker to trigger
152 * a GT reset after an engine reset fails
154 struct work_struct reset_fail_worker
;
156 * @submission_state.reset_fail_mask: mask of engines that
159 intel_engine_mask_t reset_fail_mask
;
161 * @submission_state.sched_disable_delay_ms: schedule
162 * disable delay, in ms, for contexts
164 unsigned int sched_disable_delay_ms
;
166 * @submission_state.sched_disable_gucid_threshold:
167 * threshold of min remaining available guc_ids before
168 * we start bypassing the schedule disable delay
170 unsigned int sched_disable_gucid_threshold
;
174 * @submission_supported: tracks whether we support GuC submission on
175 * the current platform
177 bool submission_supported
;
178 /** @submission_selected: tracks whether the user enabled GuC submission */
179 bool submission_selected
;
180 /** @submission_initialized: tracks whether GuC submission has been initialised */
181 bool submission_initialized
;
182 /** @submission_version: Submission API version of the currently loaded firmware */
183 struct intel_uc_fw_ver submission_version
;
186 * @rc_supported: tracks whether we support GuC rc on the current platform
189 /** @rc_selected: tracks whether the user enabled GuC rc */
192 /** @ads_vma: object allocated to hold the GuC ADS */
193 struct i915_vma
*ads_vma
;
194 /** @ads_map: contents of the GuC ADS */
195 struct iosys_map ads_map
;
196 /** @ads_regset_size: size of the save/restore regsets in the ADS */
199 * @ads_regset_count: number of save/restore registers in the ADS for
202 u32 ads_regset_count
[I915_NUM_ENGINES
];
203 /** @ads_regset: save/restore regsets in the ADS */
204 struct guc_mmio_reg
*ads_regset
;
205 /** @ads_golden_ctxt_size: size of the golden contexts in the ADS */
206 u32 ads_golden_ctxt_size
;
207 /** @ads_capture_size: size of register lists in the ADS used for error capture */
208 u32 ads_capture_size
;
210 /** @lrc_desc_pool_v69: object allocated to hold the GuC LRC descriptor pool */
211 struct i915_vma
*lrc_desc_pool_v69
;
212 /** @lrc_desc_pool_vaddr_v69: contents of the GuC LRC descriptor pool */
213 void *lrc_desc_pool_vaddr_v69
;
216 * @context_lookup: used to resolve intel_context from guc_id, if a
217 * context is present in this structure it is registered with the GuC
219 struct xarray context_lookup
;
221 /** @params: Control params for fw initialization */
222 u32 params
[GUC_CTL_MAX_DWORDS
];
224 /** @send_regs: GuC's FW specific registers used for sending MMIO H2G */
228 enum forcewake_domains fw_domains
;
231 /** @notify_reg: register used to send interrupts to the GuC FW */
232 i915_reg_t notify_reg
;
235 * @mmio_msg: notification bitmask that the GuC writes in one of its
236 * registers when the CT channel is disabled, to be processed when the
237 * channel is back up.
241 /** @send_mutex: used to serialize the intel_guc_send actions */
242 struct mutex send_mutex
;
245 * @timestamp: GT timestamp object that stores a copy of the timestamp
246 * and adjusts it for overflow using a worker.
250 * @timestamp.lock: Lock protecting the below fields and
256 * @timestamp.gt_stamp: 64-bit extended value of the GT
262 * @timestamp.ping_delay: Period for polling the GT
263 * timestamp for overflow.
265 unsigned long ping_delay
;
268 * @timestamp.work: Periodic work to adjust GT timestamp,
269 * engine and context usage for overflows.
271 struct delayed_work work
;
274 * @timestamp.shift: Right shift value for the gpm timestamp
279 * @timestamp.last_stat_jiffies: jiffies at last actual
280 * stats collection time. We use this timestamp to ensure
281 * we don't oversample the stats because runtime power
282 * management events can trigger stats collection at much
283 * higher rates than required.
285 unsigned long last_stat_jiffies
;
289 * @dead_guc_worker: Asynchronous worker thread for forcing a GuC reset.
290 * Specifically used when the G2H handler wants to issue a reset. Resets
291 * require flushing the G2H queue. So, the G2H processing itself must not
292 * trigger a reset directly. Instead, go via this worker.
294 struct work_struct dead_guc_worker
;
296 * @last_dead_guc_jiffies: timestamp of previous 'dead guc' occurrance
297 * used to prevent a fundamentally broken system from continuously
300 unsigned long last_dead_guc_jiffies
;
302 #ifdef CONFIG_DRM_I915_SELFTEST
304 * @number_guc_id_stolen: The number of guc_ids that have been stolen
306 int number_guc_id_stolen
;
308 * @fast_response_selftest: Backdoor to CT handler for fast response selftest
310 u32 fast_response_selftest
;
314 struct intel_guc_tlb_wait
{
315 struct wait_queue_head wq
;
320 * GuC version number components are only 8-bit, so converting to a 32bit 8.8.8
323 #define MAKE_GUC_VER(maj, min, pat) (((maj) << 16) | ((min) << 8) | (pat))
324 #define MAKE_GUC_VER_STRUCT(ver) MAKE_GUC_VER((ver).major, (ver).minor, (ver).patch)
325 #define GUC_SUBMIT_VER(guc) MAKE_GUC_VER_STRUCT((guc)->submission_version)
326 #define GUC_FIRMWARE_VER(guc) MAKE_GUC_VER_STRUCT((guc)->fw.file_selected.ver)
328 static inline struct intel_guc
*log_to_guc(struct intel_guc_log
*log
)
330 return container_of(log
, struct intel_guc
, log
);
334 inline int intel_guc_send(struct intel_guc
*guc
, const u32
*action
, u32 len
)
336 return intel_guc_ct_send(&guc
->ct
, action
, len
, NULL
, 0, 0);
340 inline int intel_guc_send_nb(struct intel_guc
*guc
, const u32
*action
, u32 len
,
343 return intel_guc_ct_send(&guc
->ct
, action
, len
, NULL
, 0,
344 MAKE_SEND_FLAGS(g2h_len_dw
));
348 intel_guc_send_and_receive(struct intel_guc
*guc
, const u32
*action
, u32 len
,
349 u32
*response_buf
, u32 response_buf_size
)
351 return intel_guc_ct_send(&guc
->ct
, action
, len
,
352 response_buf
, response_buf_size
, 0);
355 static inline int intel_guc_send_busy_loop(struct intel_guc
*guc
,
362 unsigned int sleep_period_ms
= 1;
363 bool not_atomic
= !in_atomic() && !irqs_disabled();
366 * FIXME: Have caller pass in if we are in an atomic context to avoid
367 * using in_atomic(). It is likely safe here as we check for irqs
368 * disabled which basically all the spin locks in the i915 do but
369 * regardless this should be cleaned up.
372 /* No sleeping with spin locks, just busy loop */
373 might_sleep_if(loop
&& not_atomic
);
376 err
= intel_guc_send_nb(guc
, action
, len
, g2h_len_dw
);
377 if (unlikely(err
== -EBUSY
&& loop
)) {
378 if (likely(not_atomic
)) {
379 if (msleep_interruptible(sleep_period_ms
))
381 sleep_period_ms
= sleep_period_ms
<< 1;
391 /* Only call this from the interrupt handler code */
392 static inline void intel_guc_to_host_event_handler(struct intel_guc
*guc
)
394 if (guc
->interrupts
.enabled
)
395 intel_guc_ct_event_handler(&guc
->ct
);
398 /* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */
399 #define GUC_GGTT_TOP 0xFEE00000
402 * intel_guc_ggtt_offset() - Get and validate the GGTT offset of @vma
403 * @guc: intel_guc structure.
404 * @vma: i915 graphics virtual memory area.
406 * GuC does not allow any gfx GGTT address that falls into range
407 * [0, ggtt.pin_bias), which is reserved for Boot ROM, SRAM and WOPCM.
408 * Currently, in order to exclude [0, ggtt.pin_bias) address space from
409 * GGTT, all gfx objects used by GuC are allocated with intel_guc_allocate_vma()
410 * and pinned with PIN_OFFSET_BIAS along with the value of ggtt.pin_bias.
412 * Return: GGTT offset of the @vma.
414 static inline u32
intel_guc_ggtt_offset(struct intel_guc
*guc
,
415 struct i915_vma
*vma
)
417 u32 offset
= i915_ggtt_offset(vma
);
419 GEM_BUG_ON(offset
< i915_ggtt_pin_bias(vma
));
420 GEM_BUG_ON(range_overflows_t(u64
, offset
, vma
->size
, GUC_GGTT_TOP
));
425 void intel_guc_init_early(struct intel_guc
*guc
);
426 void intel_guc_init_late(struct intel_guc
*guc
);
427 void intel_guc_init_send_regs(struct intel_guc
*guc
);
428 void intel_guc_write_params(struct intel_guc
*guc
);
429 int intel_guc_init(struct intel_guc
*guc
);
430 void intel_guc_fini(struct intel_guc
*guc
);
431 void intel_guc_notify(struct intel_guc
*guc
);
432 int intel_guc_send_mmio(struct intel_guc
*guc
, const u32
*action
, u32 len
,
433 u32
*response_buf
, u32 response_buf_size
);
434 int intel_guc_to_host_process_recv_msg(struct intel_guc
*guc
,
435 const u32
*payload
, u32 len
);
436 int intel_guc_auth_huc(struct intel_guc
*guc
, u32 rsa_offset
);
437 int intel_guc_suspend(struct intel_guc
*guc
);
438 int intel_guc_resume(struct intel_guc
*guc
);
439 struct i915_vma
*intel_guc_allocate_vma(struct intel_guc
*guc
, u32 size
);
440 int intel_guc_allocate_and_map_vma(struct intel_guc
*guc
, u32 size
,
441 struct i915_vma
**out_vma
, void **out_vaddr
);
442 int intel_guc_self_cfg32(struct intel_guc
*guc
, u16 key
, u32 value
);
443 int intel_guc_self_cfg64(struct intel_guc
*guc
, u16 key
, u64 value
);
445 static inline bool intel_guc_is_supported(struct intel_guc
*guc
)
447 return intel_uc_fw_is_supported(&guc
->fw
);
450 static inline bool intel_guc_is_wanted(struct intel_guc
*guc
)
452 return intel_uc_fw_is_enabled(&guc
->fw
);
455 static inline bool intel_guc_is_used(struct intel_guc
*guc
)
457 GEM_BUG_ON(__intel_uc_fw_status(&guc
->fw
) == INTEL_UC_FIRMWARE_SELECTED
);
458 return intel_uc_fw_is_available(&guc
->fw
);
461 static inline bool intel_guc_is_fw_running(struct intel_guc
*guc
)
463 return intel_uc_fw_is_running(&guc
->fw
);
466 static inline bool intel_guc_is_ready(struct intel_guc
*guc
)
468 return intel_guc_is_fw_running(guc
) && intel_guc_ct_enabled(&guc
->ct
);
471 static inline void intel_guc_reset_interrupts(struct intel_guc
*guc
)
473 guc
->interrupts
.reset(guc
);
476 static inline void intel_guc_enable_interrupts(struct intel_guc
*guc
)
478 guc
->interrupts
.enable(guc
);
481 static inline void intel_guc_disable_interrupts(struct intel_guc
*guc
)
483 guc
->interrupts
.disable(guc
);
486 static inline int intel_guc_sanitize(struct intel_guc
*guc
)
488 intel_uc_fw_sanitize(&guc
->fw
);
489 intel_guc_disable_interrupts(guc
);
490 intel_guc_ct_sanitize(&guc
->ct
);
496 static inline void intel_guc_enable_msg(struct intel_guc
*guc
, u32 mask
)
498 spin_lock_irq(&guc
->irq_lock
);
499 guc
->msg_enabled_mask
|= mask
;
500 spin_unlock_irq(&guc
->irq_lock
);
503 static inline void intel_guc_disable_msg(struct intel_guc
*guc
, u32 mask
)
505 spin_lock_irq(&guc
->irq_lock
);
506 guc
->msg_enabled_mask
&= ~mask
;
507 spin_unlock_irq(&guc
->irq_lock
);
510 int intel_guc_wait_for_idle(struct intel_guc
*guc
, long timeout
);
512 int intel_guc_deregister_done_process_msg(struct intel_guc
*guc
,
513 const u32
*msg
, u32 len
);
514 int intel_guc_sched_done_process_msg(struct intel_guc
*guc
,
515 const u32
*msg
, u32 len
);
516 int intel_guc_context_reset_process_msg(struct intel_guc
*guc
,
517 const u32
*msg
, u32 len
);
518 int intel_guc_engine_failure_process_msg(struct intel_guc
*guc
,
519 const u32
*msg
, u32 len
);
520 int intel_guc_error_capture_process_msg(struct intel_guc
*guc
,
521 const u32
*msg
, u32 len
);
522 int intel_guc_crash_process_msg(struct intel_guc
*guc
, u32 action
);
524 struct intel_engine_cs
*
525 intel_guc_lookup_engine(struct intel_guc
*guc
, u8 guc_class
, u8 instance
);
527 void intel_guc_find_hung_context(struct intel_engine_cs
*engine
);
529 int intel_guc_global_policies_update(struct intel_guc
*guc
);
531 void intel_guc_context_ban(struct intel_context
*ce
, struct i915_request
*rq
);
533 void intel_guc_submission_reset_prepare(struct intel_guc
*guc
);
534 void intel_guc_submission_reset(struct intel_guc
*guc
, intel_engine_mask_t stalled
);
535 void intel_guc_submission_reset_finish(struct intel_guc
*guc
);
536 void intel_guc_submission_cancel_requests(struct intel_guc
*guc
);
538 void intel_guc_load_status(struct intel_guc
*guc
, struct drm_printer
*p
);
540 void intel_guc_write_barrier(struct intel_guc
*guc
);
542 void intel_guc_dump_time_info(struct intel_guc
*guc
, struct drm_printer
*p
);
544 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc
*guc
);
546 bool intel_guc_tlb_invalidation_is_available(struct intel_guc
*guc
);
547 int intel_guc_invalidate_tlb_engines(struct intel_guc
*guc
);
548 int intel_guc_invalidate_tlb_guc(struct intel_guc
*guc
);
549 int intel_guc_tlb_invalidation_done(struct intel_guc
*guc
,
550 const u32
*payload
, u32 len
);
551 void wake_up_all_tlb_invalidate(struct intel_guc
*guc
);