1 // SPDX-License-Identifier: MIT
3 * Copyright © 2014 Intel Corporation
6 #include <linux/circ_buf.h>
8 #include "gem/i915_gem_context.h"
9 #include "gt/intel_context.h"
10 #include "gt/intel_engine_pm.h"
11 #include "gt/intel_gt.h"
12 #include "gt/intel_gt_pm.h"
13 #include "gt/intel_lrc_reg.h"
14 #include "gt/intel_ring.h"
16 #include "intel_guc_submission.h"
19 #include "i915_trace.h"
22 * DOC: GuC-based command submission
24 * IMPORTANT NOTE: GuC submission is currently not supported in i915. The GuC
25 * firmware is moving to an updated submission interface and we plan to
26 * turn submission back on when that lands. The below documentation (and related
27 * code) matches the old submission model and will be updated as part of the
28 * upgrade to the new flow.
30 * GuC stage descriptor:
31 * During initialization, the driver allocates a static pool of 1024 such
32 * descriptors, and shares them with the GuC. Currently, we only use one
33 * descriptor. This stage descriptor lets the GuC know about the workqueue and
34 * process descriptor. Theoretically, it also lets the GuC know about our HW
35 * contexts (context ID, etc...), but we actually employ a kind of submission
36 * where the GuC uses the LRCA sent via the work item instead. This is called
37 * a "proxy" submission.
39 * The Scratch registers:
40 * There are 16 MMIO-based registers start from 0xC180. The kernel driver writes
41 * a value to the action register (SOFT_SCRATCH_0) along with any data. It then
42 * triggers an interrupt on the GuC via another register write (0xC4C8).
43 * Firmware writes a success/fail code back to the action register after
44 * processes the request. The kernel driver polls waiting for this update and
48 * There are several types of work items that the host may place into a
49 * workqueue, each with its own requirements and limitations. Currently only
50 * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
51 * represents in-order queue. The kernel driver packs ring tail pointer and an
52 * ELSP context descriptor dword into Work Item.
53 * See guc_add_request()
57 static inline struct i915_priolist
*to_priolist(struct rb_node
*rb
)
59 return rb_entry(rb
, struct i915_priolist
, node
);
62 static struct guc_stage_desc
*__get_stage_desc(struct intel_guc
*guc
, u32 id
)
64 struct guc_stage_desc
*base
= guc
->stage_desc_pool_vaddr
;
69 static int guc_workqueue_create(struct intel_guc
*guc
)
71 return intel_guc_allocate_and_map_vma(guc
, GUC_WQ_SIZE
, &guc
->workqueue
,
72 &guc
->workqueue_vaddr
);
75 static void guc_workqueue_destroy(struct intel_guc
*guc
)
77 i915_vma_unpin_and_release(&guc
->workqueue
, I915_VMA_RELEASE_MAP
);
81 * Initialise the process descriptor shared with the GuC firmware.
83 static int guc_proc_desc_create(struct intel_guc
*guc
)
85 const u32 size
= PAGE_ALIGN(sizeof(struct guc_process_desc
));
87 return intel_guc_allocate_and_map_vma(guc
, size
, &guc
->proc_desc
,
88 &guc
->proc_desc_vaddr
);
91 static void guc_proc_desc_destroy(struct intel_guc
*guc
)
93 i915_vma_unpin_and_release(&guc
->proc_desc
, I915_VMA_RELEASE_MAP
);
96 static void guc_proc_desc_init(struct intel_guc
*guc
)
98 struct guc_process_desc
*desc
;
100 desc
= memset(guc
->proc_desc_vaddr
, 0, sizeof(*desc
));
103 * XXX: pDoorbell and WQVBaseAddress are pointers in process address
104 * space for ring3 clients (set them as in mmap_ioctl) or kernel
105 * space for kernel clients (map on demand instead? May make debug
106 * easier to have it mapped).
108 desc
->wq_base_addr
= 0;
109 desc
->db_base_addr
= 0;
111 desc
->wq_size_bytes
= GUC_WQ_SIZE
;
112 desc
->wq_status
= WQ_STATUS_ACTIVE
;
113 desc
->priority
= GUC_CLIENT_PRIORITY_KMD_NORMAL
;
116 static void guc_proc_desc_fini(struct intel_guc
*guc
)
118 memset(guc
->proc_desc_vaddr
, 0, sizeof(struct guc_process_desc
));
121 static int guc_stage_desc_pool_create(struct intel_guc
*guc
)
123 u32 size
= PAGE_ALIGN(sizeof(struct guc_stage_desc
) *
124 GUC_MAX_STAGE_DESCRIPTORS
);
126 return intel_guc_allocate_and_map_vma(guc
, size
, &guc
->stage_desc_pool
,
127 &guc
->stage_desc_pool_vaddr
);
130 static void guc_stage_desc_pool_destroy(struct intel_guc
*guc
)
132 i915_vma_unpin_and_release(&guc
->stage_desc_pool
, I915_VMA_RELEASE_MAP
);
136 * Initialise/clear the stage descriptor shared with the GuC firmware.
138 * This descriptor tells the GuC where (in GGTT space) to find the important
139 * data structures related to work submission (process descriptor, write queue,
142 static void guc_stage_desc_init(struct intel_guc
*guc
)
144 struct guc_stage_desc
*desc
;
146 /* we only use 1 stage desc, so hardcode it to 0 */
147 desc
= __get_stage_desc(guc
, 0);
148 memset(desc
, 0, sizeof(*desc
));
150 desc
->attribute
= GUC_STAGE_DESC_ATTR_ACTIVE
|
151 GUC_STAGE_DESC_ATTR_KERNEL
;
154 desc
->priority
= GUC_CLIENT_PRIORITY_KMD_NORMAL
;
156 desc
->process_desc
= intel_guc_ggtt_offset(guc
, guc
->proc_desc
);
157 desc
->wq_addr
= intel_guc_ggtt_offset(guc
, guc
->workqueue
);
158 desc
->wq_size
= GUC_WQ_SIZE
;
161 static void guc_stage_desc_fini(struct intel_guc
*guc
)
163 struct guc_stage_desc
*desc
;
165 desc
= __get_stage_desc(guc
, 0);
166 memset(desc
, 0, sizeof(*desc
));
169 /* Construct a Work Item and append it to the GuC's Work Queue */
170 static void guc_wq_item_append(struct intel_guc
*guc
,
171 u32 target_engine
, u32 context_desc
,
172 u32 ring_tail
, u32 fence_id
)
174 /* wqi_len is in DWords, and does not include the one-word header */
175 const size_t wqi_size
= sizeof(struct guc_wq_item
);
176 const u32 wqi_len
= wqi_size
/ sizeof(u32
) - 1;
177 struct guc_process_desc
*desc
= guc
->proc_desc_vaddr
;
178 struct guc_wq_item
*wqi
;
181 lockdep_assert_held(&guc
->wq_lock
);
183 /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
184 * should not have the case where structure wqi is across page, neither
185 * wrapped to the beginning. This simplifies the implementation below.
187 * XXX: if not the case, we need save data to a temp wqi and copy it to
188 * workqueue buffer dw by dw.
190 BUILD_BUG_ON(wqi_size
!= 16);
192 /* We expect the WQ to be active if we're appending items to it */
193 GEM_BUG_ON(desc
->wq_status
!= WQ_STATUS_ACTIVE
);
195 /* Free space is guaranteed. */
196 wq_off
= READ_ONCE(desc
->tail
);
197 GEM_BUG_ON(CIRC_SPACE(wq_off
, READ_ONCE(desc
->head
),
198 GUC_WQ_SIZE
) < wqi_size
);
199 GEM_BUG_ON(wq_off
& (wqi_size
- 1));
201 wqi
= guc
->workqueue_vaddr
+ wq_off
;
203 /* Now fill in the 4-word work queue item */
204 wqi
->header
= WQ_TYPE_INORDER
|
205 (wqi_len
<< WQ_LEN_SHIFT
) |
206 (target_engine
<< WQ_TARGET_SHIFT
) |
208 wqi
->context_desc
= context_desc
;
209 wqi
->submit_element_info
= ring_tail
<< WQ_RING_TAIL_SHIFT
;
210 GEM_BUG_ON(ring_tail
> WQ_RING_TAIL_MAX
);
211 wqi
->fence_id
= fence_id
;
213 /* Make the update visible to GuC */
214 WRITE_ONCE(desc
->tail
, (wq_off
+ wqi_size
) & (GUC_WQ_SIZE
- 1));
217 static void guc_add_request(struct intel_guc
*guc
, struct i915_request
*rq
)
219 struct intel_engine_cs
*engine
= rq
->engine
;
220 u32 ctx_desc
= lower_32_bits(rq
->context
->lrc_desc
);
221 u32 ring_tail
= intel_ring_set_tail(rq
->ring
, rq
->tail
) / sizeof(u64
);
223 guc_wq_item_append(guc
, engine
->guc_id
, ctx_desc
,
224 ring_tail
, rq
->fence
.seqno
);
228 * When we're doing submissions using regular execlists backend, writing to
229 * ELSP from CPU side is enough to make sure that writes to ringbuffer pages
230 * pinned in mappable aperture portion of GGTT are visible to command streamer.
231 * Writes done by GuC on our behalf are not guaranteeing such ordering,
232 * therefore, to ensure the flush, we're issuing a POSTING READ.
234 static void flush_ggtt_writes(struct i915_vma
*vma
)
236 if (i915_vma_is_map_and_fenceable(vma
))
237 intel_uncore_posting_read_fw(vma
->vm
->gt
->uncore
,
241 static void guc_submit(struct intel_engine_cs
*engine
,
242 struct i915_request
**out
,
243 struct i915_request
**end
)
245 struct intel_guc
*guc
= &engine
->gt
->uc
.guc
;
247 spin_lock(&guc
->wq_lock
);
250 struct i915_request
*rq
= *out
++;
252 flush_ggtt_writes(rq
->ring
->vma
);
253 guc_add_request(guc
, rq
);
254 } while (out
!= end
);
256 spin_unlock(&guc
->wq_lock
);
259 static inline int rq_prio(const struct i915_request
*rq
)
261 return rq
->sched
.attr
.priority
| __NO_PREEMPTION
;
264 static struct i915_request
*schedule_in(struct i915_request
*rq
, int idx
)
266 trace_i915_request_in(rq
, idx
);
269 * Currently we are not tracking the rq->context being inflight
270 * (ce->inflight = rq->engine). It is only used by the execlists
271 * backend at the moment, a similar counting strategy would be
272 * required if we generalise the inflight tracking.
275 __intel_gt_pm_get(rq
->engine
->gt
);
276 return i915_request_get(rq
);
279 static void schedule_out(struct i915_request
*rq
)
281 trace_i915_request_out(rq
);
283 intel_gt_pm_put_async(rq
->engine
->gt
);
284 i915_request_put(rq
);
287 static void __guc_dequeue(struct intel_engine_cs
*engine
)
289 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
290 struct i915_request
**first
= execlists
->inflight
;
291 struct i915_request
** const last_port
= first
+ execlists
->port_mask
;
292 struct i915_request
*last
= first
[0];
293 struct i915_request
**port
;
297 lockdep_assert_held(&engine
->active
.lock
);
307 * We write directly into the execlists->inflight queue and don't use
308 * the execlists->pending queue, as we don't have a distinct switch
312 while ((rb
= rb_first_cached(&execlists
->queue
))) {
313 struct i915_priolist
*p
= to_priolist(rb
);
314 struct i915_request
*rq
, *rn
;
317 priolist_for_each_request_consume(rq
, rn
, p
, i
) {
318 if (last
&& rq
->context
!= last
->context
) {
319 if (port
== last_port
)
322 *port
= schedule_in(last
,
323 port
- execlists
->inflight
);
327 list_del_init(&rq
->sched
.link
);
328 __i915_request_submit(rq
);
333 rb_erase_cached(&p
->node
, &execlists
->queue
);
334 i915_priolist_free(p
);
337 execlists
->queue_priority_hint
=
338 rb
? to_priolist(rb
)->priority
: INT_MIN
;
340 *port
= schedule_in(last
, port
- execlists
->inflight
);
342 guc_submit(engine
, first
, port
);
344 execlists
->active
= execlists
->inflight
;
347 static void guc_submission_tasklet(unsigned long data
)
349 struct intel_engine_cs
* const engine
= (struct intel_engine_cs
*)data
;
350 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
351 struct i915_request
**port
, *rq
;
354 spin_lock_irqsave(&engine
->active
.lock
, flags
);
356 for (port
= execlists
->inflight
; (rq
= *port
); port
++) {
357 if (!i915_request_completed(rq
))
362 if (port
!= execlists
->inflight
) {
363 int idx
= port
- execlists
->inflight
;
364 int rem
= ARRAY_SIZE(execlists
->inflight
) - idx
;
365 memmove(execlists
->inflight
, port
, rem
* sizeof(*port
));
368 __guc_dequeue(engine
);
370 spin_unlock_irqrestore(&engine
->active
.lock
, flags
);
373 static void guc_reset_prepare(struct intel_engine_cs
*engine
)
375 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
377 ENGINE_TRACE(engine
, "\n");
380 * Prevent request submission to the hardware until we have
381 * completed the reset in i915_gem_reset_finish(). If a request
382 * is completed by one engine, it may then queue a request
383 * to a second via its execlists->tasklet *just* as we are
384 * calling engine->init_hw() and also writing the ELSP.
385 * Turning off the execlists->tasklet until the reset is over
388 __tasklet_disable_sync_once(&execlists
->tasklet
);
392 cancel_port_requests(struct intel_engine_execlists
* const execlists
)
394 struct i915_request
* const *port
, *rq
;
396 /* Note we are only using the inflight and not the pending queue */
398 for (port
= execlists
->active
; (rq
= *port
); port
++)
401 memset(execlists
->inflight
, 0, sizeof(execlists
->inflight
));
404 static void guc_reset_rewind(struct intel_engine_cs
*engine
, bool stalled
)
406 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
407 struct i915_request
*rq
;
410 spin_lock_irqsave(&engine
->active
.lock
, flags
);
412 cancel_port_requests(execlists
);
414 /* Push back any incomplete requests for replay after the reset. */
415 rq
= execlists_unwind_incomplete_requests(execlists
);
419 if (!i915_request_started(rq
))
422 __i915_request_reset(rq
, stalled
);
423 intel_lr_context_reset(engine
, rq
->context
, rq
->head
, stalled
);
426 spin_unlock_irqrestore(&engine
->active
.lock
, flags
);
429 static void guc_reset_cancel(struct intel_engine_cs
*engine
)
431 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
432 struct i915_request
*rq
, *rn
;
436 ENGINE_TRACE(engine
, "\n");
439 * Before we call engine->cancel_requests(), we should have exclusive
440 * access to the submission state. This is arranged for us by the
441 * caller disabling the interrupt generation, the tasklet and other
442 * threads that may then access the same state, giving us a free hand
443 * to reset state. However, we still need to let lockdep be aware that
444 * we know this state may be accessed in hardirq context, so we
445 * disable the irq around this manipulation and we want to keep
446 * the spinlock focused on its duties and not accidentally conflate
447 * coverage to the submission's irq state. (Similarly, although we
448 * shouldn't need to disable irq around the manipulation of the
449 * submission's irq state, we also wish to remind ourselves that
452 spin_lock_irqsave(&engine
->active
.lock
, flags
);
454 /* Cancel the requests on the HW and clear the ELSP tracker. */
455 cancel_port_requests(execlists
);
457 /* Mark all executing requests as skipped. */
458 list_for_each_entry(rq
, &engine
->active
.requests
, sched
.link
) {
459 i915_request_set_error_once(rq
, -EIO
);
460 i915_request_mark_complete(rq
);
463 /* Flush the queued requests to the timeline list (for retiring). */
464 while ((rb
= rb_first_cached(&execlists
->queue
))) {
465 struct i915_priolist
*p
= to_priolist(rb
);
468 priolist_for_each_request_consume(rq
, rn
, p
, i
) {
469 list_del_init(&rq
->sched
.link
);
470 __i915_request_submit(rq
);
471 dma_fence_set_error(&rq
->fence
, -EIO
);
472 i915_request_mark_complete(rq
);
475 rb_erase_cached(&p
->node
, &execlists
->queue
);
476 i915_priolist_free(p
);
479 /* Remaining _unready_ requests will be nop'ed when submitted */
481 execlists
->queue_priority_hint
= INT_MIN
;
482 execlists
->queue
= RB_ROOT_CACHED
;
484 spin_unlock_irqrestore(&engine
->active
.lock
, flags
);
487 static void guc_reset_finish(struct intel_engine_cs
*engine
)
489 struct intel_engine_execlists
* const execlists
= &engine
->execlists
;
491 if (__tasklet_enable(&execlists
->tasklet
))
492 /* And kick in case we missed a new request submission. */
493 tasklet_hi_schedule(&execlists
->tasklet
);
495 ENGINE_TRACE(engine
, "depth->%d\n",
496 atomic_read(&execlists
->tasklet
.count
));
500 * Everything below here is concerned with setup & teardown, and is
501 * therefore not part of the somewhat time-critical batch-submission
502 * path of guc_submit() above.
506 * Set up the memory resources to be shared with the GuC (via the GGTT)
507 * at firmware loading time.
509 int intel_guc_submission_init(struct intel_guc
*guc
)
513 if (guc
->stage_desc_pool
)
516 ret
= guc_stage_desc_pool_create(guc
);
520 * Keep static analysers happy, let them know that we allocated the
521 * vma after testing that it didn't exist earlier.
523 GEM_BUG_ON(!guc
->stage_desc_pool
);
525 ret
= guc_workqueue_create(guc
);
529 ret
= guc_proc_desc_create(guc
);
533 spin_lock_init(&guc
->wq_lock
);
538 guc_workqueue_destroy(guc
);
540 guc_stage_desc_pool_destroy(guc
);
544 void intel_guc_submission_fini(struct intel_guc
*guc
)
546 if (guc
->stage_desc_pool
) {
547 guc_proc_desc_destroy(guc
);
548 guc_workqueue_destroy(guc
);
549 guc_stage_desc_pool_destroy(guc
);
553 static void guc_interrupts_capture(struct intel_gt
*gt
)
555 struct intel_uncore
*uncore
= gt
->uncore
;
556 u32 irqs
= GT_CONTEXT_SWITCH_INTERRUPT
;
557 u32 dmask
= irqs
<< 16 | irqs
;
559 GEM_BUG_ON(INTEL_GEN(gt
->i915
) < 11);
561 /* Don't handle the ctx switch interrupt in GuC submission mode */
562 intel_uncore_rmw(uncore
, GEN11_RENDER_COPY_INTR_ENABLE
, dmask
, 0);
563 intel_uncore_rmw(uncore
, GEN11_VCS_VECS_INTR_ENABLE
, dmask
, 0);
566 static void guc_interrupts_release(struct intel_gt
*gt
)
568 struct intel_uncore
*uncore
= gt
->uncore
;
569 u32 irqs
= GT_CONTEXT_SWITCH_INTERRUPT
;
570 u32 dmask
= irqs
<< 16 | irqs
;
572 GEM_BUG_ON(INTEL_GEN(gt
->i915
) < 11);
574 /* Handle ctx switch interrupts again */
575 intel_uncore_rmw(uncore
, GEN11_RENDER_COPY_INTR_ENABLE
, 0, dmask
);
576 intel_uncore_rmw(uncore
, GEN11_VCS_VECS_INTR_ENABLE
, 0, dmask
);
579 static void guc_set_default_submission(struct intel_engine_cs
*engine
)
582 * We inherit a bunch of functions from execlists that we'd like
585 * engine->submit_request = execlists_submit_request;
586 * engine->cancel_requests = execlists_cancel_requests;
587 * engine->schedule = execlists_schedule;
589 * But we need to override the actual submission backend in order
590 * to talk to the GuC.
592 intel_execlists_set_default_submission(engine
);
594 engine
->execlists
.tasklet
.func
= guc_submission_tasklet
;
596 /* do not use execlists park/unpark */
597 engine
->park
= engine
->unpark
= NULL
;
599 engine
->reset
.prepare
= guc_reset_prepare
;
600 engine
->reset
.rewind
= guc_reset_rewind
;
601 engine
->reset
.cancel
= guc_reset_cancel
;
602 engine
->reset
.finish
= guc_reset_finish
;
604 engine
->flags
&= ~I915_ENGINE_SUPPORTS_STATS
;
605 engine
->flags
|= I915_ENGINE_NEEDS_BREADCRUMB_TASKLET
;
608 * For the breadcrumb irq to work we need the interrupts to stay
609 * enabled. However, on all platforms on which we'll have support for
610 * GuC submission we don't allow disabling the interrupts at runtime, so
611 * we're always safe with the current flow.
613 GEM_BUG_ON(engine
->irq_enable
|| engine
->irq_disable
);
616 void intel_guc_submission_enable(struct intel_guc
*guc
)
618 struct intel_gt
*gt
= guc_to_gt(guc
);
619 struct intel_engine_cs
*engine
;
620 enum intel_engine_id id
;
623 * We're using GuC work items for submitting work through GuC. Since
624 * we're coalescing multiple requests from a single context into a
625 * single work item prior to assigning it to execlist_port, we can
626 * never have more work items than the total number of ports (for all
627 * engines). The GuC firmware is controlling the HEAD of work queue,
628 * and it is guaranteed that it will remove the work item from the
629 * queue before our request is completed.
631 BUILD_BUG_ON(ARRAY_SIZE(engine
->execlists
.inflight
) *
632 sizeof(struct guc_wq_item
) *
633 I915_NUM_ENGINES
> GUC_WQ_SIZE
);
635 guc_proc_desc_init(guc
);
636 guc_stage_desc_init(guc
);
638 /* Take over from manual control of ELSP (execlists) */
639 guc_interrupts_capture(gt
);
641 for_each_engine(engine
, gt
, id
) {
642 engine
->set_default_submission
= guc_set_default_submission
;
643 engine
->set_default_submission(engine
);
647 void intel_guc_submission_disable(struct intel_guc
*guc
)
649 struct intel_gt
*gt
= guc_to_gt(guc
);
651 GEM_BUG_ON(gt
->awake
); /* GT should be parked first */
653 /* Note: By the time we're here, GuC may have already been reset */
655 guc_interrupts_release(gt
);
657 guc_stage_desc_fini(guc
);
658 guc_proc_desc_fini(guc
);
661 static bool __guc_submission_selected(struct intel_guc
*guc
)
663 if (!intel_guc_submission_is_supported(guc
))
666 return i915_modparams
.enable_guc
& ENABLE_GUC_SUBMISSION
;
669 void intel_guc_submission_init_early(struct intel_guc
*guc
)
671 guc
->submission_selected
= __guc_submission_selected(guc
);
674 bool intel_engine_in_guc_submission_mode(const struct intel_engine_cs
*engine
)
676 return engine
->set_default_submission
== guc_set_default_submission
;