]> git.ipfire.org Git - thirdparty/linux.git/blame - drivers/gpu/drm/i915/gt/selftest_lrc.c
projects / thirdparty / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'io_uring-5.7-2020-05-22' of git://git.kernel.dk/linux-block
[thirdparty/linux.git] / drivers / gpu / drm / i915 / gt / selftest_lrc.c
CommitLineData
2c66555e
CW		 1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2018 Intel Corporation
5 */
6
a21f453c
CW		 7#include <linux/prime_numbers.h>
8
10be98a7 9#include "gem/i915_gem_pm.h"
d12acee8 10#include "gt/intel_engine_heartbeat.h"
112ed2d3 11#include "gt/intel_reset.h"
10be98a7 12
112ed2d3
CW		 13#include "i915_selftest.h"
14#include "selftests/i915_random.h"
15#include "selftests/igt_flush_test.h"
16#include "selftests/igt_live_test.h"
17#include "selftests/igt_spinner.h"
ee113690 18#include "selftests/lib_sw_fence.h"
10be98a7
CW		 19
20#include "gem/selftests/igt_gem_utils.h"
21#include "gem/selftests/mock_context.h"
2c66555e 22
9506c23d
CW		 23#define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
24#define NUM_GPR_DW (16 * 2) /* each GPR is 2 dwords */
25
86027e31
CW		 26static struct i915_vma *create_scratch(struct intel_gt *gt)
27{
28 struct drm_i915_gem_object *obj;
29 struct i915_vma *vma;
30 int err;
31
32 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
33 if (IS_ERR(obj))
34 return ERR_CAST(obj);
35
36 i915_gem_object_set_cache_coherency(obj, I915_CACHING_CACHED);
37
38 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
39 if (IS_ERR(vma)) {
40 i915_gem_object_put(obj);
41 return vma;
42 }
43
44 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
45 if (err) {
46 i915_gem_object_put(obj);
47 return ERR_PTR(err);
48 }
49
50 return vma;
51}
52
e6ba7648
CW		 53static void engine_heartbeat_disable(struct intel_engine_cs *engine,
54 unsigned long *saved)
55{
56 *saved = engine->props.heartbeat_interval_ms;
57 engine->props.heartbeat_interval_ms = 0;
58
59 intel_engine_pm_get(engine);
60 intel_engine_park_heartbeat(engine);
61}
62
63static void engine_heartbeat_enable(struct intel_engine_cs *engine,
64 unsigned long saved)
65{
66 intel_engine_pm_put(engine);
67
68 engine->props.heartbeat_interval_ms = saved;
69}
70
70a76a9b
CW		 71static int wait_for_submit(struct intel_engine_cs *engine,
72 struct i915_request *rq,
73 unsigned long timeout)
74{
75 timeout += jiffies;
76 do {
77 cond_resched();
78 intel_engine_flush_submission(engine);
fcf7df7a 79
d30d3d5f
CW		 80 if (READ_ONCE(engine->execlists.pending[0]))
81 continue;
82
83 if (i915_request_is_active(rq))
70a76a9b 84 return 0;
fcf7df7a 85
06289949 86 if (i915_request_started(rq)) /* that was quick! */
fcf7df7a 87 return 0;
70a76a9b
CW		 88 } while (time_before(jiffies, timeout));
89
90 return -ETIME;
91}
92
280e285d
CW		 93static int wait_for_reset(struct intel_engine_cs *engine,
94 struct i915_request *rq,
95 unsigned long timeout)
96{
97 timeout += jiffies;
98
99 do {
100 cond_resched();
101 intel_engine_flush_submission(engine);
102
103 if (READ_ONCE(engine->execlists.pending[0]))
104 continue;
105
106 if (i915_request_completed(rq))
107 break;
108
109 if (READ_ONCE(rq->fence.error))
110 break;
111 } while (time_before(jiffies, timeout));
112
113 flush_scheduled_work();
114
115 if (rq->fence.error != -EIO) {
116 pr_err("%s: hanging request %llx:%lld not reset\n",
117 engine->name,
118 rq->fence.context,
119 rq->fence.seqno);
120 return -EINVAL;
121 }
122
123 /* Give the request a jiffie to complete after flushing the worker */
124 if (i915_request_wait(rq, 0,
125 max(0l, (long)(timeout - jiffies)) + 1) < 0) {
126 pr_err("%s: hanging request %llx:%lld did not complete\n",
127 engine->name,
128 rq->fence.context,
129 rq->fence.seqno);
130 return -ETIME;
131 }
132
133 return 0;
134}
135
2c66555e
CW		 136static int live_sanitycheck(void *arg)
137{
1357fa81 138 struct intel_gt *gt = arg;
e6ba7648
CW		 139 struct intel_engine_cs *engine;
140 enum intel_engine_id id;
8d2f6e2f 141 struct igt_spinner spin;
e6ba7648 142 int err = 0;
2c66555e 143
1357fa81 144 if (!HAS_LOGICAL_RING_CONTEXTS(gt->i915))
2c66555e
CW		 145 return 0;
146
1357fa81 147 if (igt_spinner_init(&spin, gt))
2af40298 148 return -ENOMEM;
2c66555e 149
e6ba7648
CW		 150 for_each_engine(engine, gt, id) {
151 struct intel_context *ce;
2c66555e
CW		 152 struct i915_request *rq;
153
e6ba7648
CW		 154 ce = intel_context_create(engine);
155 if (IS_ERR(ce)) {
156 err = PTR_ERR(ce);
157 break;
158 }
159
f277bc0c 160 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
2c66555e
CW		 161 if (IS_ERR(rq)) {
162 err = PTR_ERR(rq);
e6ba7648 163 goto out_ctx;
2c66555e
CW		 164 }
165
166 i915_request_add(rq);
8d2f6e2f 167 if (!igt_wait_for_spinner(&spin, rq)) {
2c66555e
CW		 168 GEM_TRACE("spinner failed to start\n");
169 GEM_TRACE_DUMP();
1357fa81 170 intel_gt_set_wedged(gt);
2c66555e 171 err = -EIO;
e6ba7648 172 goto out_ctx;
2c66555e
CW		 173 }
174
8d2f6e2f 175 igt_spinner_end(&spin);
1357fa81 176 if (igt_flush_test(gt->i915)) {
2c66555e 177 err = -EIO;
e6ba7648 178 goto out_ctx;
2c66555e 179 }
e6ba7648
CW		 180
181out_ctx:
182 intel_context_put(ce);
183 if (err)
184 break;
2c66555e
CW		 185 }
186
8d2f6e2f 187 igt_spinner_fini(&spin);
2c66555e
CW		 188 return err;
189}
190
1357fa81 191static int live_unlite_restore(struct intel_gt *gt, int prio)
fcde8c7e
CW		 192{
193 struct intel_engine_cs *engine;
fcde8c7e 194 enum intel_engine_id id;
fcde8c7e
CW		 195 struct igt_spinner spin;
196 int err = -ENOMEM;
197
198 /*
199 * Check that we can correctly context switch between 2 instances
200 * on the same engine from the same parent context.
201 */
202
1357fa81 203 if (igt_spinner_init(&spin, gt))
85746855 204 return err;
fcde8c7e 205
fcde8c7e 206 err = 0;
5d904e3c 207 for_each_engine(engine, gt, id) {
fcde8c7e
CW		 208 struct intel_context *ce[2] = {};
209 struct i915_request *rq[2];
210 struct igt_live_test t;
e6ba7648 211 unsigned long saved;
fcde8c7e
CW		 212 int n;
213
214 if (prio && !intel_engine_has_preemption(engine))
215 continue;
216
217 if (!intel_engine_can_store_dword(engine))
218 continue;
219
1357fa81 220 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
fcde8c7e
CW		 221 err = -EIO;
222 break;
223 }
e6ba7648 224 engine_heartbeat_disable(engine, &saved);
fcde8c7e
CW		 225
226 for (n = 0; n < ARRAY_SIZE(ce); n++) {
227 struct intel_context *tmp;
228
e6ba7648 229 tmp = intel_context_create(engine);
fcde8c7e
CW		 230 if (IS_ERR(tmp)) {
231 err = PTR_ERR(tmp);
232 goto err_ce;
233 }
234
235 err = intel_context_pin(tmp);
236 if (err) {
237 intel_context_put(tmp);
238 goto err_ce;
239 }
240
241 /*
242 * Setup the pair of contexts such that if we
243 * lite-restore using the RING_TAIL from ce[1] it
244 * will execute garbage from ce[0]->ring.
245 */
246 memset(tmp->ring->vaddr,
247 POISON_INUSE, /* IPEHR: 0x5a5a5a5a [hung!] */
248 tmp->ring->vma->size);
249
250 ce[n] = tmp;
251 }
252 GEM_BUG_ON(!ce[1]->ring->size);
253 intel_ring_reset(ce[1]->ring, ce[1]->ring->size / 2);
42827350 254 __execlists_update_reg_state(ce[1], engine, ce[1]->ring->head);
fcde8c7e
CW		 255
256 rq[0] = igt_spinner_create_request(&spin, ce[0], MI_ARB_CHECK);
257 if (IS_ERR(rq[0])) {
258 err = PTR_ERR(rq[0]);
259 goto err_ce;
260 }
261
262 i915_request_get(rq[0]);
263 i915_request_add(rq[0]);
264 GEM_BUG_ON(rq[0]->postfix > ce[1]->ring->emit);
265
266 if (!igt_wait_for_spinner(&spin, rq[0])) {
267 i915_request_put(rq[0]);
268 goto err_ce;
269 }
270
271 rq[1] = i915_request_create(ce[1]);
272 if (IS_ERR(rq[1])) {
273 err = PTR_ERR(rq[1]);
274 i915_request_put(rq[0]);
275 goto err_ce;
276 }
277
278 if (!prio) {
279 /*
280 * Ensure we do the switch to ce[1] on completion.
281 *
282 * rq[0] is already submitted, so this should reduce
283 * to a no-op (a wait on a request on the same engine
284 * uses the submit fence, not the completion fence),
285 * but it will install a dependency on rq[1] for rq[0]
286 * that will prevent the pair being reordered by
287 * timeslicing.
288 */
289 i915_request_await_dma_fence(rq[1], &rq[0]->fence);
290 }
291
292 i915_request_get(rq[1]);
293 i915_request_add(rq[1]);
294 GEM_BUG_ON(rq[1]->postfix <= rq[0]->postfix);
295 i915_request_put(rq[0]);
296
297 if (prio) {
298 struct i915_sched_attr attr = {
299 .priority = prio,
300 };
301
302 /* Alternatively preempt the spinner with ce[1] */
303 engine->schedule(rq[1], &attr);
304 }
305
306 /* And switch back to ce[0] for good measure */
307 rq[0] = i915_request_create(ce[0]);
308 if (IS_ERR(rq[0])) {
309 err = PTR_ERR(rq[0]);
310 i915_request_put(rq[1]);
311 goto err_ce;
312 }
313
314 i915_request_await_dma_fence(rq[0], &rq[1]->fence);
315 i915_request_get(rq[0]);
316 i915_request_add(rq[0]);
317 GEM_BUG_ON(rq[0]->postfix > rq[1]->postfix);
318 i915_request_put(rq[1]);
319 i915_request_put(rq[0]);
320
321err_ce:
322 tasklet_kill(&engine->execlists.tasklet); /* flush submission */
323 igt_spinner_end(&spin);
324 for (n = 0; n < ARRAY_SIZE(ce); n++) {
325 if (IS_ERR_OR_NULL(ce[n]))
326 break;
327
328 intel_context_unpin(ce[n]);
329 intel_context_put(ce[n]);
330 }
331
e6ba7648 332 engine_heartbeat_enable(engine, saved);
fcde8c7e
CW		 333 if (igt_live_test_end(&t))
334 err = -EIO;
335 if (err)
336 break;
337 }
338
fcde8c7e 339 igt_spinner_fini(&spin);
fcde8c7e
CW		 340 return err;
341}
342
343static int live_unlite_switch(void *arg)
344{
345 return live_unlite_restore(arg, 0);
346}
347
348static int live_unlite_preempt(void *arg)
349{
350 return live_unlite_restore(arg, I915_USER_PRIORITY(I915_PRIORITY_MAX));
351}
352
37305ede
CW		 353static int live_pin_rewind(void *arg)
354{
355 struct intel_gt *gt = arg;
356 struct intel_engine_cs *engine;
357 enum intel_engine_id id;
358 int err = 0;
359
360 /*
361 * We have to be careful not to trust intel_ring too much, for example
362 * ring->head is updated upon retire which is out of sync with pinning
363 * the context. Thus we cannot use ring->head to set CTX_RING_HEAD,
364 * or else we risk writing an older, stale value.
365 *
366 * To simulate this, let's apply a bit of deliberate sabotague.
367 */
368
369 for_each_engine(engine, gt, id) {
370 struct intel_context *ce;
371 struct i915_request *rq;
372 struct intel_ring *ring;
373 struct igt_live_test t;
374
375 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
376 err = -EIO;
377 break;
378 }
379
380 ce = intel_context_create(engine);
381 if (IS_ERR(ce)) {
382 err = PTR_ERR(ce);
383 break;
384 }
385
386 err = intel_context_pin(ce);
387 if (err) {
388 intel_context_put(ce);
389 break;
390 }
391
392 /* Keep the context awake while we play games */
393 err = i915_active_acquire(&ce->active);
394 if (err) {
395 intel_context_unpin(ce);
396 intel_context_put(ce);
397 break;
398 }
399 ring = ce->ring;
400
401 /* Poison the ring, and offset the next request from HEAD */
402 memset32(ring->vaddr, STACK_MAGIC, ring->size / sizeof(u32));
403 ring->emit = ring->size / 2;
404 ring->tail = ring->emit;
405 GEM_BUG_ON(ring->head);
406
407 intel_context_unpin(ce);
408
409 /* Submit a simple nop request */
410 GEM_BUG_ON(intel_context_is_pinned(ce));
411 rq = intel_context_create_request(ce);
412 i915_active_release(&ce->active); /* e.g. async retire */
413 intel_context_put(ce);
414 if (IS_ERR(rq)) {
415 err = PTR_ERR(rq);
416 break;
417 }
418 GEM_BUG_ON(!rq->head);
419 i915_request_add(rq);
420
421 /* Expect not to hang! */
422 if (igt_live_test_end(&t)) {
423 err = -EIO;
424 break;
425 }
426 }
427
428 return err;
429}
430
32ff621f
CW		 431static int live_hold_reset(void *arg)
432{
433 struct intel_gt *gt = arg;
434 struct intel_engine_cs *engine;
435 enum intel_engine_id id;
436 struct igt_spinner spin;
437 int err = 0;
438
439 /*
440 * In order to support offline error capture for fast preempt reset,
441 * we need to decouple the guilty request and ensure that it and its
442 * descendents are not executed while the capture is in progress.
443 */
444
445 if (!intel_has_reset_engine(gt))
446 return 0;
447
448 if (igt_spinner_init(&spin, gt))
449 return -ENOMEM;
450
451 for_each_engine(engine, gt, id) {
452 struct intel_context *ce;
453 unsigned long heartbeat;
454 struct i915_request *rq;
455
456 ce = intel_context_create(engine);
457 if (IS_ERR(ce)) {
458 err = PTR_ERR(ce);
459 break;
460 }
461
462 engine_heartbeat_disable(engine, &heartbeat);
463
464 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
465 if (IS_ERR(rq)) {
466 err = PTR_ERR(rq);
467 goto out;
468 }
469 i915_request_add(rq);
470
471 if (!igt_wait_for_spinner(&spin, rq)) {
472 intel_gt_set_wedged(gt);
473 err = -ETIME;
474 goto out;
475 }
476
477 /* We have our request executing, now remove it and reset */
478
479 if (test_and_set_bit(I915_RESET_ENGINE + id,
480 &gt->reset.flags)) {
32ff621f
CW		 481 intel_gt_set_wedged(gt);
482 err = -EBUSY;
483 goto out;
484 }
485 tasklet_disable(&engine->execlists.tasklet);
486
487 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
488 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
489
4ba5c086 490 i915_request_get(rq);
32ff621f
CW		 491 execlists_hold(engine, rq);
492 GEM_BUG_ON(!i915_request_on_hold(rq));
493
494 intel_engine_reset(engine, NULL);
495 GEM_BUG_ON(rq->fence.error != -EIO);
496
497 tasklet_enable(&engine->execlists.tasklet);
498 clear_and_wake_up_bit(I915_RESET_ENGINE + id,
499 &gt->reset.flags);
500
501 /* Check that we do not resubmit the held request */
32ff621f
CW		 502 if (!i915_request_wait(rq, 0, HZ / 5)) {
503 pr_err("%s: on hold request completed!\n",
504 engine->name);
505 i915_request_put(rq);
506 err = -EIO;
507 goto out;
508 }
509 GEM_BUG_ON(!i915_request_on_hold(rq));
510
511 /* But is resubmitted on release */
512 execlists_unhold(engine, rq);
513 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
514 pr_err("%s: held request did not complete!\n",
515 engine->name);
516 intel_gt_set_wedged(gt);
517 err = -ETIME;
518 }
519 i915_request_put(rq);
520
521out:
522 engine_heartbeat_enable(engine, heartbeat);
523 intel_context_put(ce);
524 if (err)
525 break;
526 }
527
528 igt_spinner_fini(&spin);
529 return err;
530}
531
70a76a9b
CW		 532static const char *error_repr(int err)
533{
534 return err ? "bad" : "good";
535}
536
537static int live_error_interrupt(void *arg)
538{
539 static const struct error_phase {
540 enum { GOOD = 0, BAD = -EIO } error[2];
541 } phases[] = {
542 { { BAD, GOOD } },
543 { { BAD, BAD } },
544 { { BAD, GOOD } },
545 { { GOOD, GOOD } }, /* sentinel */
546 };
547 struct intel_gt *gt = arg;
548 struct intel_engine_cs *engine;
549 enum intel_engine_id id;
550
551 /*
552 * We hook up the CS_MASTER_ERROR_INTERRUPT to have forewarning
553 * of invalid commands in user batches that will cause a GPU hang.
554 * This is a faster mechanism than using hangcheck/heartbeats, but
555 * only detects problems the HW knows about -- it will not warn when
556 * we kill the HW!
557 *
558 * To verify our detection and reset, we throw some invalid commands
559 * at the HW and wait for the interrupt.
560 */
561
562 if (!intel_has_reset_engine(gt))
563 return 0;
564
565 for_each_engine(engine, gt, id) {
566 const struct error_phase *p;
567 unsigned long heartbeat;
568 int err = 0;
569
570 engine_heartbeat_disable(engine, &heartbeat);
571
572 for (p = phases; p->error[0] != GOOD; p++) {
573 struct i915_request *client[ARRAY_SIZE(phases->error)];
574 u32 *cs;
575 int i;
576
577 memset(client, 0, sizeof(*client));
578 for (i = 0; i < ARRAY_SIZE(client); i++) {
579 struct intel_context *ce;
580 struct i915_request *rq;
581
582 ce = intel_context_create(engine);
583 if (IS_ERR(ce)) {
584 err = PTR_ERR(ce);
585 goto out;
586 }
587
588 rq = intel_context_create_request(ce);
589 intel_context_put(ce);
590 if (IS_ERR(rq)) {
591 err = PTR_ERR(rq);
592 goto out;
593 }
594
595 if (rq->engine->emit_init_breadcrumb) {
596 err = rq->engine->emit_init_breadcrumb(rq);
597 if (err) {
598 i915_request_add(rq);
599 goto out;
600 }
601 }
602
603 cs = intel_ring_begin(rq, 2);
604 if (IS_ERR(cs)) {
605 i915_request_add(rq);
606 err = PTR_ERR(cs);
607 goto out;
608 }
609
610 if (p->error[i]) {
611 *cs++ = 0xdeadbeef;
612 *cs++ = 0xdeadbeef;
613 } else {
614 *cs++ = MI_NOOP;
615 *cs++ = MI_NOOP;
616 }
617
618 client[i] = i915_request_get(rq);
619 i915_request_add(rq);
620 }
621
622 err = wait_for_submit(engine, client[0], HZ / 2);
623 if (err) {
624 pr_err("%s: first request did not start within time!\n",
625 engine->name);
626 err = -ETIME;
627 goto out;
628 }
629
630 for (i = 0; i < ARRAY_SIZE(client); i++) {
6313e78e
CW		 631 if (i915_request_wait(client[i], 0, HZ / 5) < 0)
632 pr_debug("%s: %s request incomplete!\n",
633 engine->name,
634 error_repr(p->error[i]));
635
636 if (!i915_request_started(client[i])) {
637 pr_debug("%s: %s request not stated!\n",
638 engine->name,
639 error_repr(p->error[i]));
70a76a9b
CW		 640 err = -ETIME;
641 goto out;
642 }
643
6313e78e
CW		 644 /* Kick the tasklet to process the error */
645 intel_engine_flush_submission(engine);
70a76a9b
CW		 646 if (client[i]->fence.error != p->error[i]) {
647 pr_err("%s: %s request completed with wrong error code: %d\n",
648 engine->name,
649 error_repr(p->error[i]),
650 client[i]->fence.error);
651 err = -EINVAL;
652 goto out;
653 }
654 }
655
656out:
657 for (i = 0; i < ARRAY_SIZE(client); i++)
658 if (client[i])
659 i915_request_put(client[i]);
660 if (err) {
661 pr_err("%s: failed at phase[%zd] { %d, %d }\n",
662 engine->name, p - phases,
663 p->error[0], p->error[1]);
664 break;
665 }
666 }
667
668 engine_heartbeat_enable(engine, heartbeat);
669 if (err) {
670 intel_gt_set_wedged(gt);
671 return err;
672 }
673 }
674
675 return 0;
676}
677
8ee36e04
CW		 678static int
679emit_semaphore_chain(struct i915_request *rq, struct i915_vma *vma, int idx)
680{
681 u32 *cs;
682
683 cs = intel_ring_begin(rq, 10);
684 if (IS_ERR(cs))
685 return PTR_ERR(cs);
686
687 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
688
689 *cs++ = MI_SEMAPHORE_WAIT |
690 MI_SEMAPHORE_GLOBAL_GTT |
691 MI_SEMAPHORE_POLL |
692 MI_SEMAPHORE_SAD_NEQ_SDD;
693 *cs++ = 0;
694 *cs++ = i915_ggtt_offset(vma) + 4 * idx;
695 *cs++ = 0;
696
697 if (idx > 0) {
698 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
699 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
700 *cs++ = 0;
701 *cs++ = 1;
702 } else {
703 *cs++ = MI_NOOP;
704 *cs++ = MI_NOOP;
705 *cs++ = MI_NOOP;
706 *cs++ = MI_NOOP;
707 }
708
709 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
710
711 intel_ring_advance(rq, cs);
712 return 0;
713}
714
715static struct i915_request *
716semaphore_queue(struct intel_engine_cs *engine, struct i915_vma *vma, int idx)
717{
e6ba7648 718 struct intel_context *ce;
8ee36e04
CW		 719 struct i915_request *rq;
720 int err;
721
e6ba7648
CW		 722 ce = intel_context_create(engine);
723 if (IS_ERR(ce))
724 return ERR_CAST(ce);
8ee36e04 725
e6ba7648 726 rq = intel_context_create_request(ce);
8ee36e04 727 if (IS_ERR(rq))
e6ba7648 728 goto out_ce;
8ee36e04 729
253a774b
CW		 730 err = 0;
731 if (rq->engine->emit_init_breadcrumb)
732 err = rq->engine->emit_init_breadcrumb(rq);
733 if (err == 0)
734 err = emit_semaphore_chain(rq, vma, idx);
735 if (err == 0)
736 i915_request_get(rq);
8ee36e04
CW		 737 i915_request_add(rq);
738 if (err)
739 rq = ERR_PTR(err);
740
e6ba7648
CW		 741out_ce:
742 intel_context_put(ce);
8ee36e04
CW		 743 return rq;
744}
745
746static int
747release_queue(struct intel_engine_cs *engine,
748 struct i915_vma *vma,
253a774b 749 int idx, int prio)
8ee36e04
CW		 750{
751 struct i915_sched_attr attr = {
253a774b 752 .priority = prio,
8ee36e04
CW		 753 };
754 struct i915_request *rq;
755 u32 *cs;
756
de5825be 757 rq = intel_engine_create_kernel_request(engine);
8ee36e04
CW		 758 if (IS_ERR(rq))
759 return PTR_ERR(rq);
760
761 cs = intel_ring_begin(rq, 4);
762 if (IS_ERR(cs)) {
763 i915_request_add(rq);
764 return PTR_ERR(cs);
765 }
766
767 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
768 *cs++ = i915_ggtt_offset(vma) + 4 * (idx - 1);
769 *cs++ = 0;
770 *cs++ = 1;
771
772 intel_ring_advance(rq, cs);
253a774b
CW		 773
774 i915_request_get(rq);
8ee36e04
CW		 775 i915_request_add(rq);
776
253a774b 777 local_bh_disable();
8ee36e04 778 engine->schedule(rq, &attr);
253a774b
CW		 779 local_bh_enable(); /* kick tasklet */
780
781 i915_request_put(rq);
8ee36e04
CW		 782
783 return 0;
784}
785
786static int
787slice_semaphore_queue(struct intel_engine_cs *outer,
788 struct i915_vma *vma,
789 int count)
790{
791 struct intel_engine_cs *engine;
792 struct i915_request *head;
793 enum intel_engine_id id;
794 int err, i, n = 0;
795
796 head = semaphore_queue(outer, vma, n++);
797 if (IS_ERR(head))
798 return PTR_ERR(head);
799
5d904e3c 800 for_each_engine(engine, outer->gt, id) {
8ee36e04
CW		 801 for (i = 0; i < count; i++) {
802 struct i915_request *rq;
803
804 rq = semaphore_queue(engine, vma, n++);
805 if (IS_ERR(rq)) {
806 err = PTR_ERR(rq);
807 goto out;
808 }
253a774b
CW		 809
810 i915_request_put(rq);
8ee36e04
CW		 811 }
812 }
813
253a774b 814 err = release_queue(outer, vma, n, INT_MAX);
8ee36e04
CW		 815 if (err)
816 goto out;
817
7e805762 818 if (i915_request_wait(head, 0,
8ee36e04
CW		 819 2 * RUNTIME_INFO(outer->i915)->num_engines * (count + 2) * (count + 3)) < 0) {
820 pr_err("Failed to slice along semaphore chain of length (%d, %d)!\n",
821 count, n);
822 GEM_TRACE_DUMP();
cb823ed9 823 intel_gt_set_wedged(outer->gt);
8ee36e04
CW		 824 err = -EIO;
825 }
826
827out:
828 i915_request_put(head);
829 return err;
830}
831
832static int live_timeslice_preempt(void *arg)
833{
1357fa81 834 struct intel_gt *gt = arg;
8ee36e04 835 struct drm_i915_gem_object *obj;
8ee36e04
CW		 836 struct i915_vma *vma;
837 void *vaddr;
838 int err = 0;
839 int count;
840
841 /*
842 * If a request takes too long, we would like to give other users
843 * a fair go on the GPU. In particular, users may create batches
844 * that wait upon external input, where that input may even be
845 * supplied by another GPU job. To avoid blocking forever, we
846 * need to preempt the current task and replace it with another
847 * ready task.
848 */
b79029b2
CW		 849 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
850 return 0;
8ee36e04 851
1357fa81 852 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
2af40298
CW		 853 if (IS_ERR(obj))
854 return PTR_ERR(obj);
8ee36e04 855
1357fa81 856 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
8ee36e04
CW		 857 if (IS_ERR(vma)) {
858 err = PTR_ERR(vma);
859 goto err_obj;
860 }
861
862 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
863 if (IS_ERR(vaddr)) {
864 err = PTR_ERR(vaddr);
865 goto err_obj;
866 }
867
868 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
869 if (err)
870 goto err_map;
871
bd46aa22
CW		 872 err = i915_vma_sync(vma);
873 if (err)
874 goto err_pin;
875
8ee36e04
CW		 876 for_each_prime_number_from(count, 1, 16) {
877 struct intel_engine_cs *engine;
878 enum intel_engine_id id;
879
5d904e3c 880 for_each_engine(engine, gt, id) {
e6ba7648
CW		 881 unsigned long saved;
882
09975b86
CW		 883 if (!intel_engine_has_preemption(engine))
884 continue;
885
8ee36e04
CW		 886 memset(vaddr, 0, PAGE_SIZE);
887
e6ba7648 888 engine_heartbeat_disable(engine, &saved);
8ee36e04 889 err = slice_semaphore_queue(engine, vma, count);
e6ba7648 890 engine_heartbeat_enable(engine, saved);
8ee36e04
CW		 891 if (err)
892 goto err_pin;
893
1357fa81 894 if (igt_flush_test(gt->i915)) {
8ee36e04
CW		 895 err = -EIO;
896 goto err_pin;
897 }
898 }
899 }
900
901err_pin:
902 i915_vma_unpin(vma);
903err_map:
904 i915_gem_object_unpin_map(obj);
905err_obj:
906 i915_gem_object_put(obj);
8ee36e04
CW		 907 return err;
908}
909
4c8ed8b1
CW		 910static struct i915_request *
911create_rewinder(struct intel_context *ce,
912 struct i915_request *wait,
913 void *slot, int idx)
914{
915 const u32 offset =
916 i915_ggtt_offset(ce->engine->status_page.vma) +
917 offset_in_page(slot);
918 struct i915_request *rq;
919 u32 *cs;
920 int err;
921
922 rq = intel_context_create_request(ce);
923 if (IS_ERR(rq))
924 return rq;
925
926 if (wait) {
927 err = i915_request_await_dma_fence(rq, &wait->fence);
928 if (err)
929 goto err;
930 }
931
220dcfc1 932 cs = intel_ring_begin(rq, 14);
4c8ed8b1
CW		 933 if (IS_ERR(cs)) {
934 err = PTR_ERR(cs);
935 goto err;
936 }
937
938 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
939 *cs++ = MI_NOOP;
940
941 *cs++ = MI_SEMAPHORE_WAIT |
942 MI_SEMAPHORE_GLOBAL_GTT |
943 MI_SEMAPHORE_POLL |
220dcfc1
CW		 944 MI_SEMAPHORE_SAD_GTE_SDD;
945 *cs++ = idx;
4c8ed8b1
CW		 946 *cs++ = offset;
947 *cs++ = 0;
948
949 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
950 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
951 *cs++ = offset + idx * sizeof(u32);
952 *cs++ = 0;
953
220dcfc1
CW		 954 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
955 *cs++ = offset;
956 *cs++ = 0;
957 *cs++ = idx + 1;
958
4c8ed8b1
CW		 959 intel_ring_advance(rq, cs);
960
961 rq->sched.attr.priority = I915_PRIORITY_MASK;
962 err = 0;
963err:
964 i915_request_get(rq);
965 i915_request_add(rq);
966 if (err) {
967 i915_request_put(rq);
968 return ERR_PTR(err);
969 }
970
971 return rq;
972}
973
974static int live_timeslice_rewind(void *arg)
975{
976 struct intel_gt *gt = arg;
977 struct intel_engine_cs *engine;
978 enum intel_engine_id id;
979
980 /*
981 * The usual presumption on timeslice expiration is that we replace
982 * the active context with another. However, given a chain of
983 * dependencies we may end up with replacing the context with itself,
984 * but only a few of those requests, forcing us to rewind the
985 * RING_TAIL of the original request.
986 */
987 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
988 return 0;
989
990 for_each_engine(engine, gt, id) {
991 enum { A1, A2, B1 };
220dcfc1 992 enum { X = 1, Z, Y };
4c8ed8b1
CW		 993 struct i915_request *rq[3] = {};
994 struct intel_context *ce;
995 unsigned long heartbeat;
996 unsigned long timeslice;
997 int i, err = 0;
998 u32 *slot;
999
1000 if (!intel_engine_has_timeslices(engine))
1001 continue;
1002
1003 /*
1004 * A:rq1 -- semaphore wait, timestamp X
1005 * A:rq2 -- write timestamp Y
1006 *
1007 * B:rq1 [await A:rq1] -- write timestamp Z
1008 *
1009 * Force timeslice, release semaphore.
1010 *
1011 * Expect execution/evaluation order XZY
1012 */
1013
1014 engine_heartbeat_disable(engine, &heartbeat);
1015 timeslice = xchg(&engine->props.timeslice_duration_ms, 1);
1016
1017 slot = memset32(engine->status_page.addr + 1000, 0, 4);
1018
1019 ce = intel_context_create(engine);
1020 if (IS_ERR(ce)) {
1021 err = PTR_ERR(ce);
1022 goto err;
1023 }
1024
220dcfc1 1025 rq[0] = create_rewinder(ce, NULL, slot, X);
4c8ed8b1
CW		 1026 if (IS_ERR(rq[0])) {
1027 intel_context_put(ce);
1028 goto err;
1029 }
1030
220dcfc1 1031 rq[1] = create_rewinder(ce, NULL, slot, Y);
4c8ed8b1
CW		 1032 intel_context_put(ce);
1033 if (IS_ERR(rq[1]))
1034 goto err;
1035
1036 err = wait_for_submit(engine, rq[1], HZ / 2);
1037 if (err) {
1038 pr_err("%s: failed to submit first context\n",
1039 engine->name);
1040 goto err;
1041 }
1042
1043 ce = intel_context_create(engine);
1044 if (IS_ERR(ce)) {
1045 err = PTR_ERR(ce);
1046 goto err;
1047 }
1048
220dcfc1 1049 rq[2] = create_rewinder(ce, rq[0], slot, Z);
4c8ed8b1
CW		 1050 intel_context_put(ce);
1051 if (IS_ERR(rq[2]))
1052 goto err;
1053
1054 err = wait_for_submit(engine, rq[2], HZ / 2);
1055 if (err) {
1056 pr_err("%s: failed to submit second context\n",
1057 engine->name);
1058 goto err;
1059 }
1060 GEM_BUG_ON(!timer_pending(&engine->execlists.timer));
1061
1062 /* ELSP[] = { { A:rq1, A:rq2 }, { B:rq1 } } */
220dcfc1
CW		 1063 if (i915_request_is_active(rq[A2])) { /* semaphore yielded! */
1064 /* Wait for the timeslice to kick in */
1065 del_timer(&engine->execlists.timer);
1066 tasklet_hi_schedule(&engine->execlists.tasklet);
1067 intel_engine_flush_submission(engine);
1068 }
4c8ed8b1
CW		 1069 /* -> ELSP[] = { { A:rq1 }, { B:rq1 } } */
1070 GEM_BUG_ON(!i915_request_is_active(rq[A1]));
1071 GEM_BUG_ON(!i915_request_is_active(rq[B1]));
1072 GEM_BUG_ON(i915_request_is_active(rq[A2]));
1073
1074 /* Release the hounds! */
1075 slot[0] = 1;
1076 wmb(); /* "pairs" with GPU; paranoid kick of internal CPU$ */
1077
1078 for (i = 1; i <= 3; i++) {
1079 unsigned long timeout = jiffies + HZ / 2;
1080
1081 while (!READ_ONCE(slot[i]) &&
1082 time_before(jiffies, timeout))
1083 ;
1084
1085 if (!time_before(jiffies, timeout)) {
1086 pr_err("%s: rq[%d] timed out\n",
1087 engine->name, i - 1);
1088 err = -ETIME;
1089 goto err;
1090 }
1091
1092 pr_debug("%s: slot[%d]:%x\n", engine->name, i, slot[i]);
1093 }
1094
1095 /* XZY: XZ < XY */
1096 if (slot[Z] - slot[X] >= slot[Y] - slot[X]) {
1097 pr_err("%s: timeslicing did not run context B [%u] before A [%u]!\n",
1098 engine->name,
1099 slot[Z] - slot[X],
1100 slot[Y] - slot[X]);
1101 err = -EINVAL;
1102 }
1103
1104err:
1105 memset32(&slot[0], -1, 4);
1106 wmb();
1107
1108 engine->props.timeslice_duration_ms = timeslice;
1109 engine_heartbeat_enable(engine, heartbeat);
1110 for (i = 0; i < 3; i++)
1111 i915_request_put(rq[i]);
1112 if (igt_flush_test(gt->i915))
1113 err = -EIO;
1114 if (err)
1115 return err;
1116 }
1117
1118 return 0;
1119}
1120
253a774b
CW		 1121static struct i915_request *nop_request(struct intel_engine_cs *engine)
1122{
1123 struct i915_request *rq;
1124
de5825be 1125 rq = intel_engine_create_kernel_request(engine);
253a774b
CW		 1126 if (IS_ERR(rq))
1127 return rq;
1128
1129 i915_request_get(rq);
1130 i915_request_add(rq);
1131
1132 return rq;
1133}
1134
b79029b2
CW		 1135static long timeslice_threshold(const struct intel_engine_cs *engine)
1136{
1137 return 2 * msecs_to_jiffies_timeout(timeslice(engine)) + 1;
1138}
1139
253a774b
CW		 1140static int live_timeslice_queue(void *arg)
1141{
1142 struct intel_gt *gt = arg;
1143 struct drm_i915_gem_object *obj;
1144 struct intel_engine_cs *engine;
1145 enum intel_engine_id id;
1146 struct i915_vma *vma;
1147 void *vaddr;
1148 int err = 0;
1149
1150 /*
1151 * Make sure that even if ELSP[0] and ELSP[1] are filled with
1152 * timeslicing between them disabled, we *do* enable timeslicing
1153 * if the queue demands it. (Normally, we do not submit if
1154 * ELSP[1] is already occupied, so must rely on timeslicing to
1155 * eject ELSP[0] in favour of the queue.)
1156 */
b79029b2
CW		 1157 if (!IS_ACTIVE(CONFIG_DRM_I915_TIMESLICE_DURATION))
1158 return 0;
253a774b
CW		 1159
1160 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
1161 if (IS_ERR(obj))
1162 return PTR_ERR(obj);
1163
1164 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
1165 if (IS_ERR(vma)) {
1166 err = PTR_ERR(vma);
1167 goto err_obj;
1168 }
1169
1170 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1171 if (IS_ERR(vaddr)) {
1172 err = PTR_ERR(vaddr);
1173 goto err_obj;
1174 }
1175
1176 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1177 if (err)
1178 goto err_map;
1179
bd46aa22
CW		 1180 err = i915_vma_sync(vma);
1181 if (err)
1182 goto err_pin;
1183
253a774b
CW		 1184 for_each_engine(engine, gt, id) {
1185 struct i915_sched_attr attr = {
1186 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
1187 };
1188 struct i915_request *rq, *nop;
e6ba7648 1189 unsigned long saved;
253a774b
CW		 1190
1191 if (!intel_engine_has_preemption(engine))
1192 continue;
1193
e6ba7648 1194 engine_heartbeat_disable(engine, &saved);
253a774b
CW		 1195 memset(vaddr, 0, PAGE_SIZE);
1196
1197 /* ELSP[0]: semaphore wait */
1198 rq = semaphore_queue(engine, vma, 0);
1199 if (IS_ERR(rq)) {
1200 err = PTR_ERR(rq);
e6ba7648 1201 goto err_heartbeat;
253a774b
CW		 1202 }
1203 engine->schedule(rq, &attr);
6d728d92
CW		 1204 err = wait_for_submit(engine, rq, HZ / 2);
1205 if (err) {
1206 pr_err("%s: Timed out trying to submit semaphores\n",
1207 engine->name);
1208 goto err_rq;
1209 }
253a774b
CW		 1210
1211 /* ELSP[1]: nop request */
1212 nop = nop_request(engine);
1213 if (IS_ERR(nop)) {
1214 err = PTR_ERR(nop);
e6ba7648 1215 goto err_rq;
253a774b 1216 }
6d728d92 1217 err = wait_for_submit(engine, nop, HZ / 2);
253a774b 1218 i915_request_put(nop);
6d728d92
CW		 1219 if (err) {
1220 pr_err("%s: Timed out trying to submit nop\n",
1221 engine->name);
1222 goto err_rq;
1223 }
253a774b
CW		 1224
1225 GEM_BUG_ON(i915_request_completed(rq));
1226 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
1227
1228 /* Queue: semaphore signal, matching priority as semaphore */
1229 err = release_queue(engine, vma, 1, effective_prio(rq));
e6ba7648
CW		 1230 if (err)
1231 goto err_rq;
253a774b
CW		 1232
1233 intel_engine_flush_submission(engine);
1234 if (!READ_ONCE(engine->execlists.timer.expires) &&
1235 !i915_request_completed(rq)) {
1236 struct drm_printer p =
1237 drm_info_printer(gt->i915->drm.dev);
1238
1239 GEM_TRACE_ERR("%s: Failed to enable timeslicing!\n",
1240 engine->name);
1241 intel_engine_dump(engine, &p,
1242 "%s\n", engine->name);
1243 GEM_TRACE_DUMP();
1244
1245 memset(vaddr, 0xff, PAGE_SIZE);
1246 err = -EINVAL;
1247 }
1248
b79029b2
CW		 1249 /* Timeslice every jiffy, so within 2 we should signal */
1250 if (i915_request_wait(rq, 0, timeslice_threshold(engine)) < 0) {
253a774b
CW		 1251 struct drm_printer p =
1252 drm_info_printer(gt->i915->drm.dev);
1253
1254 pr_err("%s: Failed to timeslice into queue\n",
1255 engine->name);
1256 intel_engine_dump(engine, &p,
1257 "%s\n", engine->name);
1258
1259 memset(vaddr, 0xff, PAGE_SIZE);
1260 err = -EIO;
1261 }
e6ba7648 1262err_rq:
253a774b 1263 i915_request_put(rq);
e6ba7648
CW		 1264err_heartbeat:
1265 engine_heartbeat_enable(engine, saved);
253a774b
CW		 1266 if (err)
1267 break;
1268 }
1269
bd46aa22 1270err_pin:
253a774b
CW		 1271 i915_vma_unpin(vma);
1272err_map:
1273 i915_gem_object_unpin_map(obj);
1274err_obj:
1275 i915_gem_object_put(obj);
1276 return err;
1277}
1278
bac24f59
CW		 1279static int live_busywait_preempt(void *arg)
1280{
1357fa81 1281 struct intel_gt *gt = arg;
bac24f59
CW		 1282 struct i915_gem_context *ctx_hi, *ctx_lo;
1283 struct intel_engine_cs *engine;
1284 struct drm_i915_gem_object *obj;
1285 struct i915_vma *vma;
1286 enum intel_engine_id id;
bac24f59
CW		 1287 int err = -ENOMEM;
1288 u32 *map;
1289
1290 /*
1291 * Verify that even without HAS_LOGICAL_RING_PREEMPTION, we can
1292 * preempt the busywaits used to synchronise between rings.
1293 */
1294
1357fa81 1295 ctx_hi = kernel_context(gt->i915);
bac24f59 1296 if (!ctx_hi)
2af40298 1297 return -ENOMEM;
6e7eb7a8
CW		 1298 ctx_hi->sched.priority =
1299 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
bac24f59 1300
1357fa81 1301 ctx_lo = kernel_context(gt->i915);
bac24f59
CW		 1302 if (!ctx_lo)
1303 goto err_ctx_hi;
6e7eb7a8
CW		 1304 ctx_lo->sched.priority =
1305 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
bac24f59 1306
1357fa81 1307 obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
bac24f59
CW		 1308 if (IS_ERR(obj)) {
1309 err = PTR_ERR(obj);
1310 goto err_ctx_lo;
1311 }
1312
1313 map = i915_gem_object_pin_map(obj, I915_MAP_WC);
1314 if (IS_ERR(map)) {
1315 err = PTR_ERR(map);
1316 goto err_obj;
1317 }
1318
1357fa81 1319 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL);
bac24f59
CW		 1320 if (IS_ERR(vma)) {
1321 err = PTR_ERR(vma);
1322 goto err_map;
1323 }
1324
1325 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
1326 if (err)
1327 goto err_map;
1328
bd46aa22
CW		 1329 err = i915_vma_sync(vma);
1330 if (err)
1331 goto err_vma;
1332
5d904e3c 1333 for_each_engine(engine, gt, id) {
bac24f59
CW		 1334 struct i915_request *lo, *hi;
1335 struct igt_live_test t;
1336 u32 *cs;
1337
09975b86
CW		 1338 if (!intel_engine_has_preemption(engine))
1339 continue;
1340
bac24f59
CW		 1341 if (!intel_engine_can_store_dword(engine))
1342 continue;
1343
1357fa81 1344 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
bac24f59
CW		 1345 err = -EIO;
1346 goto err_vma;
1347 }
1348
1349 /*
1350 * We create two requests. The low priority request
1351 * busywaits on a semaphore (inside the ringbuffer where
1352 * is should be preemptible) and the high priority requests
1353 * uses a MI_STORE_DWORD_IMM to update the semaphore value
1354 * allowing the first request to complete. If preemption
1355 * fails, we hang instead.
1356 */
1357
46472b3e 1358 lo = igt_request_alloc(ctx_lo, engine);
bac24f59
CW		 1359 if (IS_ERR(lo)) {
1360 err = PTR_ERR(lo);
1361 goto err_vma;
1362 }
1363
1364 cs = intel_ring_begin(lo, 8);
1365 if (IS_ERR(cs)) {
1366 err = PTR_ERR(cs);
1367 i915_request_add(lo);
1368 goto err_vma;
1369 }
1370
1371 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1372 *cs++ = i915_ggtt_offset(vma);
1373 *cs++ = 0;
1374 *cs++ = 1;
1375
1376 /* XXX Do we need a flush + invalidate here? */
1377
1378 *cs++ = MI_SEMAPHORE_WAIT |
1379 MI_SEMAPHORE_GLOBAL_GTT |
1380 MI_SEMAPHORE_POLL |
1381 MI_SEMAPHORE_SAD_EQ_SDD;
1382 *cs++ = 0;
1383 *cs++ = i915_ggtt_offset(vma);
1384 *cs++ = 0;
1385
1386 intel_ring_advance(lo, cs);
1ff2f9e2
CW		 1387
1388 i915_request_get(lo);
bac24f59
CW		 1389 i915_request_add(lo);
1390
1391 if (wait_for(READ_ONCE(*map), 10)) {
1ff2f9e2 1392 i915_request_put(lo);
bac24f59
CW		 1393 err = -ETIMEDOUT;
1394 goto err_vma;
1395 }
1396
1397 /* Low priority request should be busywaiting now */
2f530945 1398 if (i915_request_wait(lo, 0, 1) != -ETIME) {
1ff2f9e2 1399 i915_request_put(lo);
bac24f59
CW		 1400 pr_err("%s: Busywaiting request did not!\n",
1401 engine->name);
1402 err = -EIO;
1403 goto err_vma;
1404 }
1405
46472b3e 1406 hi = igt_request_alloc(ctx_hi, engine);
bac24f59
CW		 1407 if (IS_ERR(hi)) {
1408 err = PTR_ERR(hi);
1ff2f9e2 1409 i915_request_put(lo);
bac24f59
CW		 1410 goto err_vma;
1411 }
1412
1413 cs = intel_ring_begin(hi, 4);
1414 if (IS_ERR(cs)) {
1415 err = PTR_ERR(cs);
1416 i915_request_add(hi);
1ff2f9e2 1417 i915_request_put(lo);
bac24f59
CW		 1418 goto err_vma;
1419 }
1420
1421 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1422 *cs++ = i915_ggtt_offset(vma);
1423 *cs++ = 0;
1424 *cs++ = 0;
1425
1426 intel_ring_advance(hi, cs);
1427 i915_request_add(hi);
1428
2f530945 1429 if (i915_request_wait(lo, 0, HZ / 5) < 0) {
1357fa81 1430 struct drm_printer p = drm_info_printer(gt->i915->drm.dev);
bac24f59
CW		 1431
1432 pr_err("%s: Failed to preempt semaphore busywait!\n",
1433 engine->name);
1434
1435 intel_engine_dump(engine, &p, "%s\n", engine->name);
1436 GEM_TRACE_DUMP();
1437
1ff2f9e2 1438 i915_request_put(lo);
1357fa81 1439 intel_gt_set_wedged(gt);
bac24f59
CW		 1440 err = -EIO;
1441 goto err_vma;
1442 }
1443 GEM_BUG_ON(READ_ONCE(*map));
1ff2f9e2 1444 i915_request_put(lo);
bac24f59
CW		 1445
1446 if (igt_live_test_end(&t)) {
1447 err = -EIO;
1448 goto err_vma;
1449 }
1450 }
1451
1452 err = 0;
1453err_vma:
1454 i915_vma_unpin(vma);
1455err_map:
1456 i915_gem_object_unpin_map(obj);
1457err_obj:
1458 i915_gem_object_put(obj);
1459err_ctx_lo:
1460 kernel_context_close(ctx_lo);
1461err_ctx_hi:
1462 kernel_context_close(ctx_hi);
bac24f59
CW		 1463 return err;
1464}
1465
f277bc0c
CW		 1466static struct i915_request *
1467spinner_create_request(struct igt_spinner *spin,
1468 struct i915_gem_context *ctx,
1469 struct intel_engine_cs *engine,
1470 u32 arb)
1471{
1472 struct intel_context *ce;
1473 struct i915_request *rq;
1474
f1c4d157 1475 ce = i915_gem_context_get_engine(ctx, engine->legacy_idx);
f277bc0c
CW		 1476 if (IS_ERR(ce))
1477 return ERR_CAST(ce);
1478
1479 rq = igt_spinner_create_request(spin, ce, arb);
1480 intel_context_put(ce);
1481 return rq;
1482}
1483
2c66555e
CW		 1484static int live_preempt(void *arg)
1485{
1357fa81 1486 struct intel_gt *gt = arg;
2c66555e 1487 struct i915_gem_context *ctx_hi, *ctx_lo;
8d2f6e2f 1488 struct igt_spinner spin_hi, spin_lo;
2c66555e
CW		 1489 struct intel_engine_cs *engine;
1490 enum intel_engine_id id;
1491 int err = -ENOMEM;
1492
1357fa81 1493 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2c66555e
CW		 1494 return 0;
1495
1357fa81 1496 if (!(gt->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION))
3123ada8
CW		 1497 pr_err("Logical preemption supported, but not exposed\n");
1498
1357fa81 1499 if (igt_spinner_init(&spin_hi, gt))
2af40298 1500 return -ENOMEM;
2c66555e 1501
1357fa81 1502 if (igt_spinner_init(&spin_lo, gt))
2c66555e
CW		 1503 goto err_spin_hi;
1504
1357fa81 1505 ctx_hi = kernel_context(gt->i915);
2c66555e
CW		 1506 if (!ctx_hi)
1507 goto err_spin_lo;
7651a445
CW		 1508 ctx_hi->sched.priority =
1509 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
2c66555e 1510
1357fa81 1511 ctx_lo = kernel_context(gt->i915);
2c66555e
CW		 1512 if (!ctx_lo)
1513 goto err_ctx_hi;
7651a445
CW		 1514 ctx_lo->sched.priority =
1515 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
2c66555e 1516
5d904e3c 1517 for_each_engine(engine, gt, id) {
e70d3d80 1518 struct igt_live_test t;
2c66555e
CW		 1519 struct i915_request *rq;
1520
3123ada8
CW		 1521 if (!intel_engine_has_preemption(engine))
1522 continue;
1523
1357fa81 1524 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
e70d3d80
CW		 1525 err = -EIO;
1526 goto err_ctx_lo;
1527 }
1528
f277bc0c
CW		 1529 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1530 MI_ARB_CHECK);
2c66555e
CW		 1531 if (IS_ERR(rq)) {
1532 err = PTR_ERR(rq);
1533 goto err_ctx_lo;
1534 }
1535
1536 i915_request_add(rq);
8d2f6e2f 1537 if (!igt_wait_for_spinner(&spin_lo, rq)) {
2c66555e
CW		 1538 GEM_TRACE("lo spinner failed to start\n");
1539 GEM_TRACE_DUMP();
1357fa81 1540 intel_gt_set_wedged(gt);
2c66555e
CW		 1541 err = -EIO;
1542 goto err_ctx_lo;
1543 }
1544
f277bc0c
CW		 1545 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1546 MI_ARB_CHECK);
2c66555e 1547 if (IS_ERR(rq)) {
8d2f6e2f 1548 igt_spinner_end(&spin_lo);
2c66555e
CW		 1549 err = PTR_ERR(rq);
1550 goto err_ctx_lo;
1551 }
1552
1553 i915_request_add(rq);
8d2f6e2f 1554 if (!igt_wait_for_spinner(&spin_hi, rq)) {
2c66555e
CW		 1555 GEM_TRACE("hi spinner failed to start\n");
1556 GEM_TRACE_DUMP();
1357fa81 1557 intel_gt_set_wedged(gt);
2c66555e
CW		 1558 err = -EIO;
1559 goto err_ctx_lo;
1560 }
1561
8d2f6e2f
TU		 1562 igt_spinner_end(&spin_hi);
1563 igt_spinner_end(&spin_lo);
e70d3d80
CW		 1564
1565 if (igt_live_test_end(&t)) {
2c66555e
CW		 1566 err = -EIO;
1567 goto err_ctx_lo;
1568 }
1569 }
1570
1571 err = 0;
1572err_ctx_lo:
1573 kernel_context_close(ctx_lo);
1574err_ctx_hi:
1575 kernel_context_close(ctx_hi);
1576err_spin_lo:
8d2f6e2f 1577 igt_spinner_fini(&spin_lo);
2c66555e 1578err_spin_hi:
8d2f6e2f 1579 igt_spinner_fini(&spin_hi);
2c66555e
CW		 1580 return err;
1581}
1582
1583static int live_late_preempt(void *arg)
1584{
1357fa81 1585 struct intel_gt *gt = arg;
2c66555e 1586 struct i915_gem_context *ctx_hi, *ctx_lo;
8d2f6e2f 1587 struct igt_spinner spin_hi, spin_lo;
2c66555e 1588 struct intel_engine_cs *engine;
b7268c5e 1589 struct i915_sched_attr attr = {};
2c66555e
CW		 1590 enum intel_engine_id id;
1591 int err = -ENOMEM;
1592
1357fa81 1593 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2c66555e
CW		 1594 return 0;
1595
1357fa81 1596 if (igt_spinner_init(&spin_hi, gt))
2af40298 1597 return -ENOMEM;
2c66555e 1598
1357fa81 1599 if (igt_spinner_init(&spin_lo, gt))
2c66555e
CW		 1600 goto err_spin_hi;
1601
1357fa81 1602 ctx_hi = kernel_context(gt->i915);
2c66555e
CW		 1603 if (!ctx_hi)
1604 goto err_spin_lo;
1605
1357fa81 1606 ctx_lo = kernel_context(gt->i915);
2c66555e
CW		 1607 if (!ctx_lo)
1608 goto err_ctx_hi;
1609
8ee36e04
CW		 1610 /* Make sure ctx_lo stays before ctx_hi until we trigger preemption. */
1611 ctx_lo->sched.priority = I915_USER_PRIORITY(1);
1612
5d904e3c 1613 for_each_engine(engine, gt, id) {
e70d3d80 1614 struct igt_live_test t;
2c66555e
CW		 1615 struct i915_request *rq;
1616
3123ada8
CW		 1617 if (!intel_engine_has_preemption(engine))
1618 continue;
1619
1357fa81 1620 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
e70d3d80
CW		 1621 err = -EIO;
1622 goto err_ctx_lo;
1623 }
1624
f277bc0c
CW		 1625 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
1626 MI_ARB_CHECK);
2c66555e
CW		 1627 if (IS_ERR(rq)) {
1628 err = PTR_ERR(rq);
1629 goto err_ctx_lo;
1630 }
1631
1632 i915_request_add(rq);
8d2f6e2f 1633 if (!igt_wait_for_spinner(&spin_lo, rq)) {
2c66555e
CW		 1634 pr_err("First context failed to start\n");
1635 goto err_wedged;
1636 }
1637
f277bc0c
CW		 1638 rq = spinner_create_request(&spin_hi, ctx_hi, engine,
1639 MI_NOOP);
2c66555e 1640 if (IS_ERR(rq)) {
8d2f6e2f 1641 igt_spinner_end(&spin_lo);
2c66555e
CW		 1642 err = PTR_ERR(rq);
1643 goto err_ctx_lo;
1644 }
1645
1646 i915_request_add(rq);
8d2f6e2f 1647 if (igt_wait_for_spinner(&spin_hi, rq)) {
2c66555e
CW		 1648 pr_err("Second context overtook first?\n");
1649 goto err_wedged;
1650 }
1651
7651a445 1652 attr.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
b7268c5e 1653 engine->schedule(rq, &attr);
2c66555e 1654
8d2f6e2f 1655 if (!igt_wait_for_spinner(&spin_hi, rq)) {
2c66555e
CW		 1656 pr_err("High priority context failed to preempt the low priority context\n");
1657 GEM_TRACE_DUMP();
1658 goto err_wedged;
1659 }
1660
8d2f6e2f
TU		 1661 igt_spinner_end(&spin_hi);
1662 igt_spinner_end(&spin_lo);
e70d3d80
CW		 1663
1664 if (igt_live_test_end(&t)) {
2c66555e
CW		 1665 err = -EIO;
1666 goto err_ctx_lo;
1667 }
1668 }
1669
1670 err = 0;
1671err_ctx_lo:
1672 kernel_context_close(ctx_lo);
1673err_ctx_hi:
1674 kernel_context_close(ctx_hi);
1675err_spin_lo:
8d2f6e2f 1676 igt_spinner_fini(&spin_lo);
2c66555e 1677err_spin_hi:
8d2f6e2f 1678 igt_spinner_fini(&spin_hi);
2c66555e
CW		 1679 return err;
1680
1681err_wedged:
8d2f6e2f
TU		 1682 igt_spinner_end(&spin_hi);
1683 igt_spinner_end(&spin_lo);
1357fa81 1684 intel_gt_set_wedged(gt);
2c66555e
CW		 1685 err = -EIO;
1686 goto err_ctx_lo;
1687}
1688
c9a64622
CW		 1689struct preempt_client {
1690 struct igt_spinner spin;
1691 struct i915_gem_context *ctx;
1692};
1693
1357fa81 1694static int preempt_client_init(struct intel_gt *gt, struct preempt_client *c)
c9a64622 1695{
1357fa81 1696 c->ctx = kernel_context(gt->i915);
c9a64622
CW		 1697 if (!c->ctx)
1698 return -ENOMEM;
1699
1357fa81 1700 if (igt_spinner_init(&c->spin, gt))
c9a64622
CW		 1701 goto err_ctx;
1702
1703 return 0;
1704
1705err_ctx:
1706 kernel_context_close(c->ctx);
1707 return -ENOMEM;
1708}
1709
1710static void preempt_client_fini(struct preempt_client *c)
1711{
1712 igt_spinner_fini(&c->spin);
1713 kernel_context_close(c->ctx);
1714}
1715
2a98f4e6
LL		 1716static int live_nopreempt(void *arg)
1717{
1357fa81 1718 struct intel_gt *gt = arg;
2a98f4e6
LL		 1719 struct intel_engine_cs *engine;
1720 struct preempt_client a, b;
1721 enum intel_engine_id id;
2a98f4e6
LL		 1722 int err = -ENOMEM;
1723
1724 /*
1725 * Verify that we can disable preemption for an individual request
1726 * that may be being observed and not want to be interrupted.
1727 */
1728
1357fa81 1729 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2a98f4e6
LL		 1730 return 0;
1731
1357fa81 1732 if (preempt_client_init(gt, &a))
2af40298 1733 return -ENOMEM;
1357fa81 1734 if (preempt_client_init(gt, &b))
2a98f4e6
LL		 1735 goto err_client_a;
1736 b.ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_MAX);
1737
5d904e3c 1738 for_each_engine(engine, gt, id) {
2a98f4e6
LL		 1739 struct i915_request *rq_a, *rq_b;
1740
1741 if (!intel_engine_has_preemption(engine))
1742 continue;
1743
1744 engine->execlists.preempt_hang.count = 0;
1745
f277bc0c
CW		 1746 rq_a = spinner_create_request(&a.spin,
1747 a.ctx, engine,
1748 MI_ARB_CHECK);
2a98f4e6
LL		 1749 if (IS_ERR(rq_a)) {
1750 err = PTR_ERR(rq_a);
1751 goto err_client_b;
1752 }
1753
1754 /* Low priority client, but unpreemptable! */
e1c31fb5 1755 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq_a->fence.flags);
2a98f4e6
LL		 1756
1757 i915_request_add(rq_a);
1758 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
1759 pr_err("First client failed to start\n");
1760 goto err_wedged;
1761 }
1762
f277bc0c
CW		 1763 rq_b = spinner_create_request(&b.spin,
1764 b.ctx, engine,
1765 MI_ARB_CHECK);
2a98f4e6
LL		 1766 if (IS_ERR(rq_b)) {
1767 err = PTR_ERR(rq_b);
1768 goto err_client_b;
1769 }
1770
1771 i915_request_add(rq_b);
1772
1773 /* B is much more important than A! (But A is unpreemptable.) */
1774 GEM_BUG_ON(rq_prio(rq_b) <= rq_prio(rq_a));
1775
1776 /* Wait long enough for preemption and timeslicing */
1777 if (igt_wait_for_spinner(&b.spin, rq_b)) {
1778 pr_err("Second client started too early!\n");
1779 goto err_wedged;
1780 }
1781
1782 igt_spinner_end(&a.spin);
1783
1784 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
1785 pr_err("Second client failed to start\n");
1786 goto err_wedged;
1787 }
1788
1789 igt_spinner_end(&b.spin);
1790
1791 if (engine->execlists.preempt_hang.count) {
1792 pr_err("Preemption recorded x%d; should have been suppressed!\n",
1793 engine->execlists.preempt_hang.count);
1794 err = -EINVAL;
1795 goto err_wedged;
1796 }
1797
1357fa81 1798 if (igt_flush_test(gt->i915))
2a98f4e6
LL		 1799 goto err_wedged;
1800 }
1801
1802 err = 0;
1803err_client_b:
1804 preempt_client_fini(&b);
1805err_client_a:
1806 preempt_client_fini(&a);
2a98f4e6
LL		 1807 return err;
1808
1809err_wedged:
1810 igt_spinner_end(&b.spin);
1811 igt_spinner_end(&a.spin);
1357fa81 1812 intel_gt_set_wedged(gt);
2a98f4e6
LL		 1813 err = -EIO;
1814 goto err_client_b;
1815}
1816
d12acee8
CW		 1817struct live_preempt_cancel {
1818 struct intel_engine_cs *engine;
1819 struct preempt_client a, b;
1820};
1821
1822static int __cancel_active0(struct live_preempt_cancel *arg)
1823{
1824 struct i915_request *rq;
1825 struct igt_live_test t;
1826 int err;
1827
1828 /* Preempt cancel of ELSP0 */
1829 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1830 if (igt_live_test_begin(&t, arg->engine->i915,
1831 __func__, arg->engine->name))
1832 return -EIO;
1833
d12acee8
CW		 1834 rq = spinner_create_request(&arg->a.spin,
1835 arg->a.ctx, arg->engine,
1836 MI_ARB_CHECK);
1837 if (IS_ERR(rq))
1838 return PTR_ERR(rq);
1839
9f3ccd40 1840 clear_bit(CONTEXT_BANNED, &rq->context->flags);
d12acee8
CW		 1841 i915_request_get(rq);
1842 i915_request_add(rq);
1843 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
1844 err = -EIO;
1845 goto out;
1846 }
1847
9f3ccd40 1848 intel_context_set_banned(rq->context);
d12acee8
CW		 1849 err = intel_engine_pulse(arg->engine);
1850 if (err)
1851 goto out;
1852
280e285d
CW		 1853 err = wait_for_reset(arg->engine, rq, HZ / 2);
1854 if (err) {
1855 pr_err("Cancelled inflight0 request did not reset\n");
d12acee8
CW		 1856 goto out;
1857 }
1858
1859out:
1860 i915_request_put(rq);
1861 if (igt_live_test_end(&t))
1862 err = -EIO;
1863 return err;
1864}
1865
1866static int __cancel_active1(struct live_preempt_cancel *arg)
1867{
1868 struct i915_request *rq[2] = {};
1869 struct igt_live_test t;
1870 int err;
1871
1872 /* Preempt cancel of ELSP1 */
1873 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1874 if (igt_live_test_begin(&t, arg->engine->i915,
1875 __func__, arg->engine->name))
1876 return -EIO;
1877
d12acee8
CW		 1878 rq[0] = spinner_create_request(&arg->a.spin,
1879 arg->a.ctx, arg->engine,
1880 MI_NOOP); /* no preemption */
1881 if (IS_ERR(rq[0]))
1882 return PTR_ERR(rq[0]);
1883
9f3ccd40 1884 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
d12acee8
CW		 1885 i915_request_get(rq[0]);
1886 i915_request_add(rq[0]);
1887 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1888 err = -EIO;
1889 goto out;
1890 }
1891
d12acee8
CW		 1892 rq[1] = spinner_create_request(&arg->b.spin,
1893 arg->b.ctx, arg->engine,
1894 MI_ARB_CHECK);
1895 if (IS_ERR(rq[1])) {
1896 err = PTR_ERR(rq[1]);
1897 goto out;
1898 }
1899
9f3ccd40 1900 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
d12acee8
CW		 1901 i915_request_get(rq[1]);
1902 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1903 i915_request_add(rq[1]);
1904 if (err)
1905 goto out;
1906
9f3ccd40 1907 intel_context_set_banned(rq[1]->context);
d12acee8
CW		 1908 err = intel_engine_pulse(arg->engine);
1909 if (err)
1910 goto out;
1911
1912 igt_spinner_end(&arg->a.spin);
280e285d
CW		 1913 err = wait_for_reset(arg->engine, rq[1], HZ / 2);
1914 if (err)
d12acee8 1915 goto out;
d12acee8
CW		 1916
1917 if (rq[0]->fence.error != 0) {
1918 pr_err("Normal inflight0 request did not complete\n");
1919 err = -EINVAL;
1920 goto out;
1921 }
1922
1923 if (rq[1]->fence.error != -EIO) {
1924 pr_err("Cancelled inflight1 request did not report -EIO\n");
1925 err = -EINVAL;
1926 goto out;
1927 }
1928
1929out:
1930 i915_request_put(rq[1]);
1931 i915_request_put(rq[0]);
1932 if (igt_live_test_end(&t))
1933 err = -EIO;
1934 return err;
1935}
1936
1937static int __cancel_queued(struct live_preempt_cancel *arg)
1938{
1939 struct i915_request *rq[3] = {};
1940 struct igt_live_test t;
1941 int err;
1942
1943 /* Full ELSP and one in the wings */
1944 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
1945 if (igt_live_test_begin(&t, arg->engine->i915,
1946 __func__, arg->engine->name))
1947 return -EIO;
1948
d12acee8
CW		 1949 rq[0] = spinner_create_request(&arg->a.spin,
1950 arg->a.ctx, arg->engine,
1951 MI_ARB_CHECK);
1952 if (IS_ERR(rq[0]))
1953 return PTR_ERR(rq[0]);
1954
9f3ccd40 1955 clear_bit(CONTEXT_BANNED, &rq[0]->context->flags);
d12acee8
CW		 1956 i915_request_get(rq[0]);
1957 i915_request_add(rq[0]);
1958 if (!igt_wait_for_spinner(&arg->a.spin, rq[0])) {
1959 err = -EIO;
1960 goto out;
1961 }
1962
d12acee8
CW		 1963 rq[1] = igt_request_alloc(arg->b.ctx, arg->engine);
1964 if (IS_ERR(rq[1])) {
1965 err = PTR_ERR(rq[1]);
1966 goto out;
1967 }
1968
9f3ccd40 1969 clear_bit(CONTEXT_BANNED, &rq[1]->context->flags);
d12acee8
CW		 1970 i915_request_get(rq[1]);
1971 err = i915_request_await_dma_fence(rq[1], &rq[0]->fence);
1972 i915_request_add(rq[1]);
1973 if (err)
1974 goto out;
1975
1976 rq[2] = spinner_create_request(&arg->b.spin,
1977 arg->a.ctx, arg->engine,
1978 MI_ARB_CHECK);
1979 if (IS_ERR(rq[2])) {
1980 err = PTR_ERR(rq[2]);
1981 goto out;
1982 }
1983
1984 i915_request_get(rq[2]);
1985 err = i915_request_await_dma_fence(rq[2], &rq[1]->fence);
1986 i915_request_add(rq[2]);
1987 if (err)
1988 goto out;
1989
9f3ccd40 1990 intel_context_set_banned(rq[2]->context);
d12acee8
CW		 1991 err = intel_engine_pulse(arg->engine);
1992 if (err)
1993 goto out;
1994
280e285d
CW		 1995 err = wait_for_reset(arg->engine, rq[2], HZ / 2);
1996 if (err)
d12acee8 1997 goto out;
d12acee8
CW		 1998
1999 if (rq[0]->fence.error != -EIO) {
2000 pr_err("Cancelled inflight0 request did not report -EIO\n");
2001 err = -EINVAL;
2002 goto out;
2003 }
2004
2005 if (rq[1]->fence.error != 0) {
2006 pr_err("Normal inflight1 request did not complete\n");
2007 err = -EINVAL;
2008 goto out;
2009 }
2010
2011 if (rq[2]->fence.error != -EIO) {
2012 pr_err("Cancelled queued request did not report -EIO\n");
2013 err = -EINVAL;
2014 goto out;
2015 }
2016
2017out:
2018 i915_request_put(rq[2]);
2019 i915_request_put(rq[1]);
2020 i915_request_put(rq[0]);
2021 if (igt_live_test_end(&t))
2022 err = -EIO;
2023 return err;
2024}
2025
2026static int __cancel_hostile(struct live_preempt_cancel *arg)
2027{
2028 struct i915_request *rq;
2029 int err;
2030
2031 /* Preempt cancel non-preemptible spinner in ELSP0 */
b79029b2 2032 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
d12acee8
CW		 2033 return 0;
2034
2035 GEM_TRACE("%s(%s)\n", __func__, arg->engine->name);
d12acee8
CW		 2036 rq = spinner_create_request(&arg->a.spin,
2037 arg->a.ctx, arg->engine,
2038 MI_NOOP); /* preemption disabled */
2039 if (IS_ERR(rq))
2040 return PTR_ERR(rq);
2041
9f3ccd40 2042 clear_bit(CONTEXT_BANNED, &rq->context->flags);
d12acee8
CW		 2043 i915_request_get(rq);
2044 i915_request_add(rq);
2045 if (!igt_wait_for_spinner(&arg->a.spin, rq)) {
2046 err = -EIO;
2047 goto out;
2048 }
2049
9f3ccd40 2050 intel_context_set_banned(rq->context);
d12acee8
CW		 2051 err = intel_engine_pulse(arg->engine); /* force reset */
2052 if (err)
2053 goto out;
2054
280e285d
CW		 2055 err = wait_for_reset(arg->engine, rq, HZ / 2);
2056 if (err) {
2057 pr_err("Cancelled inflight0 request did not reset\n");
d12acee8
CW		 2058 goto out;
2059 }
2060
2061out:
2062 i915_request_put(rq);
2063 if (igt_flush_test(arg->engine->i915))
2064 err = -EIO;
2065 return err;
2066}
2067
2068static int live_preempt_cancel(void *arg)
2069{
2070 struct intel_gt *gt = arg;
2071 struct live_preempt_cancel data;
2072 enum intel_engine_id id;
2073 int err = -ENOMEM;
2074
2075 /*
2076 * To cancel an inflight context, we need to first remove it from the
2077 * GPU. That sounds like preemption! Plus a little bit of bookkeeping.
2078 */
2079
2080 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2081 return 0;
2082
2083 if (preempt_client_init(gt, &data.a))
2084 return -ENOMEM;
2085 if (preempt_client_init(gt, &data.b))
2086 goto err_client_a;
2087
2088 for_each_engine(data.engine, gt, id) {
2089 if (!intel_engine_has_preemption(data.engine))
2090 continue;
2091
2092 err = __cancel_active0(&data);
2093 if (err)
2094 goto err_wedged;
2095
2096 err = __cancel_active1(&data);
2097 if (err)
2098 goto err_wedged;
2099
2100 err = __cancel_queued(&data);
2101 if (err)
2102 goto err_wedged;
2103
2104 err = __cancel_hostile(&data);
2105 if (err)
2106 goto err_wedged;
2107 }
2108
2109 err = 0;
2110err_client_b:
2111 preempt_client_fini(&data.b);
2112err_client_a:
2113 preempt_client_fini(&data.a);
2114 return err;
2115
2116err_wedged:
2117 GEM_TRACE_DUMP();
2118 igt_spinner_end(&data.b.spin);
2119 igt_spinner_end(&data.a.spin);
2120 intel_gt_set_wedged(gt);
2121 goto err_client_b;
2122}
2123
c9a64622
CW		 2124static int live_suppress_self_preempt(void *arg)
2125{
1357fa81 2126 struct intel_gt *gt = arg;
c9a64622
CW		 2127 struct intel_engine_cs *engine;
2128 struct i915_sched_attr attr = {
2129 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX)
2130 };
2131 struct preempt_client a, b;
2132 enum intel_engine_id id;
c9a64622
CW		 2133 int err = -ENOMEM;
2134
2135 /*
2136 * Verify that if a preemption request does not cause a change in
2137 * the current execution order, the preempt-to-idle injection is
2138 * skipped and that we do not accidentally apply it after the CS
2139 * completion event.
2140 */
2141
1357fa81 2142 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
c9a64622
CW		 2143 return 0;
2144
065273f7 2145 if (intel_uc_uses_guc_submission(&gt->uc))
c9a64622
CW		 2146 return 0; /* presume black blox */
2147
1357fa81 2148 if (intel_vgpu_active(gt->i915))
506927ec
CW		 2149 return 0; /* GVT forces single port & request submission */
2150
1357fa81 2151 if (preempt_client_init(gt, &a))
2af40298 2152 return -ENOMEM;
1357fa81 2153 if (preempt_client_init(gt, &b))
c9a64622
CW		 2154 goto err_client_a;
2155
5d904e3c 2156 for_each_engine(engine, gt, id) {
c9a64622
CW		 2157 struct i915_request *rq_a, *rq_b;
2158 int depth;
2159
3123ada8
CW		 2160 if (!intel_engine_has_preemption(engine))
2161 continue;
2162
1357fa81 2163 if (igt_flush_test(gt->i915))
0b8d6273
CW		 2164 goto err_wedged;
2165
2166 intel_engine_pm_get(engine);
c9a64622
CW		 2167 engine->execlists.preempt_hang.count = 0;
2168
f277bc0c
CW		 2169 rq_a = spinner_create_request(&a.spin,
2170 a.ctx, engine,
2171 MI_NOOP);
c9a64622
CW		 2172 if (IS_ERR(rq_a)) {
2173 err = PTR_ERR(rq_a);
0b8d6273 2174 intel_engine_pm_put(engine);
c9a64622
CW		 2175 goto err_client_b;
2176 }
2177
2178 i915_request_add(rq_a);
2179 if (!igt_wait_for_spinner(&a.spin, rq_a)) {
2180 pr_err("First client failed to start\n");
0b8d6273 2181 intel_engine_pm_put(engine);
c9a64622
CW		 2182 goto err_wedged;
2183 }
2184
acb9488d
CW		 2185 /* Keep postponing the timer to avoid premature slicing */
2186 mod_timer(&engine->execlists.timer, jiffies + HZ);
c9a64622 2187 for (depth = 0; depth < 8; depth++) {
f277bc0c
CW		 2188 rq_b = spinner_create_request(&b.spin,
2189 b.ctx, engine,
2190 MI_NOOP);
c9a64622
CW		 2191 if (IS_ERR(rq_b)) {
2192 err = PTR_ERR(rq_b);
0b8d6273 2193 intel_engine_pm_put(engine);
c9a64622
CW		 2194 goto err_client_b;
2195 }
2196 i915_request_add(rq_b);
2197
2198 GEM_BUG_ON(i915_request_completed(rq_a));
2199 engine->schedule(rq_a, &attr);
2200 igt_spinner_end(&a.spin);
2201
2202 if (!igt_wait_for_spinner(&b.spin, rq_b)) {
2203 pr_err("Second client failed to start\n");
0b8d6273 2204 intel_engine_pm_put(engine);
c9a64622
CW		 2205 goto err_wedged;
2206 }
2207
2208 swap(a, b);
2209 rq_a = rq_b;
2210 }
2211 igt_spinner_end(&a.spin);
2212
2213 if (engine->execlists.preempt_hang.count) {
acb9488d
CW		 2214 pr_err("Preemption on %s recorded x%d, depth %d; should have been suppressed!\n",
2215 engine->name,
c9a64622
CW		 2216 engine->execlists.preempt_hang.count,
2217 depth);
0b8d6273 2218 intel_engine_pm_put(engine);
c9a64622
CW		 2219 err = -EINVAL;
2220 goto err_client_b;
2221 }
2222
0b8d6273 2223 intel_engine_pm_put(engine);
1357fa81 2224 if (igt_flush_test(gt->i915))
c9a64622
CW		 2225 goto err_wedged;
2226 }
2227
2228 err = 0;
2229err_client_b:
2230 preempt_client_fini(&b);
2231err_client_a:
2232 preempt_client_fini(&a);
c9a64622
CW		 2233 return err;
2234
2235err_wedged:
2236 igt_spinner_end(&b.spin);
2237 igt_spinner_end(&a.spin);
1357fa81 2238 intel_gt_set_wedged(gt);
c9a64622
CW		 2239 err = -EIO;
2240 goto err_client_b;
2241}
2242
b5773a36
CW		 2243static int __i915_sw_fence_call
2244dummy_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
2245{
2246 return NOTIFY_DONE;
2247}
2248
2249static struct i915_request *dummy_request(struct intel_engine_cs *engine)
2250{
2251 struct i915_request *rq;
2252
2253 rq = kzalloc(sizeof(*rq), GFP_KERNEL);
2254 if (!rq)
2255 return NULL;
2256
b5773a36
CW		 2257 rq->engine = engine;
2258
b1e3177b
CW		 2259 spin_lock_init(&rq->lock);
2260 INIT_LIST_HEAD(&rq->fence.cb_list);
2261 rq->fence.lock = &rq->lock;
2262 rq->fence.ops = &i915_fence_ops;
2263
b5773a36
CW		 2264 i915_sched_node_init(&rq->sched);
2265
2266 /* mark this request as permanently incomplete */
2267 rq->fence.seqno = 1;
2268 BUILD_BUG_ON(sizeof(rq->fence.seqno) != 8); /* upper 32b == 0 */
2269 rq->hwsp_seqno = (u32 *)&rq->fence.seqno + 1;
2270 GEM_BUG_ON(i915_request_completed(rq));
2271
2272 i915_sw_fence_init(&rq->submit, dummy_notify);
25d851ad 2273 set_bit(I915_FENCE_FLAG_ACTIVE, &rq->fence.flags);
b5773a36 2274
cbcec57e
CW		 2275 spin_lock_init(&rq->lock);
2276 rq->fence.lock = &rq->lock;
2277 INIT_LIST_HEAD(&rq->fence.cb_list);
2278
b5773a36
CW		 2279 return rq;
2280}
2281
2282static void dummy_request_free(struct i915_request *dummy)
2283{
25d851ad
CW		 2284 /* We have to fake the CS interrupt to kick the next request */
2285 i915_sw_fence_commit(&dummy->submit);
2286
b5773a36 2287 i915_request_mark_complete(dummy);
25d851ad
CW		 2288 dma_fence_signal(&dummy->fence);
2289
b5773a36
CW		 2290 i915_sched_node_fini(&dummy->sched);
2291 i915_sw_fence_fini(&dummy->submit);
2292
2293 dma_fence_free(&dummy->fence);
2294}
2295
2296static int live_suppress_wait_preempt(void *arg)
2297{
1357fa81 2298 struct intel_gt *gt = arg;
b5773a36 2299 struct preempt_client client[4];
1ff2f9e2 2300 struct i915_request *rq[ARRAY_SIZE(client)] = {};
b5773a36
CW		 2301 struct intel_engine_cs *engine;
2302 enum intel_engine_id id;
b5773a36
CW		 2303 int err = -ENOMEM;
2304 int i;
2305
2306 /*
2307 * Waiters are given a little priority nudge, but not enough
2308 * to actually cause any preemption. Double check that we do
2309 * not needlessly generate preempt-to-idle cycles.
2310 */
2311
1357fa81 2312 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
b5773a36
CW		 2313 return 0;
2314
1357fa81 2315 if (preempt_client_init(gt, &client[0])) /* ELSP[0] */
2af40298 2316 return -ENOMEM;
1357fa81 2317 if (preempt_client_init(gt, &client[1])) /* ELSP[1] */
b5773a36 2318 goto err_client_0;
1357fa81 2319 if (preempt_client_init(gt, &client[2])) /* head of queue */
b5773a36 2320 goto err_client_1;
1357fa81 2321 if (preempt_client_init(gt, &client[3])) /* bystander */
b5773a36
CW		 2322 goto err_client_2;
2323
5d904e3c 2324 for_each_engine(engine, gt, id) {
b5773a36
CW		 2325 int depth;
2326
3123ada8
CW		 2327 if (!intel_engine_has_preemption(engine))
2328 continue;
2329
b5773a36
CW		 2330 if (!engine->emit_init_breadcrumb)
2331 continue;
2332
2333 for (depth = 0; depth < ARRAY_SIZE(client); depth++) {
b5773a36
CW		 2334 struct i915_request *dummy;
2335
2336 engine->execlists.preempt_hang.count = 0;
2337
2338 dummy = dummy_request(engine);
2339 if (!dummy)
2340 goto err_client_3;
2341
2342 for (i = 0; i < ARRAY_SIZE(client); i++) {
1ff2f9e2
CW		 2343 struct i915_request *this;
2344
2345 this = spinner_create_request(&client[i].spin,
2346 client[i].ctx, engine,
2347 MI_NOOP);
2348 if (IS_ERR(this)) {
2349 err = PTR_ERR(this);
b5773a36
CW		 2350 goto err_wedged;
2351 }
2352
2353 /* Disable NEWCLIENT promotion */
1ff2f9e2 2354 __i915_active_fence_set(&i915_request_timeline(this)->last_request,
b1e3177b 2355 &dummy->fence);
1ff2f9e2
CW		 2356
2357 rq[i] = i915_request_get(this);
2358 i915_request_add(this);
b5773a36
CW		 2359 }
2360
2361 dummy_request_free(dummy);
2362
2363 GEM_BUG_ON(i915_request_completed(rq[0]));
2364 if (!igt_wait_for_spinner(&client[0].spin, rq[0])) {
2365 pr_err("%s: First client failed to start\n",
2366 engine->name);
2367 goto err_wedged;
2368 }
2369 GEM_BUG_ON(!i915_request_started(rq[0]));
2370
2371 if (i915_request_wait(rq[depth],
b5773a36
CW		 2372 I915_WAIT_PRIORITY,
2373 1) != -ETIME) {
2374 pr_err("%s: Waiter depth:%d completed!\n",
2375 engine->name, depth);
2376 goto err_wedged;
2377 }
2378
1ff2f9e2 2379 for (i = 0; i < ARRAY_SIZE(client); i++) {
b5773a36 2380 igt_spinner_end(&client[i].spin);
1ff2f9e2
CW		 2381 i915_request_put(rq[i]);
2382 rq[i] = NULL;
2383 }
b5773a36 2384
1357fa81 2385 if (igt_flush_test(gt->i915))
b5773a36
CW		 2386 goto err_wedged;
2387
2388 if (engine->execlists.preempt_hang.count) {
2389 pr_err("%s: Preemption recorded x%d, depth %d; should have been suppressed!\n",
2390 engine->name,
2391 engine->execlists.preempt_hang.count,
2392 depth);
2393 err = -EINVAL;
2394 goto err_client_3;
2395 }
2396 }
2397 }
2398
2399 err = 0;
2400err_client_3:
2401 preempt_client_fini(&client[3]);
2402err_client_2:
2403 preempt_client_fini(&client[2]);
2404err_client_1:
2405 preempt_client_fini(&client[1]);
2406err_client_0:
2407 preempt_client_fini(&client[0]);
b5773a36
CW		 2408 return err;
2409
2410err_wedged:
1ff2f9e2 2411 for (i = 0; i < ARRAY_SIZE(client); i++) {
b5773a36 2412 igt_spinner_end(&client[i].spin);
1ff2f9e2
CW		 2413 i915_request_put(rq[i]);
2414 }
1357fa81 2415 intel_gt_set_wedged(gt);
b5773a36
CW		 2416 err = -EIO;
2417 goto err_client_3;
2418}
2419
a21f453c
CW		 2420static int live_chain_preempt(void *arg)
2421{
1357fa81 2422 struct intel_gt *gt = arg;
a21f453c
CW		 2423 struct intel_engine_cs *engine;
2424 struct preempt_client hi, lo;
2425 enum intel_engine_id id;
a21f453c
CW		 2426 int err = -ENOMEM;
2427
2428 /*
2429 * Build a chain AB...BA between two contexts (A, B) and request
2430 * preemption of the last request. It should then complete before
2431 * the previously submitted spinner in B.
2432 */
2433
1357fa81 2434 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
a21f453c
CW		 2435 return 0;
2436
1357fa81 2437 if (preempt_client_init(gt, &hi))
2af40298 2438 return -ENOMEM;
a21f453c 2439
1357fa81 2440 if (preempt_client_init(gt, &lo))
a21f453c
CW		 2441 goto err_client_hi;
2442
5d904e3c 2443 for_each_engine(engine, gt, id) {
a21f453c
CW		 2444 struct i915_sched_attr attr = {
2445 .priority = I915_USER_PRIORITY(I915_PRIORITY_MAX),
2446 };
e70d3d80 2447 struct igt_live_test t;
d067994c
CW		 2448 struct i915_request *rq;
2449 int ring_size, count, i;
a21f453c 2450
3123ada8
CW		 2451 if (!intel_engine_has_preemption(engine))
2452 continue;
2453
f277bc0c
CW		 2454 rq = spinner_create_request(&lo.spin,
2455 lo.ctx, engine,
2456 MI_ARB_CHECK);
d067994c
CW		 2457 if (IS_ERR(rq))
2458 goto err_wedged;
1ff2f9e2
CW		 2459
2460 i915_request_get(rq);
d067994c
CW		 2461 i915_request_add(rq);
2462
2463 ring_size = rq->wa_tail - rq->head;
2464 if (ring_size < 0)
2465 ring_size += rq->ring->size;
2466 ring_size = rq->ring->size / ring_size;
2467 pr_debug("%s(%s): Using maximum of %d requests\n",
2468 __func__, engine->name, ring_size);
a21f453c 2469
d067994c 2470 igt_spinner_end(&lo.spin);
2f530945 2471 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
d067994c 2472 pr_err("Timed out waiting to flush %s\n", engine->name);
1ff2f9e2 2473 i915_request_put(rq);
d067994c
CW		 2474 goto err_wedged;
2475 }
1ff2f9e2 2476 i915_request_put(rq);
d067994c 2477
1357fa81 2478 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name)) {
e70d3d80
CW		 2479 err = -EIO;
2480 goto err_wedged;
2481 }
2482
d067994c 2483 for_each_prime_number_from(count, 1, ring_size) {
f277bc0c
CW		 2484 rq = spinner_create_request(&hi.spin,
2485 hi.ctx, engine,
2486 MI_ARB_CHECK);
a21f453c
CW		 2487 if (IS_ERR(rq))
2488 goto err_wedged;
2489 i915_request_add(rq);
2490 if (!igt_wait_for_spinner(&hi.spin, rq))
2491 goto err_wedged;
2492
f277bc0c
CW		 2493 rq = spinner_create_request(&lo.spin,
2494 lo.ctx, engine,
2495 MI_ARB_CHECK);
a21f453c
CW		 2496 if (IS_ERR(rq))
2497 goto err_wedged;
2498 i915_request_add(rq);
2499
2500 for (i = 0; i < count; i++) {
46472b3e 2501 rq = igt_request_alloc(lo.ctx, engine);
a21f453c
CW		 2502 if (IS_ERR(rq))
2503 goto err_wedged;
2504 i915_request_add(rq);
2505 }
2506
46472b3e 2507 rq = igt_request_alloc(hi.ctx, engine);
a21f453c
CW		 2508 if (IS_ERR(rq))
2509 goto err_wedged;
1ff2f9e2
CW		 2510
2511 i915_request_get(rq);
a21f453c
CW		 2512 i915_request_add(rq);
2513 engine->schedule(rq, &attr);
2514
2515 igt_spinner_end(&hi.spin);
2f530945 2516 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
a21f453c 2517 struct drm_printer p =
1357fa81 2518 drm_info_printer(gt->i915->drm.dev);
a21f453c
CW		 2519
2520 pr_err("Failed to preempt over chain of %d\n",
2521 count);
2522 intel_engine_dump(engine, &p,
2523 "%s\n", engine->name);
1ff2f9e2 2524 i915_request_put(rq);
a21f453c
CW		 2525 goto err_wedged;
2526 }
2527 igt_spinner_end(&lo.spin);
1ff2f9e2 2528 i915_request_put(rq);
d067994c 2529
46472b3e 2530 rq = igt_request_alloc(lo.ctx, engine);
d067994c
CW		 2531 if (IS_ERR(rq))
2532 goto err_wedged;
1ff2f9e2
CW		 2533
2534 i915_request_get(rq);
d067994c 2535 i915_request_add(rq);
1ff2f9e2 2536
2f530945 2537 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
d067994c 2538 struct drm_printer p =
1357fa81 2539 drm_info_printer(gt->i915->drm.dev);
d067994c
CW		 2540
2541 pr_err("Failed to flush low priority chain of %d requests\n",
2542 count);
2543 intel_engine_dump(engine, &p,
2544 "%s\n", engine->name);
1ff2f9e2
CW		 2545
2546 i915_request_put(rq);
d067994c
CW		 2547 goto err_wedged;
2548 }
1ff2f9e2 2549 i915_request_put(rq);
a21f453c 2550 }
e70d3d80
CW		 2551
2552 if (igt_live_test_end(&t)) {
2553 err = -EIO;
2554 goto err_wedged;
2555 }
a21f453c
CW		 2556 }
2557
2558 err = 0;
2559err_client_lo:
2560 preempt_client_fini(&lo);
2561err_client_hi:
2562 preempt_client_fini(&hi);
a21f453c
CW		 2563 return err;
2564
2565err_wedged:
2566 igt_spinner_end(&hi.spin);
2567 igt_spinner_end(&lo.spin);
1357fa81 2568 intel_gt_set_wedged(gt);
a21f453c
CW		 2569 err = -EIO;
2570 goto err_client_lo;
2571}
2572
2d19a71c
CW		 2573static int create_gang(struct intel_engine_cs *engine,
2574 struct i915_request **prev)
2575{
2576 struct drm_i915_gem_object *obj;
2577 struct intel_context *ce;
2578 struct i915_request *rq;
2579 struct i915_vma *vma;
2580 u32 *cs;
2581 int err;
2582
e6ba7648 2583 ce = intel_context_create(engine);
2d19a71c
CW		 2584 if (IS_ERR(ce))
2585 return PTR_ERR(ce);
2586
2587 obj = i915_gem_object_create_internal(engine->i915, 4096);
2588 if (IS_ERR(obj)) {
2589 err = PTR_ERR(obj);
2590 goto err_ce;
2591 }
2592
2593 vma = i915_vma_instance(obj, ce->vm, NULL);
2594 if (IS_ERR(vma)) {
2595 err = PTR_ERR(vma);
2596 goto err_obj;
2597 }
2598
2599 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2600 if (err)
2601 goto err_obj;
2602
2603 cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2604 if (IS_ERR(cs))
2605 goto err_obj;
2606
2607 /* Semaphore target: spin until zero */
2608 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
2609
2610 *cs++ = MI_SEMAPHORE_WAIT |
2611 MI_SEMAPHORE_POLL |
2612 MI_SEMAPHORE_SAD_EQ_SDD;
2613 *cs++ = 0;
2614 *cs++ = lower_32_bits(vma->node.start);
2615 *cs++ = upper_32_bits(vma->node.start);
2616
2617 if (*prev) {
2618 u64 offset = (*prev)->batch->node.start;
2619
2620 /* Terminate the spinner in the next lower priority batch. */
2621 *cs++ = MI_STORE_DWORD_IMM_GEN4;
2622 *cs++ = lower_32_bits(offset);
2623 *cs++ = upper_32_bits(offset);
2624 *cs++ = 0;
2625 }
2626
2627 *cs++ = MI_BATCH_BUFFER_END;
2628 i915_gem_object_flush_map(obj);
2629 i915_gem_object_unpin_map(obj);
2630
2631 rq = intel_context_create_request(ce);
2632 if (IS_ERR(rq))
2633 goto err_obj;
2634
2635 rq->batch = vma;
2636 i915_request_get(rq);
2637
2638 i915_vma_lock(vma);
2639 err = i915_request_await_object(rq, vma->obj, false);
2640 if (!err)
2641 err = i915_vma_move_to_active(vma, rq, 0);
2642 if (!err)
2643 err = rq->engine->emit_bb_start(rq,
2644 vma->node.start,
2645 PAGE_SIZE, 0);
2646 i915_vma_unlock(vma);
2647 i915_request_add(rq);
2648 if (err)
2649 goto err_rq;
2650
2651 i915_gem_object_put(obj);
2652 intel_context_put(ce);
2653
2654 rq->client_link.next = &(*prev)->client_link;
2655 *prev = rq;
2656 return 0;
2657
2658err_rq:
2659 i915_request_put(rq);
2660err_obj:
2661 i915_gem_object_put(obj);
2662err_ce:
2663 intel_context_put(ce);
2664 return err;
2665}
2666
2667static int live_preempt_gang(void *arg)
2668{
2669 struct intel_gt *gt = arg;
2670 struct intel_engine_cs *engine;
2671 enum intel_engine_id id;
2672
2673 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2674 return 0;
2675
2676 /*
2677 * Build as long a chain of preempters as we can, with each
2678 * request higher priority than the last. Once we are ready, we release
2679 * the last batch which then precolates down the chain, each releasing
2680 * the next oldest in turn. The intent is to simply push as hard as we
2681 * can with the number of preemptions, trying to exceed narrow HW
2682 * limits. At a minimum, we insist that we can sort all the user
2683 * high priority levels into execution order.
2684 */
2685
2686 for_each_engine(engine, gt, id) {
2687 struct i915_request *rq = NULL;
2688 struct igt_live_test t;
2689 IGT_TIMEOUT(end_time);
2690 int prio = 0;
2691 int err = 0;
2692 u32 *cs;
2693
2694 if (!intel_engine_has_preemption(engine))
2695 continue;
2696
2697 if (igt_live_test_begin(&t, gt->i915, __func__, engine->name))
2698 return -EIO;
2699
2700 do {
2701 struct i915_sched_attr attr = {
2702 .priority = I915_USER_PRIORITY(prio++),
2703 };
2704
2705 err = create_gang(engine, &rq);
2706 if (err)
2707 break;
2708
2709 /* Submit each spinner at increasing priority */
2710 engine->schedule(rq, &attr);
2711
2712 if (prio <= I915_PRIORITY_MAX)
2713 continue;
2714
2715 if (prio > (INT_MAX >> I915_USER_PRIORITY_SHIFT))
2716 break;
2717
2718 if (__igt_timeout(end_time, NULL))
2719 break;
2720 } while (1);
2721 pr_debug("%s: Preempt chain of %d requests\n",
2722 engine->name, prio);
2723
2724 /*
2725 * Such that the last spinner is the highest priority and
2726 * should execute first. When that spinner completes,
2727 * it will terminate the next lowest spinner until there
2728 * are no more spinners and the gang is complete.
2729 */
2730 cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
2731 if (!IS_ERR(cs)) {
2732 *cs = 0;
2733 i915_gem_object_unpin_map(rq->batch->obj);
2734 } else {
2735 err = PTR_ERR(cs);
2736 intel_gt_set_wedged(gt);
2737 }
2738
2739 while (rq) { /* wait for each rq from highest to lowest prio */
2740 struct i915_request *n =
2741 list_next_entry(rq, client_link);
2742
2743 if (err == 0 && i915_request_wait(rq, 0, HZ / 5) < 0) {
2744 struct drm_printer p =
2745 drm_info_printer(engine->i915->drm.dev);
2746
2747 pr_err("Failed to flush chain of %d requests, at %d\n",
2748 prio, rq_prio(rq) >> I915_USER_PRIORITY_SHIFT);
2749 intel_engine_dump(engine, &p,
2750 "%s\n", engine->name);
2751
2752 err = -ETIME;
2753 }
2754
2755 i915_request_put(rq);
2756 rq = n;
2757 }
2758
2759 if (igt_live_test_end(&t))
2760 err = -EIO;
2761 if (err)
2762 return err;
2763 }
2764
2765 return 0;
2766}
2767
3a7a92ab
CW		 2768static int live_preempt_timeout(void *arg)
2769{
2770 struct intel_gt *gt = arg;
2771 struct i915_gem_context *ctx_hi, *ctx_lo;
2772 struct igt_spinner spin_lo;
2773 struct intel_engine_cs *engine;
2774 enum intel_engine_id id;
2775 int err = -ENOMEM;
2776
2777 /*
2778 * Check that we force preemption to occur by cancelling the previous
2779 * context if it refuses to yield the GPU.
2780 */
b79029b2 2781 if (!IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT))
3a7a92ab
CW		 2782 return 0;
2783
2784 if (!HAS_LOGICAL_RING_PREEMPTION(gt->i915))
2785 return 0;
2786
2787 if (!intel_has_reset_engine(gt))
2788 return 0;
2789
2790 if (igt_spinner_init(&spin_lo, gt))
2791 return -ENOMEM;
2792
2793 ctx_hi = kernel_context(gt->i915);
2794 if (!ctx_hi)
2795 goto err_spin_lo;
2796 ctx_hi->sched.priority =
2797 I915_USER_PRIORITY(I915_CONTEXT_MAX_USER_PRIORITY);
2798
2799 ctx_lo = kernel_context(gt->i915);
2800 if (!ctx_lo)
2801 goto err_ctx_hi;
2802 ctx_lo->sched.priority =
2803 I915_USER_PRIORITY(I915_CONTEXT_MIN_USER_PRIORITY);
2804
2805 for_each_engine(engine, gt, id) {
2806 unsigned long saved_timeout;
2807 struct i915_request *rq;
2808
2809 if (!intel_engine_has_preemption(engine))
2810 continue;
2811
2812 rq = spinner_create_request(&spin_lo, ctx_lo, engine,
2813 MI_NOOP); /* preemption disabled */
2814 if (IS_ERR(rq)) {
2815 err = PTR_ERR(rq);
2816 goto err_ctx_lo;
2817 }
2818
2819 i915_request_add(rq);
2820 if (!igt_wait_for_spinner(&spin_lo, rq)) {
2821 intel_gt_set_wedged(gt);
2822 err = -EIO;
2823 goto err_ctx_lo;
2824 }
2825
2826 rq = igt_request_alloc(ctx_hi, engine);
2827 if (IS_ERR(rq)) {
2828 igt_spinner_end(&spin_lo);
2829 err = PTR_ERR(rq);
2830 goto err_ctx_lo;
2831 }
2832
2833 /* Flush the previous CS ack before changing timeouts */
2834 while (READ_ONCE(engine->execlists.pending[0]))
2835 cpu_relax();
2836
2837 saved_timeout = engine->props.preempt_timeout_ms;
2838 engine->props.preempt_timeout_ms = 1; /* in ms, -> 1 jiffie */
2839
2840 i915_request_get(rq);
2841 i915_request_add(rq);
2842
2843 intel_engine_flush_submission(engine);
2844 engine->props.preempt_timeout_ms = saved_timeout;
2845
2846 if (i915_request_wait(rq, 0, HZ / 10) < 0) {
2847 intel_gt_set_wedged(gt);
2848 i915_request_put(rq);
2849 err = -ETIME;
2850 goto err_ctx_lo;
2851 }
2852
2853 igt_spinner_end(&spin_lo);
2854 i915_request_put(rq);
2855 }
2856
2857 err = 0;
2858err_ctx_lo:
2859 kernel_context_close(ctx_lo);
2860err_ctx_hi:
2861 kernel_context_close(ctx_hi);
2862err_spin_lo:
2863 igt_spinner_fini(&spin_lo);
2864 return err;
2865}
2866
dee4a0f8
CW		 2867static int random_range(struct rnd_state *rnd, int min, int max)
2868{
2869 return i915_prandom_u32_max_state(max - min, rnd) + min;
2870}
2871
2872static int random_priority(struct rnd_state *rnd)
2873{
2874 return random_range(rnd, I915_PRIORITY_MIN, I915_PRIORITY_MAX);
2875}
2876
2877struct preempt_smoke {
1357fa81 2878 struct intel_gt *gt;
dee4a0f8 2879 struct i915_gem_context **contexts;
992d2098 2880 struct intel_engine_cs *engine;
567a6057 2881 struct drm_i915_gem_object *batch;
dee4a0f8
CW		 2882 unsigned int ncontext;
2883 struct rnd_state prng;
992d2098 2884 unsigned long count;
dee4a0f8
CW		 2885};
2886
2887static struct i915_gem_context *smoke_context(struct preempt_smoke *smoke)
2888{
2889 return smoke->contexts[i915_prandom_u32_max_state(smoke->ncontext,
2890 &smoke->prng)];
2891}
2892
567a6057
CW		 2893static int smoke_submit(struct preempt_smoke *smoke,
2894 struct i915_gem_context *ctx, int prio,
2895 struct drm_i915_gem_object *batch)
2896{
2897 struct i915_request *rq;
2898 struct i915_vma *vma = NULL;
2899 int err = 0;
2900
2901 if (batch) {
a4e7ccda
CW		 2902 struct i915_address_space *vm;
2903
2904 vm = i915_gem_context_get_vm_rcu(ctx);
2905 vma = i915_vma_instance(batch, vm, NULL);
2906 i915_vm_put(vm);
567a6057
CW		 2907 if (IS_ERR(vma))
2908 return PTR_ERR(vma);
2909
2910 err = i915_vma_pin(vma, 0, 0, PIN_USER);
2911 if (err)
2912 return err;
2913 }
2914
2915 ctx->sched.priority = prio;
2916
46472b3e 2917 rq = igt_request_alloc(ctx, smoke->engine);
567a6057
CW		 2918 if (IS_ERR(rq)) {
2919 err = PTR_ERR(rq);
2920 goto unpin;
2921 }
2922
2923 if (vma) {
6951e589 2924 i915_vma_lock(vma);
70d6894d 2925 err = i915_request_await_object(rq, vma->obj, false);
567a6057
CW		 2926 if (!err)
2927 err = i915_vma_move_to_active(vma, rq, 0);
70d6894d
CW		 2928 if (!err)
2929 err = rq->engine->emit_bb_start(rq,
2930 vma->node.start,
2931 PAGE_SIZE, 0);
6951e589 2932 i915_vma_unlock(vma);
567a6057
CW		 2933 }
2934
2935 i915_request_add(rq);
2936
2937unpin:
2938 if (vma)
2939 i915_vma_unpin(vma);
2940
2941 return err;
2942}
2943
992d2098
CW		 2944static int smoke_crescendo_thread(void *arg)
2945{
2946 struct preempt_smoke *smoke = arg;
2947 IGT_TIMEOUT(end_time);
2948 unsigned long count;
2949
2950 count = 0;
2951 do {
2952 struct i915_gem_context *ctx = smoke_context(smoke);
567a6057 2953 int err;
992d2098 2954
567a6057
CW		 2955 err = smoke_submit(smoke,
2956 ctx, count % I915_PRIORITY_MAX,
2957 smoke->batch);
567a6057
CW		 2958 if (err)
2959 return err;
992d2098
CW		 2960
2961 count++;
2962 } while (!__igt_timeout(end_time, NULL));
2963
2964 smoke->count = count;
2965 return 0;
2966}
2967
567a6057
CW		 2968static int smoke_crescendo(struct preempt_smoke *smoke, unsigned int flags)
2969#define BATCH BIT(0)
dee4a0f8 2970{
992d2098
CW		 2971 struct task_struct *tsk[I915_NUM_ENGINES] = {};
2972 struct preempt_smoke arg[I915_NUM_ENGINES];
dee4a0f8
CW		 2973 struct intel_engine_cs *engine;
2974 enum intel_engine_id id;
2975 unsigned long count;
992d2098
CW		 2976 int err = 0;
2977
5d904e3c 2978 for_each_engine(engine, smoke->gt, id) {
992d2098
CW		 2979 arg[id] = *smoke;
2980 arg[id].engine = engine;
567a6057
CW		 2981 if (!(flags & BATCH))
2982 arg[id].batch = NULL;
992d2098
CW		 2983 arg[id].count = 0;
2984
2985 tsk[id] = kthread_run(smoke_crescendo_thread, &arg,
2986 "igt/smoke:%d", id);
2987 if (IS_ERR(tsk[id])) {
2988 err = PTR_ERR(tsk[id]);
2989 break;
2990 }
5ec244f4 2991 get_task_struct(tsk[id]);
992d2098 2992 }
dee4a0f8 2993
e5661c6a
CW		 2994 yield(); /* start all threads before we kthread_stop() */
2995
992d2098 2996 count = 0;
5d904e3c 2997 for_each_engine(engine, smoke->gt, id) {
992d2098 2998 int status;
dee4a0f8 2999
992d2098
CW		 3000 if (IS_ERR_OR_NULL(tsk[id]))
3001 continue;
dee4a0f8 3002
992d2098
CW		 3003 status = kthread_stop(tsk[id]);
3004 if (status && !err)
3005 err = status;
dee4a0f8 3006
992d2098 3007 count += arg[id].count;
5ec244f4
CW		 3008
3009 put_task_struct(tsk[id]);
dee4a0f8
CW		 3010 }
3011
567a6057
CW		 3012 pr_info("Submitted %lu crescendo:%x requests across %d engines and %d contexts\n",
3013 count, flags,
1357fa81 3014 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
dee4a0f8
CW		 3015 return 0;
3016}
3017
567a6057 3018static int smoke_random(struct preempt_smoke *smoke, unsigned int flags)
dee4a0f8 3019{
dee4a0f8
CW		 3020 enum intel_engine_id id;
3021 IGT_TIMEOUT(end_time);
3022 unsigned long count;
3023
3024 count = 0;
3025 do {
5d904e3c 3026 for_each_engine(smoke->engine, smoke->gt, id) {
dee4a0f8 3027 struct i915_gem_context *ctx = smoke_context(smoke);
567a6057 3028 int err;
dee4a0f8 3029
567a6057
CW		 3030 err = smoke_submit(smoke,
3031 ctx, random_priority(&smoke->prng),
3032 flags & BATCH ? smoke->batch : NULL);
3033 if (err)
3034 return err;
dee4a0f8 3035
dee4a0f8
CW		 3036 count++;
3037 }
3038 } while (!__igt_timeout(end_time, NULL));
3039
567a6057
CW		 3040 pr_info("Submitted %lu random:%x requests across %d engines and %d contexts\n",
3041 count, flags,
1357fa81 3042 RUNTIME_INFO(smoke->gt->i915)->num_engines, smoke->ncontext);
dee4a0f8
CW		 3043 return 0;
3044}
3045
3046static int live_preempt_smoke(void *arg)
3047{
3048 struct preempt_smoke smoke = {
1357fa81 3049 .gt = arg,
dee4a0f8
CW		 3050 .prng = I915_RND_STATE_INITIALIZER(i915_selftest.random_seed),
3051 .ncontext = 1024,
3052 };
567a6057 3053 const unsigned int phase[] = { 0, BATCH };
e70d3d80 3054 struct igt_live_test t;
dee4a0f8 3055 int err = -ENOMEM;
567a6057 3056 u32 *cs;
dee4a0f8
CW		 3057 int n;
3058
1357fa81 3059 if (!HAS_LOGICAL_RING_PREEMPTION(smoke.gt->i915))
dee4a0f8
CW		 3060 return 0;
3061
3062 smoke.contexts = kmalloc_array(smoke.ncontext,
3063 sizeof(*smoke.contexts),
3064 GFP_KERNEL);
3065 if (!smoke.contexts)
3066 return -ENOMEM;
3067
1357fa81
CW		 3068 smoke.batch =
3069 i915_gem_object_create_internal(smoke.gt->i915, PAGE_SIZE);
567a6057
CW		 3070 if (IS_ERR(smoke.batch)) {
3071 err = PTR_ERR(smoke.batch);
2af40298 3072 goto err_free;
567a6057
CW		 3073 }
3074
3075 cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
3076 if (IS_ERR(cs)) {
3077 err = PTR_ERR(cs);
3078 goto err_batch;
3079 }
3080 for (n = 0; n < PAGE_SIZE / sizeof(*cs) - 1; n++)
3081 cs[n] = MI_ARB_CHECK;
3082 cs[n] = MI_BATCH_BUFFER_END;
a679f58d 3083 i915_gem_object_flush_map(smoke.batch);
567a6057
CW		 3084 i915_gem_object_unpin_map(smoke.batch);
3085
1357fa81 3086 if (igt_live_test_begin(&t, smoke.gt->i915, __func__, "all")) {
e70d3d80
CW		 3087 err = -EIO;
3088 goto err_batch;
3089 }
3090
dee4a0f8 3091 for (n = 0; n < smoke.ncontext; n++) {
1357fa81 3092 smoke.contexts[n] = kernel_context(smoke.gt->i915);
dee4a0f8
CW		 3093 if (!smoke.contexts[n])
3094 goto err_ctx;
3095 }
3096
567a6057
CW		 3097 for (n = 0; n < ARRAY_SIZE(phase); n++) {
3098 err = smoke_crescendo(&smoke, phase[n]);
3099 if (err)
3100 goto err_ctx;
dee4a0f8 3101
567a6057
CW		 3102 err = smoke_random(&smoke, phase[n]);
3103 if (err)
3104 goto err_ctx;
3105 }
dee4a0f8
CW		 3106
3107err_ctx:
e70d3d80 3108 if (igt_live_test_end(&t))
dee4a0f8
CW		 3109 err = -EIO;
3110
3111 for (n = 0; n < smoke.ncontext; n++) {
3112 if (!smoke.contexts[n])
3113 break;
3114 kernel_context_close(smoke.contexts[n]);
3115 }
3116
567a6057
CW		 3117err_batch:
3118 i915_gem_object_put(smoke.batch);
2af40298 3119err_free:
dee4a0f8
CW		 3120 kfree(smoke.contexts);
3121
3122 return err;
3123}
3124
1357fa81 3125static int nop_virtual_engine(struct intel_gt *gt,
6d06779e
CW		 3126 struct intel_engine_cs **siblings,
3127 unsigned int nsibling,
3128 unsigned int nctx,
3129 unsigned int flags)
3130#define CHAIN BIT(0)
3131{
3132 IGT_TIMEOUT(end_time);
1ff2f9e2 3133 struct i915_request *request[16] = {};
6d06779e
CW		 3134 struct intel_context *ve[16];
3135 unsigned long n, prime, nc;
3136 struct igt_live_test t;
3137 ktime_t times[2] = {};
3138 int err;
3139
e6ba7648 3140 GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ve));
6d06779e
CW		 3141
3142 for (n = 0; n < nctx; n++) {
e6ba7648 3143 ve[n] = intel_execlists_create_virtual(siblings, nsibling);
6d06779e 3144 if (IS_ERR(ve[n])) {
6d06779e
CW		 3145 err = PTR_ERR(ve[n]);
3146 nctx = n;
3147 goto out;
3148 }
3149
3150 err = intel_context_pin(ve[n]);
3151 if (err) {
3152 intel_context_put(ve[n]);
6d06779e
CW		 3153 nctx = n;
3154 goto out;
3155 }
3156 }
3157
1357fa81 3158 err = igt_live_test_begin(&t, gt->i915, __func__, ve[0]->engine->name);
6d06779e
CW		 3159 if (err)
3160 goto out;
3161
3162 for_each_prime_number_from(prime, 1, 8192) {
3163 times[1] = ktime_get_raw();
3164
3165 if (flags & CHAIN) {
3166 for (nc = 0; nc < nctx; nc++) {
3167 for (n = 0; n < prime; n++) {
1ff2f9e2
CW		 3168 struct i915_request *rq;
3169
3170 rq = i915_request_create(ve[nc]);
3171 if (IS_ERR(rq)) {
3172 err = PTR_ERR(rq);
6d06779e
CW		 3173 goto out;
3174 }
3175
1ff2f9e2
CW		 3176 if (request[nc])
3177 i915_request_put(request[nc]);
3178 request[nc] = i915_request_get(rq);
3179 i915_request_add(rq);
6d06779e
CW		 3180 }
3181 }
3182 } else {
3183 for (n = 0; n < prime; n++) {
3184 for (nc = 0; nc < nctx; nc++) {
1ff2f9e2
CW		 3185 struct i915_request *rq;
3186
3187 rq = i915_request_create(ve[nc]);
3188 if (IS_ERR(rq)) {
3189 err = PTR_ERR(rq);
6d06779e
CW		 3190 goto out;
3191 }
3192
1ff2f9e2
CW		 3193 if (request[nc])
3194 i915_request_put(request[nc]);
3195 request[nc] = i915_request_get(rq);
3196 i915_request_add(rq);
6d06779e
CW		 3197 }
3198 }
3199 }
3200
3201 for (nc = 0; nc < nctx; nc++) {
2f530945 3202 if (i915_request_wait(request[nc], 0, HZ / 10) < 0) {
6d06779e
CW		 3203 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3204 __func__, ve[0]->engine->name,
3205 request[nc]->fence.context,
3206 request[nc]->fence.seqno);
3207
3208 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3209 __func__, ve[0]->engine->name,
3210 request[nc]->fence.context,
3211 request[nc]->fence.seqno);
3212 GEM_TRACE_DUMP();
1357fa81 3213 intel_gt_set_wedged(gt);
6d06779e
CW		 3214 break;
3215 }
3216 }
3217
3218 times[1] = ktime_sub(ktime_get_raw(), times[1]);
3219 if (prime == 1)
3220 times[0] = times[1];
3221
1ff2f9e2
CW		 3222 for (nc = 0; nc < nctx; nc++) {
3223 i915_request_put(request[nc]);
3224 request[nc] = NULL;
3225 }
3226
6d06779e
CW		 3227 if (__igt_timeout(end_time, NULL))
3228 break;
3229 }
3230
3231 err = igt_live_test_end(&t);
3232 if (err)
3233 goto out;
3234
3235 pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n",
3236 nctx, ve[0]->engine->name, ktime_to_ns(times[0]),
3237 prime, div64_u64(ktime_to_ns(times[1]), prime));
3238
3239out:
1357fa81 3240 if (igt_flush_test(gt->i915))
6d06779e
CW		 3241 err = -EIO;
3242
3243 for (nc = 0; nc < nctx; nc++) {
1ff2f9e2 3244 i915_request_put(request[nc]);
6d06779e
CW		 3245 intel_context_unpin(ve[nc]);
3246 intel_context_put(ve[nc]);
6d06779e
CW		 3247 }
3248 return err;
3249}
3250
3251static int live_virtual_engine(void *arg)
3252{
1357fa81 3253 struct intel_gt *gt = arg;
6d06779e
CW		 3254 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3255 struct intel_engine_cs *engine;
3256 enum intel_engine_id id;
3257 unsigned int class, inst;
2af40298 3258 int err;
6d06779e 3259
065273f7 3260 if (intel_uc_uses_guc_submission(&gt->uc))
6d06779e
CW		 3261 return 0;
3262
5d904e3c 3263 for_each_engine(engine, gt, id) {
1357fa81 3264 err = nop_virtual_engine(gt, &engine, 1, 1, 0);
6d06779e
CW		 3265 if (err) {
3266 pr_err("Failed to wrap engine %s: err=%d\n",
3267 engine->name, err);
2af40298 3268 return err;
6d06779e
CW		 3269 }
3270 }
3271
3272 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3273 int nsibling, n;
3274
3275 nsibling = 0;
3276 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
750e76b4 3277 if (!gt->engine_class[class][inst])
6d06779e
CW		 3278 continue;
3279
750e76b4 3280 siblings[nsibling++] = gt->engine_class[class][inst];
6d06779e
CW		 3281 }
3282 if (nsibling < 2)
3283 continue;
3284
3285 for (n = 1; n <= nsibling + 1; n++) {
1357fa81 3286 err = nop_virtual_engine(gt, siblings, nsibling,
6d06779e
CW		 3287 n, 0);
3288 if (err)
2af40298 3289 return err;
6d06779e
CW		 3290 }
3291
1357fa81 3292 err = nop_virtual_engine(gt, siblings, nsibling, n, CHAIN);
6d06779e 3293 if (err)
2af40298 3294 return err;
6d06779e
CW		 3295 }
3296
2af40298 3297 return 0;
6d06779e
CW		 3298}
3299
1357fa81 3300static int mask_virtual_engine(struct intel_gt *gt,
78e41ddd
CW		 3301 struct intel_engine_cs **siblings,
3302 unsigned int nsibling)
3303{
3304 struct i915_request *request[MAX_ENGINE_INSTANCE + 1];
78e41ddd
CW		 3305 struct intel_context *ve;
3306 struct igt_live_test t;
3307 unsigned int n;
3308 int err;
3309
3310 /*
3311 * Check that by setting the execution mask on a request, we can
3312 * restrict it to our desired engine within the virtual engine.
3313 */
3314
e6ba7648 3315 ve = intel_execlists_create_virtual(siblings, nsibling);
78e41ddd
CW		 3316 if (IS_ERR(ve)) {
3317 err = PTR_ERR(ve);
3318 goto out_close;
3319 }
3320
3321 err = intel_context_pin(ve);
3322 if (err)
3323 goto out_put;
3324
1357fa81 3325 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
78e41ddd
CW		 3326 if (err)
3327 goto out_unpin;
3328
3329 for (n = 0; n < nsibling; n++) {
3330 request[n] = i915_request_create(ve);
81a04d2e
DC		 3331 if (IS_ERR(request[n])) {
3332 err = PTR_ERR(request[n]);
78e41ddd
CW		 3333 nsibling = n;
3334 goto out;
3335 }
3336
3337 /* Reverse order as it's more likely to be unnatural */
3338 request[n]->execution_mask = siblings[nsibling - n - 1]->mask;
3339
3340 i915_request_get(request[n]);
3341 i915_request_add(request[n]);
3342 }
3343
3344 for (n = 0; n < nsibling; n++) {
2f530945 3345 if (i915_request_wait(request[n], 0, HZ / 10) < 0) {
78e41ddd
CW		 3346 pr_err("%s(%s): wait for %llx:%lld timed out\n",
3347 __func__, ve->engine->name,
3348 request[n]->fence.context,
3349 request[n]->fence.seqno);
3350
3351 GEM_TRACE("%s(%s) failed at request %llx:%lld\n",
3352 __func__, ve->engine->name,
3353 request[n]->fence.context,
3354 request[n]->fence.seqno);
3355 GEM_TRACE_DUMP();
1357fa81 3356 intel_gt_set_wedged(gt);
78e41ddd
CW		 3357 err = -EIO;
3358 goto out;
3359 }
3360
3361 if (request[n]->engine != siblings[nsibling - n - 1]) {
3362 pr_err("Executed on wrong sibling '%s', expected '%s'\n",
3363 request[n]->engine->name,
3364 siblings[nsibling - n - 1]->name);
3365 err = -EINVAL;
3366 goto out;
3367 }
3368 }
3369
3370 err = igt_live_test_end(&t);
78e41ddd 3371out:
1357fa81 3372 if (igt_flush_test(gt->i915))
78e41ddd
CW		 3373 err = -EIO;
3374
3375 for (n = 0; n < nsibling; n++)
3376 i915_request_put(request[n]);
3377
3378out_unpin:
3379 intel_context_unpin(ve);
3380out_put:
3381 intel_context_put(ve);
3382out_close:
78e41ddd
CW		 3383 return err;
3384}
3385
3386static int live_virtual_mask(void *arg)
3387{
1357fa81 3388 struct intel_gt *gt = arg;
78e41ddd
CW		 3389 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3390 unsigned int class, inst;
2af40298 3391 int err;
78e41ddd 3392
065273f7 3393 if (intel_uc_uses_guc_submission(&gt->uc))
78e41ddd
CW		 3394 return 0;
3395
78e41ddd
CW		 3396 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3397 unsigned int nsibling;
3398
3399 nsibling = 0;
3400 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
750e76b4 3401 if (!gt->engine_class[class][inst])
78e41ddd
CW		 3402 break;
3403
750e76b4 3404 siblings[nsibling++] = gt->engine_class[class][inst];
78e41ddd
CW		 3405 }
3406 if (nsibling < 2)
3407 continue;
3408
1357fa81 3409 err = mask_virtual_engine(gt, siblings, nsibling);
78e41ddd 3410 if (err)
2af40298 3411 return err;
78e41ddd
CW		 3412 }
3413
2af40298 3414 return 0;
78e41ddd
CW		 3415}
3416
1357fa81 3417static int preserved_virtual_engine(struct intel_gt *gt,
86027e31
CW		 3418 struct intel_engine_cs **siblings,
3419 unsigned int nsibling)
3420{
86027e31 3421 struct i915_request *last = NULL;
86027e31
CW		 3422 struct intel_context *ve;
3423 struct i915_vma *scratch;
3424 struct igt_live_test t;
86027e31
CW		 3425 unsigned int n;
3426 int err = 0;
9506c23d 3427 u32 *cs;
86027e31 3428
86027e31 3429 scratch = create_scratch(siblings[0]->gt);
e6ba7648
CW		 3430 if (IS_ERR(scratch))
3431 return PTR_ERR(scratch);
86027e31 3432
bd46aa22
CW		 3433 err = i915_vma_sync(scratch);
3434 if (err)
3435 goto out_scratch;
3436
e6ba7648 3437 ve = intel_execlists_create_virtual(siblings, nsibling);
86027e31
CW		 3438 if (IS_ERR(ve)) {
3439 err = PTR_ERR(ve);
3440 goto out_scratch;
3441 }
3442
3443 err = intel_context_pin(ve);
3444 if (err)
3445 goto out_put;
3446
1357fa81 3447 err = igt_live_test_begin(&t, gt->i915, __func__, ve->engine->name);
86027e31
CW		 3448 if (err)
3449 goto out_unpin;
3450
9506c23d 3451 for (n = 0; n < NUM_GPR_DW; n++) {
86027e31
CW		 3452 struct intel_engine_cs *engine = siblings[n % nsibling];
3453 struct i915_request *rq;
86027e31
CW		 3454
3455 rq = i915_request_create(ve);
3456 if (IS_ERR(rq)) {
3457 err = PTR_ERR(rq);
3458 goto out_end;
3459 }
3460
3461 i915_request_put(last);
3462 last = i915_request_get(rq);
3463
3464 cs = intel_ring_begin(rq, 8);
3465 if (IS_ERR(cs)) {
3466 i915_request_add(rq);
3467 err = PTR_ERR(cs);
3468 goto out_end;
3469 }
3470
3471 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
3472 *cs++ = CS_GPR(engine, n);
3473 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
3474 *cs++ = 0;
3475
3476 *cs++ = MI_LOAD_REGISTER_IMM(1);
9506c23d 3477 *cs++ = CS_GPR(engine, (n + 1) % NUM_GPR_DW);
86027e31
CW		 3478 *cs++ = n + 1;
3479
3480 *cs++ = MI_NOOP;
3481 intel_ring_advance(rq, cs);
3482
3483 /* Restrict this request to run on a particular engine */
3484 rq->execution_mask = engine->mask;
3485 i915_request_add(rq);
3486 }
3487
3488 if (i915_request_wait(last, 0, HZ / 5) < 0) {
3489 err = -ETIME;
9506c23d
CW		 3490 goto out_end;
3491 }
86027e31 3492
9506c23d
CW		 3493 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
3494 if (IS_ERR(cs)) {
3495 err = PTR_ERR(cs);
3496 goto out_end;
3497 }
86027e31 3498
9506c23d
CW		 3499 for (n = 0; n < NUM_GPR_DW; n++) {
3500 if (cs[n] != n) {
3501 pr_err("Incorrect value[%d] found for GPR[%d]\n",
3502 cs[n], n);
3503 err = -EINVAL;
3504 break;
3505 }
86027e31
CW		 3506 }
3507
9506c23d
CW		 3508 i915_gem_object_unpin_map(scratch->obj);
3509
86027e31
CW		 3510out_end:
3511 if (igt_live_test_end(&t))
3512 err = -EIO;
3513 i915_request_put(last);
3514out_unpin:
3515 intel_context_unpin(ve);
3516out_put:
3517 intel_context_put(ve);
3518out_scratch:
3519 i915_vma_unpin_and_release(&scratch, 0);
86027e31 3520 return err;
86027e31
CW		 3521}
3522
3523static int live_virtual_preserved(void *arg)
3524{
1357fa81 3525 struct intel_gt *gt = arg;
86027e31 3526 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
86027e31
CW		 3527 unsigned int class, inst;
3528
3529 /*
3530 * Check that the context image retains non-privileged (user) registers
3531 * from one engine to the next. For this we check that the CS_GPR
3532 * are preserved.
3533 */
3534
065273f7 3535 if (intel_uc_uses_guc_submission(&gt->uc))
86027e31
CW		 3536 return 0;
3537
3538 /* As we use CS_GPR we cannot run before they existed on all engines. */
1357fa81 3539 if (INTEL_GEN(gt->i915) < 9)
86027e31
CW		 3540 return 0;
3541
3542 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3543 int nsibling, err;
3544
3545 nsibling = 0;
3546 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3547 if (!gt->engine_class[class][inst])
3548 continue;
3549
3550 siblings[nsibling++] = gt->engine_class[class][inst];
3551 }
3552 if (nsibling < 2)
3553 continue;
3554
1357fa81 3555 err = preserved_virtual_engine(gt, siblings, nsibling);
86027e31
CW		 3556 if (err)
3557 return err;
3558 }
3559
3560 return 0;
3561}
3562
1357fa81 3563static int bond_virtual_engine(struct intel_gt *gt,
ee113690
CW		 3564 unsigned int class,
3565 struct intel_engine_cs **siblings,
3566 unsigned int nsibling,
3567 unsigned int flags)
3568#define BOND_SCHEDULE BIT(0)
3569{
3570 struct intel_engine_cs *master;
ee113690
CW		 3571 struct i915_request *rq[16];
3572 enum intel_engine_id id;
cfd821b2 3573 struct igt_spinner spin;
ee113690
CW		 3574 unsigned long n;
3575 int err;
3576
cfd821b2
CW		 3577 /*
3578 * A set of bonded requests is intended to be run concurrently
3579 * across a number of engines. We use one request per-engine
3580 * and a magic fence to schedule each of the bonded requests
3581 * at the same time. A consequence of our current scheduler is that
3582 * we only move requests to the HW ready queue when the request
3583 * becomes ready, that is when all of its prerequisite fences have
3584 * been signaled. As one of those fences is the master submit fence,
3585 * there is a delay on all secondary fences as the HW may be
3586 * currently busy. Equally, as all the requests are independent,
3587 * they may have other fences that delay individual request
3588 * submission to HW. Ergo, we do not guarantee that all requests are
3589 * immediately submitted to HW at the same time, just that if the
3590 * rules are abided by, they are ready at the same time as the
3591 * first is submitted. Userspace can embed semaphores in its batch
3592 * to ensure parallel execution of its phases as it requires.
3593 * Though naturally it gets requested that perhaps the scheduler should
3594 * take care of parallel execution, even across preemption events on
3595 * different HW. (The proper answer is of course "lalalala".)
3596 *
3597 * With the submit-fence, we have identified three possible phases
3598 * of synchronisation depending on the master fence: queued (not
3599 * ready), executing, and signaled. The first two are quite simple
3600 * and checked below. However, the signaled master fence handling is
3601 * contentious. Currently we do not distinguish between a signaled
3602 * fence and an expired fence, as once signaled it does not convey
3603 * any information about the previous execution. It may even be freed
3604 * and hence checking later it may not exist at all. Ergo we currently
3605 * do not apply the bonding constraint for an already signaled fence,
3606 * as our expectation is that it should not constrain the secondaries
3607 * and is outside of the scope of the bonded request API (i.e. all
3608 * userspace requests are meant to be running in parallel). As
3609 * it imposes no constraint, and is effectively a no-op, we do not
3610 * check below as normal execution flows are checked extensively above.
3611 *
3612 * XXX Is the degenerate handling of signaled submit fences the
3613 * expected behaviour for userpace?
3614 */
3615
ee113690
CW		 3616 GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1);
3617
cfd821b2 3618 if (igt_spinner_init(&spin, gt))
ee113690
CW		 3619 return -ENOMEM;
3620
3621 err = 0;
3622 rq[0] = ERR_PTR(-ENOMEM);
5d904e3c 3623 for_each_engine(master, gt, id) {
ee113690 3624 struct i915_sw_fence fence = {};
f16ccb64 3625 struct intel_context *ce;
ee113690
CW		 3626
3627 if (master->class == class)
3628 continue;
3629
f16ccb64
CW		 3630 ce = intel_context_create(master);
3631 if (IS_ERR(ce)) {
3632 err = PTR_ERR(ce);
3633 goto out;
3634 }
3635
ee113690
CW		 3636 memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq));
3637
f16ccb64
CW		 3638 rq[0] = igt_spinner_create_request(&spin, ce, MI_NOOP);
3639 intel_context_put(ce);
ee113690
CW		 3640 if (IS_ERR(rq[0])) {
3641 err = PTR_ERR(rq[0]);
3642 goto out;
3643 }
3644 i915_request_get(rq[0]);
3645
3646 if (flags & BOND_SCHEDULE) {
3647 onstack_fence_init(&fence);
3648 err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit,
3649 &fence,
3650 GFP_KERNEL);
3651 }
cfd821b2 3652
ee113690
CW		 3653 i915_request_add(rq[0]);
3654 if (err < 0)
3655 goto out;
3656
cfd821b2
CW		 3657 if (!(flags & BOND_SCHEDULE) &&
3658 !igt_wait_for_spinner(&spin, rq[0])) {
3659 err = -EIO;
3660 goto out;
3661 }
3662
ee113690
CW		 3663 for (n = 0; n < nsibling; n++) {
3664 struct intel_context *ve;
3665
e6ba7648 3666 ve = intel_execlists_create_virtual(siblings, nsibling);
ee113690
CW		 3667 if (IS_ERR(ve)) {
3668 err = PTR_ERR(ve);
3669 onstack_fence_fini(&fence);
3670 goto out;
3671 }
3672
3673 err = intel_virtual_engine_attach_bond(ve->engine,
3674 master,
3675 siblings[n]);
3676 if (err) {
3677 intel_context_put(ve);
3678 onstack_fence_fini(&fence);
3679 goto out;
3680 }
3681
3682 err = intel_context_pin(ve);
3683 intel_context_put(ve);
3684 if (err) {
3685 onstack_fence_fini(&fence);
3686 goto out;
3687 }
3688
3689 rq[n + 1] = i915_request_create(ve);
3690 intel_context_unpin(ve);
3691 if (IS_ERR(rq[n + 1])) {
3692 err = PTR_ERR(rq[n + 1]);
3693 onstack_fence_fini(&fence);
3694 goto out;
3695 }
3696 i915_request_get(rq[n + 1]);
3697
3698 err = i915_request_await_execution(rq[n + 1],
3699 &rq[0]->fence,
3700 ve->engine->bond_execute);
3701 i915_request_add(rq[n + 1]);
3702 if (err < 0) {
3703 onstack_fence_fini(&fence);
3704 goto out;
3705 }
3706 }
3707 onstack_fence_fini(&fence);
cfd821b2
CW		 3708 intel_engine_flush_submission(master);
3709 igt_spinner_end(&spin);
ee113690 3710
2f530945 3711 if (i915_request_wait(rq[0], 0, HZ / 10) < 0) {
ee113690
CW		 3712 pr_err("Master request did not execute (on %s)!\n",
3713 rq[0]->engine->name);
3714 err = -EIO;
3715 goto out;
3716 }
3717
3718 for (n = 0; n < nsibling; n++) {
2f530945 3719 if (i915_request_wait(rq[n + 1], 0,
ee113690
CW		 3720 MAX_SCHEDULE_TIMEOUT) < 0) {
3721 err = -EIO;
3722 goto out;
3723 }
3724
3725 if (rq[n + 1]->engine != siblings[n]) {
3726 pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n",
3727 siblings[n]->name,
3728 rq[n + 1]->engine->name,
3729 rq[0]->engine->name);
3730 err = -EINVAL;
3731 goto out;
3732 }
3733 }
3734
3735 for (n = 0; !IS_ERR(rq[n]); n++)
3736 i915_request_put(rq[n]);
3737 rq[0] = ERR_PTR(-ENOMEM);
3738 }
3739
3740out:
3741 for (n = 0; !IS_ERR(rq[n]); n++)
3742 i915_request_put(rq[n]);
1357fa81 3743 if (igt_flush_test(gt->i915))
ee113690
CW		 3744 err = -EIO;
3745
cfd821b2 3746 igt_spinner_fini(&spin);
ee113690
CW		 3747 return err;
3748}
3749
3750static int live_virtual_bond(void *arg)
3751{
3752 static const struct phase {
3753 const char *name;
3754 unsigned int flags;
3755 } phases[] = {
3756 { "", 0 },
3757 { "schedule", BOND_SCHEDULE },
3758 { },
3759 };
1357fa81 3760 struct intel_gt *gt = arg;
ee113690
CW		 3761 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3762 unsigned int class, inst;
2af40298 3763 int err;
ee113690 3764
065273f7 3765 if (intel_uc_uses_guc_submission(&gt->uc))
ee113690
CW		 3766 return 0;
3767
ee113690
CW		 3768 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3769 const struct phase *p;
3770 int nsibling;
3771
3772 nsibling = 0;
3773 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
750e76b4 3774 if (!gt->engine_class[class][inst])
ee113690
CW		 3775 break;
3776
3777 GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings));
750e76b4 3778 siblings[nsibling++] = gt->engine_class[class][inst];
ee113690
CW		 3779 }
3780 if (nsibling < 2)
3781 continue;
3782
3783 for (p = phases; p->name; p++) {
1357fa81 3784 err = bond_virtual_engine(gt,
ee113690
CW		 3785 class, siblings, nsibling,
3786 p->flags);
3787 if (err) {
3788 pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n",
3789 __func__, p->name, class, nsibling, err);
2af40298 3790 return err;
ee113690
CW		 3791 }
3792 }
3793 }
3794
2af40298 3795 return 0;
ee113690
CW		 3796}
3797
989df3a7
CW		 3798static int reset_virtual_engine(struct intel_gt *gt,
3799 struct intel_engine_cs **siblings,
3800 unsigned int nsibling)
3801{
3802 struct intel_engine_cs *engine;
3803 struct intel_context *ve;
3804 unsigned long *heartbeat;
3805 struct igt_spinner spin;
3806 struct i915_request *rq;
3807 unsigned int n;
3808 int err = 0;
3809
3810 /*
3811 * In order to support offline error capture for fast preempt reset,
3812 * we need to decouple the guilty request and ensure that it and its
3813 * descendents are not executed while the capture is in progress.
3814 */
3815
3816 heartbeat = kmalloc_array(nsibling, sizeof(*heartbeat), GFP_KERNEL);
3817 if (!heartbeat)
3818 return -ENOMEM;
3819
3820 if (igt_spinner_init(&spin, gt)) {
3821 err = -ENOMEM;
3822 goto out_free;
3823 }
3824
3825 ve = intel_execlists_create_virtual(siblings, nsibling);
3826 if (IS_ERR(ve)) {
3827 err = PTR_ERR(ve);
3828 goto out_spin;
3829 }
3830
3831 for (n = 0; n < nsibling; n++)
3832 engine_heartbeat_disable(siblings[n], &heartbeat[n]);
3833
3834 rq = igt_spinner_create_request(&spin, ve, MI_ARB_CHECK);
3835 if (IS_ERR(rq)) {
3836 err = PTR_ERR(rq);
3837 goto out_heartbeat;
3838 }
3839 i915_request_add(rq);
3840
3841 if (!igt_wait_for_spinner(&spin, rq)) {
3842 intel_gt_set_wedged(gt);
3843 err = -ETIME;
3844 goto out_heartbeat;
3845 }
3846
3847 engine = rq->engine;
3848 GEM_BUG_ON(engine == ve->engine);
3849
3850 /* Take ownership of the reset and tasklet */
3851 if (test_and_set_bit(I915_RESET_ENGINE + engine->id,
3852 &gt->reset.flags)) {
3853 intel_gt_set_wedged(gt);
3854 err = -EBUSY;
3855 goto out_heartbeat;
3856 }
3857 tasklet_disable(&engine->execlists.tasklet);
3858
3859 engine->execlists.tasklet.func(engine->execlists.tasklet.data);
3860 GEM_BUG_ON(execlists_active(&engine->execlists) != rq);
3861
3862 /* Fake a preemption event; failed of course */
3863 spin_lock_irq(&engine->active.lock);
3864 __unwind_incomplete_requests(engine);
3865 spin_unlock_irq(&engine->active.lock);
3866 GEM_BUG_ON(rq->engine != ve->engine);
3867
3868 /* Reset the engine while keeping our active request on hold */
3869 execlists_hold(engine, rq);
3870 GEM_BUG_ON(!i915_request_on_hold(rq));
3871
3872 intel_engine_reset(engine, NULL);
3873 GEM_BUG_ON(rq->fence.error != -EIO);
3874
3875 /* Release our grasp on the engine, letting CS flow again */
3876 tasklet_enable(&engine->execlists.tasklet);
3877 clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, &gt->reset.flags);
3878
3879 /* Check that we do not resubmit the held request */
3880 i915_request_get(rq);
3881 if (!i915_request_wait(rq, 0, HZ / 5)) {
3882 pr_err("%s: on hold request completed!\n",
3883 engine->name);
3884 intel_gt_set_wedged(gt);
3885 err = -EIO;
3886 goto out_rq;
3887 }
3888 GEM_BUG_ON(!i915_request_on_hold(rq));
3889
3890 /* But is resubmitted on release */
3891 execlists_unhold(engine, rq);
3892 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
3893 pr_err("%s: held request did not complete!\n",
3894 engine->name);
3895 intel_gt_set_wedged(gt);
3896 err = -ETIME;
3897 }
3898
3899out_rq:
3900 i915_request_put(rq);
3901out_heartbeat:
3902 for (n = 0; n < nsibling; n++)
3903 engine_heartbeat_enable(siblings[n], heartbeat[n]);
3904
3905 intel_context_put(ve);
3906out_spin:
3907 igt_spinner_fini(&spin);
3908out_free:
3909 kfree(heartbeat);
3910 return err;
3911}
3912
3913static int live_virtual_reset(void *arg)
3914{
3915 struct intel_gt *gt = arg;
3916 struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1];
3917 unsigned int class, inst;
3918
3919 /*
3920 * Check that we handle a reset event within a virtual engine.
3921 * Only the physical engine is reset, but we have to check the flow
3922 * of the virtual requests around the reset, and make sure it is not
3923 * forgotten.
3924 */
3925
065273f7 3926 if (intel_uc_uses_guc_submission(&gt->uc))
989df3a7
CW		 3927 return 0;
3928
3929 if (!intel_has_reset_engine(gt))
3930 return 0;
3931
3932 for (class = 0; class <= MAX_ENGINE_CLASS; class++) {
3933 int nsibling, err;
3934
3935 nsibling = 0;
3936 for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) {
3937 if (!gt->engine_class[class][inst])
3938 continue;
3939
3940 siblings[nsibling++] = gt->engine_class[class][inst];
3941 }
3942 if (nsibling < 2)
3943 continue;
3944
3945 err = reset_virtual_engine(gt, siblings, nsibling);
3946 if (err)
3947 return err;
3948 }
3949
3950 return 0;
3951}
3952
2c66555e
CW		 3953int intel_execlists_live_selftests(struct drm_i915_private *i915)
3954{
3955 static const struct i915_subtest tests[] = {
3956 SUBTEST(live_sanitycheck),
fcde8c7e
CW		 3957 SUBTEST(live_unlite_switch),
3958 SUBTEST(live_unlite_preempt),
37305ede 3959 SUBTEST(live_pin_rewind),
32ff621f 3960 SUBTEST(live_hold_reset),
70a76a9b 3961 SUBTEST(live_error_interrupt),
8ee36e04 3962 SUBTEST(live_timeslice_preempt),
4c8ed8b1 3963 SUBTEST(live_timeslice_rewind),
253a774b 3964 SUBTEST(live_timeslice_queue),
bac24f59 3965 SUBTEST(live_busywait_preempt),
2c66555e
CW		 3966 SUBTEST(live_preempt),
3967 SUBTEST(live_late_preempt),
2a98f4e6 3968 SUBTEST(live_nopreempt),
d12acee8 3969 SUBTEST(live_preempt_cancel),
c9a64622 3970 SUBTEST(live_suppress_self_preempt),
b5773a36 3971 SUBTEST(live_suppress_wait_preempt),
a21f453c 3972 SUBTEST(live_chain_preempt),
2d19a71c 3973 SUBTEST(live_preempt_gang),
3a7a92ab 3974 SUBTEST(live_preempt_timeout),
dee4a0f8 3975 SUBTEST(live_preempt_smoke),
6d06779e 3976 SUBTEST(live_virtual_engine),
78e41ddd 3977 SUBTEST(live_virtual_mask),
86027e31 3978 SUBTEST(live_virtual_preserved),
ee113690 3979 SUBTEST(live_virtual_bond),
989df3a7 3980 SUBTEST(live_virtual_reset),
2c66555e 3981 };
52cc8014
CW		 3982
3983 if (!HAS_EXECLISTS(i915))
3984 return 0;
3985
cb823ed9 3986 if (intel_gt_is_wedged(&i915->gt))
03bbc508
CW		 3987 return 0;
3988
1357fa81 3989 return intel_gt_live_subtests(tests, &i915->gt);
2c66555e 3990}
7dc56af5
CW		 3991
3992static void hexdump(const void *buf, size_t len)
3993{
3994 const size_t rowsize = 8 * sizeof(u32);
3995 const void *prev = NULL;
3996 bool skip = false;
3997 size_t pos;
3998
3999 for (pos = 0; pos < len; pos += rowsize) {
4000 char line[128];
4001
4002 if (prev && !memcmp(prev, buf + pos, rowsize)) {
4003 if (!skip) {
4004 pr_info("*\n");
4005 skip = true;
4006 }
4007 continue;
4008 }
4009
4010 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
4011 rowsize, sizeof(u32),
4012 line, sizeof(line),
4013 false) >= sizeof(line));
4014 pr_info("[%04zx] %s\n", pos, line);
4015
4016 prev = buf + pos;
4017 skip = false;
4018 }
4019}
4020
e7aa531e
CW		 4021static int emit_semaphore_signal(struct intel_context *ce, void *slot)
4022{
4023 const u32 offset =
4024 i915_ggtt_offset(ce->engine->status_page.vma) +
4025 offset_in_page(slot);
4026 struct i915_request *rq;
4027 u32 *cs;
4028
4029 rq = intel_context_create_request(ce);
4030 if (IS_ERR(rq))
4031 return PTR_ERR(rq);
4032
4033 cs = intel_ring_begin(rq, 4);
4034 if (IS_ERR(cs)) {
4035 i915_request_add(rq);
4036 return PTR_ERR(cs);
4037 }
4038
4039 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
4040 *cs++ = offset;
4041 *cs++ = 0;
4042 *cs++ = 1;
4043
4044 intel_ring_advance(rq, cs);
4045
4046 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
4047 i915_request_add(rq);
4048 return 0;
4049}
4050
b0158b91
CW		 4051static int context_flush(struct intel_context *ce, long timeout)
4052{
4053 struct i915_request *rq;
4054 struct dma_fence *fence;
4055 int err = 0;
4056
4057 rq = intel_engine_create_kernel_request(ce->engine);
4058 if (IS_ERR(rq))
4059 return PTR_ERR(rq);
4060
4061 fence = i915_active_fence_get(&ce->timeline->last_request);
4062 if (fence) {
4063 i915_request_await_dma_fence(rq, fence);
4064 dma_fence_put(fence);
4065 }
4066
4067 rq = i915_request_get(rq);
4068 i915_request_add(rq);
4069 if (i915_request_wait(rq, 0, timeout) < 0)
4070 err = -ETIME;
4071 i915_request_put(rq);
4072
4073 rmb(); /* We know the request is written, make sure all state is too! */
4074 return err;
4075}
4076
7dc56af5
CW		 4077static int live_lrc_layout(void *arg)
4078{
4079 struct intel_gt *gt = arg;
4080 struct intel_engine_cs *engine;
4081 enum intel_engine_id id;
d1813ca2 4082 u32 *lrc;
7dc56af5
CW		 4083 int err;
4084
4085 /*
4086 * Check the registers offsets we use to create the initial reg state
4087 * match the layout saved by HW.
4088 */
4089
d1813ca2
CW		 4090 lrc = kmalloc(PAGE_SIZE, GFP_KERNEL);
4091 if (!lrc)
7dc56af5
CW		 4092 return -ENOMEM;
4093
4094 err = 0;
5d904e3c 4095 for_each_engine(engine, gt, id) {
d1813ca2 4096 u32 *hw;
7dc56af5
CW		 4097 int dw;
4098
4099 if (!engine->default_state)
4100 continue;
4101
4102 hw = i915_gem_object_pin_map(engine->default_state,
4103 I915_MAP_WB);
4104 if (IS_ERR(hw)) {
4105 err = PTR_ERR(hw);
4106 break;
4107 }
4108 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
4109
d1813ca2 4110 execlists_init_reg_state(memset(lrc, POISON_INUSE, PAGE_SIZE),
7dc56af5
CW		 4111 engine->kernel_context,
4112 engine,
4113 engine->kernel_context->ring,
4114 true);
4115
4116 dw = 0;
4117 do {
4118 u32 lri = hw[dw];
4119
4120 if (lri == 0) {
4121 dw++;
4122 continue;
4123 }
4124
6a505e64
CW		 4125 if (lrc[dw] == 0) {
4126 pr_debug("%s: skipped instruction %x at dword %d\n",
4127 engine->name, lri, dw);
4128 dw++;
4129 continue;
4130 }
4131
7dc56af5
CW		 4132 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
4133 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
4134 engine->name, dw, lri);
4135 err = -EINVAL;
4136 break;
4137 }
4138
4139 if (lrc[dw] != lri) {
4140 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
4141 engine->name, dw, lri, lrc[dw]);
4142 err = -EINVAL;
4143 break;
4144 }
4145
4146 lri &= 0x7f;
4147 lri++;
4148 dw++;
4149
4150 while (lri) {
4151 if (hw[dw] != lrc[dw]) {
4152 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
4153 engine->name, dw, hw[dw], lrc[dw]);
4154 err = -EINVAL;
4155 break;
4156 }
4157
4158 /*
4159 * Skip over the actual register value as we
4160 * expect that to differ.
4161 */
4162 dw += 2;
4163 lri -= 2;
4164 }
4165 } while ((lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
4166
4167 if (err) {
4168 pr_info("%s: HW register image:\n", engine->name);
4169 hexdump(hw, PAGE_SIZE);
4170
4171 pr_info("%s: SW register image:\n", engine->name);
4172 hexdump(lrc, PAGE_SIZE);
4173 }
4174
4175 i915_gem_object_unpin_map(engine->default_state);
4176 if (err)
4177 break;
4178 }
4179
d1813ca2 4180 kfree(lrc);
7dc56af5
CW		 4181 return err;
4182}
4183
35865aef
CW		 4184static int find_offset(const u32 *lri, u32 offset)
4185{
4186 int i;
4187
4188 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
4189 if (lri[i] == offset)
4190 return i;
4191
4192 return -1;
4193}
4194
4195static int live_lrc_fixed(void *arg)
4196{
4197 struct intel_gt *gt = arg;
4198 struct intel_engine_cs *engine;
4199 enum intel_engine_id id;
4200 int err = 0;
4201
4202 /*
4203 * Check the assumed register offsets match the actual locations in
4204 * the context image.
4205 */
4206
4207 for_each_engine(engine, gt, id) {
4208 const struct {
4209 u32 reg;
4210 u32 offset;
4211 const char *name;
4212 } tbl[] = {
13670f4c
CW		 4213 {
4214 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
b0b10248 4215 CTX_RING_START - 1,
13670f4c
CW		 4216 "RING_START"
4217 },
4218 {
4219 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
b0b10248 4220 CTX_RING_CTL - 1,
13670f4c
CW		 4221 "RING_CTL"
4222 },
4223 {
4224 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
4225 CTX_RING_HEAD - 1,
4226 "RING_HEAD"
4227 },
4228 {
4229 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
4230 CTX_RING_TAIL - 1,
4231 "RING_TAIL"
4232 },
35865aef
CW		 4233 {
4234 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
4235 lrc_ring_mi_mode(engine),
13670f4c
CW		 4236 "RING_MI_MODE"
4237 },
4238 {
b0b10248 4239 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
13670f4c
CW		 4240 CTX_BB_STATE - 1,
4241 "BB_STATE"
35865aef 4242 },
1883a0a4
TU		 4243 {
4244 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
4245 CTX_TIMESTAMP - 1,
4246 "RING_CTX_TIMESTAMP"
4247 },
35865aef
CW		 4248 { },
4249 }, *t;
4250 u32 *hw;
4251
4252 if (!engine->default_state)
4253 continue;
4254
4255 hw = i915_gem_object_pin_map(engine->default_state,
4256 I915_MAP_WB);
4257 if (IS_ERR(hw)) {
4258 err = PTR_ERR(hw);
4259 break;
4260 }
4261 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
4262
4263 for (t = tbl; t->name; t++) {
4264 int dw = find_offset(hw, t->reg);
4265
4266 if (dw != t->offset) {
4267 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
4268 engine->name,
4269 t->name,
4270 t->reg,
4271 dw,
4272 t->offset);
4273 err = -EINVAL;
4274 }
4275 }
4276
4277 i915_gem_object_unpin_map(engine->default_state);
4278 }
4279
4280 return err;
4281}
4282
e6ba7648 4283static int __live_lrc_state(struct intel_engine_cs *engine,
9c27462c
CW		 4284 struct i915_vma *scratch)
4285{
4286 struct intel_context *ce;
4287 struct i915_request *rq;
4288 enum {
4289 RING_START_IDX = 0,
4290 RING_TAIL_IDX,
4291 MAX_IDX
4292 };
4293 u32 expected[MAX_IDX];
4294 u32 *cs;
4295 int err;
4296 int n;
4297
e6ba7648 4298 ce = intel_context_create(engine);
9c27462c
CW		 4299 if (IS_ERR(ce))
4300 return PTR_ERR(ce);
4301
4302 err = intel_context_pin(ce);
4303 if (err)
4304 goto err_put;
4305
4306 rq = i915_request_create(ce);
4307 if (IS_ERR(rq)) {
4308 err = PTR_ERR(rq);
4309 goto err_unpin;
4310 }
4311
4312 cs = intel_ring_begin(rq, 4 * MAX_IDX);
4313 if (IS_ERR(cs)) {
4314 err = PTR_ERR(cs);
4315 i915_request_add(rq);
4316 goto err_unpin;
4317 }
4318
4319 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4320 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
4321 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
4322 *cs++ = 0;
4323
4324 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
4325
4326 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4327 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
4328 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
4329 *cs++ = 0;
4330
bd46aa22
CW		 4331 i915_vma_lock(scratch);
4332 err = i915_request_await_object(rq, scratch->obj, true);
4333 if (!err)
4334 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
4335 i915_vma_unlock(scratch);
4336
9c27462c
CW		 4337 i915_request_get(rq);
4338 i915_request_add(rq);
bd46aa22
CW		 4339 if (err)
4340 goto err_rq;
9c27462c
CW		 4341
4342 intel_engine_flush_submission(engine);
4343 expected[RING_TAIL_IDX] = ce->ring->tail;
4344
4345 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4346 err = -ETIME;
4347 goto err_rq;
4348 }
4349
4350 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4351 if (IS_ERR(cs)) {
4352 err = PTR_ERR(cs);
4353 goto err_rq;
4354 }
4355
4356 for (n = 0; n < MAX_IDX; n++) {
4357 if (cs[n] != expected[n]) {
4358 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
4359 engine->name, n, cs[n], expected[n]);
4360 err = -EINVAL;
4361 break;
4362 }
4363 }
4364
4365 i915_gem_object_unpin_map(scratch->obj);
4366
4367err_rq:
4368 i915_request_put(rq);
4369err_unpin:
4370 intel_context_unpin(ce);
4371err_put:
4372 intel_context_put(ce);
4373 return err;
4374}
4375
4376static int live_lrc_state(void *arg)
4377{
4378 struct intel_gt *gt = arg;
4379 struct intel_engine_cs *engine;
9c27462c
CW		 4380 struct i915_vma *scratch;
4381 enum intel_engine_id id;
4382 int err = 0;
4383
4384 /*
4385 * Check the live register state matches what we expect for this
4386 * intel_context.
4387 */
4388
9c27462c 4389 scratch = create_scratch(gt);
e6ba7648
CW		 4390 if (IS_ERR(scratch))
4391 return PTR_ERR(scratch);
9c27462c 4392
5d904e3c 4393 for_each_engine(engine, gt, id) {
e6ba7648 4394 err = __live_lrc_state(engine, scratch);
9c27462c
CW		 4395 if (err)
4396 break;
4397 }
4398
4399 if (igt_flush_test(gt->i915))
4400 err = -EIO;
4401
4402 i915_vma_unpin_and_release(&scratch, 0);
9c27462c
CW		 4403 return err;
4404}
4405
bd3d1f86 4406static int gpr_make_dirty(struct intel_context *ce)
9506c23d
CW		 4407{
4408 struct i915_request *rq;
4409 u32 *cs;
4410 int n;
4411
bd3d1f86 4412 rq = intel_context_create_request(ce);
9506c23d
CW		 4413 if (IS_ERR(rq))
4414 return PTR_ERR(rq);
4415
4416 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
4417 if (IS_ERR(cs)) {
4418 i915_request_add(rq);
4419 return PTR_ERR(cs);
4420 }
4421
4422 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
4423 for (n = 0; n < NUM_GPR_DW; n++) {
bd3d1f86 4424 *cs++ = CS_GPR(ce->engine, n);
9506c23d
CW		 4425 *cs++ = STACK_MAGIC;
4426 }
4427 *cs++ = MI_NOOP;
4428
4429 intel_ring_advance(rq, cs);
bd3d1f86
CW		 4430
4431 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
9506c23d
CW		 4432 i915_request_add(rq);
4433
4434 return 0;
4435}
4436
bd3d1f86
CW		 4437static struct i915_request *
4438__gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
9506c23d 4439{
bd3d1f86
CW		 4440 const u32 offset =
4441 i915_ggtt_offset(ce->engine->status_page.vma) +
4442 offset_in_page(slot);
9506c23d
CW		 4443 struct i915_request *rq;
4444 u32 *cs;
4445 int err;
4446 int n;
4447
9506c23d 4448 rq = intel_context_create_request(ce);
bd3d1f86
CW		 4449 if (IS_ERR(rq))
4450 return rq;
9506c23d 4451
bd3d1f86 4452 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
9506c23d 4453 if (IS_ERR(cs)) {
9506c23d 4454 i915_request_add(rq);
bd3d1f86 4455 return ERR_CAST(cs);
9506c23d
CW		 4456 }
4457
bd3d1f86
CW		 4458 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
4459 *cs++ = MI_NOOP;
4460
4461 *cs++ = MI_SEMAPHORE_WAIT |
4462 MI_SEMAPHORE_GLOBAL_GTT |
4463 MI_SEMAPHORE_POLL |
4464 MI_SEMAPHORE_SAD_NEQ_SDD;
4465 *cs++ = 0;
4466 *cs++ = offset;
4467 *cs++ = 0;
4468
9506c23d
CW		 4469 for (n = 0; n < NUM_GPR_DW; n++) {
4470 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
bd3d1f86 4471 *cs++ = CS_GPR(ce->engine, n);
9506c23d
CW		 4472 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
4473 *cs++ = 0;
4474 }
4475
bd46aa22
CW		 4476 i915_vma_lock(scratch);
4477 err = i915_request_await_object(rq, scratch->obj, true);
4478 if (!err)
4479 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
4480 i915_vma_unlock(scratch);
4481
9506c23d
CW		 4482 i915_request_get(rq);
4483 i915_request_add(rq);
bd3d1f86
CW		 4484 if (err) {
4485 i915_request_put(rq);
4486 rq = ERR_PTR(err);
4487 }
4488
4489 return rq;
4490}
4491
4492static int __live_lrc_gpr(struct intel_engine_cs *engine,
4493 struct i915_vma *scratch,
4494 bool preempt)
4495{
4496 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
4497 struct intel_context *ce;
4498 struct i915_request *rq;
4499 u32 *cs;
4500 int err;
4501 int n;
4502
4503 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
4504 return 0; /* GPR only on rcs0 for gen8 */
4505
4506 err = gpr_make_dirty(engine->kernel_context);
4507 if (err)
4508 return err;
4509
4510 ce = intel_context_create(engine);
4511 if (IS_ERR(ce))
4512 return PTR_ERR(ce);
4513
4514 rq = __gpr_read(ce, scratch, slot);
4515 if (IS_ERR(rq)) {
4516 err = PTR_ERR(rq);
4517 goto err_put;
4518 }
4519
4520 err = wait_for_submit(engine, rq, HZ / 2);
bd46aa22
CW		 4521 if (err)
4522 goto err_rq;
9506c23d 4523
bd3d1f86
CW		 4524 if (preempt) {
4525 err = gpr_make_dirty(engine->kernel_context);
4526 if (err)
4527 goto err_rq;
4528
4529 err = emit_semaphore_signal(engine->kernel_context, slot);
4530 if (err)
4531 goto err_rq;
4532 } else {
4533 slot[0] = 1;
4534 wmb();
4535 }
4536
9506c23d
CW		 4537 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
4538 err = -ETIME;
4539 goto err_rq;
4540 }
4541
4542 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4543 if (IS_ERR(cs)) {
4544 err = PTR_ERR(cs);
4545 goto err_rq;
4546 }
4547
4548 for (n = 0; n < NUM_GPR_DW; n++) {
4549 if (cs[n]) {
4550 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
4551 engine->name,
4552 n / 2, n & 1 ? "udw" : "ldw",
4553 cs[n]);
4554 err = -EINVAL;
4555 break;
4556 }
4557 }
4558
4559 i915_gem_object_unpin_map(scratch->obj);
4560
4561err_rq:
bd3d1f86
CW		 4562 memset32(&slot[0], -1, 4);
4563 wmb();
9506c23d
CW		 4564 i915_request_put(rq);
4565err_put:
4566 intel_context_put(ce);
4567 return err;
4568}
4569
bd3d1f86 4570static int live_lrc_gpr(void *arg)
9506c23d
CW		 4571{
4572 struct intel_gt *gt = arg;
4573 struct intel_engine_cs *engine;
9506c23d
CW		 4574 struct i915_vma *scratch;
4575 enum intel_engine_id id;
4576 int err = 0;
4577
4578 /*
4579 * Check that GPR registers are cleared in new contexts as we need
4580 * to avoid leaking any information from previous contexts.
4581 */
4582
9506c23d 4583 scratch = create_scratch(gt);
e6ba7648
CW		 4584 if (IS_ERR(scratch))
4585 return PTR_ERR(scratch);
9506c23d 4586
5d904e3c 4587 for_each_engine(engine, gt, id) {
bd3d1f86
CW		 4588 unsigned long heartbeat;
4589
4590 engine_heartbeat_disable(engine, &heartbeat);
4591
4592 err = __live_lrc_gpr(engine, scratch, false);
4593 if (err)
4594 goto err;
4595
4596 err = __live_lrc_gpr(engine, scratch, true);
4597 if (err)
4598 goto err;
4599
4600err:
4601 engine_heartbeat_enable(engine, heartbeat);
4602 if (igt_flush_test(gt->i915))
4603 err = -EIO;
9506c23d
CW		 4604 if (err)
4605 break;
4606 }
4607
9506c23d 4608 i915_vma_unpin_and_release(&scratch, 0);
9506c23d
CW		 4609 return err;
4610}
4611
e7aa531e
CW		 4612static struct i915_request *
4613create_timestamp(struct intel_context *ce, void *slot, int idx)
4614{
4615 const u32 offset =
4616 i915_ggtt_offset(ce->engine->status_page.vma) +
4617 offset_in_page(slot);
4618 struct i915_request *rq;
4619 u32 *cs;
4620 int err;
4621
4622 rq = intel_context_create_request(ce);
4623 if (IS_ERR(rq))
4624 return rq;
4625
4626 cs = intel_ring_begin(rq, 10);
4627 if (IS_ERR(cs)) {
4628 err = PTR_ERR(cs);
4629 goto err;
4630 }
4631
4632 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
4633 *cs++ = MI_NOOP;
4634
4635 *cs++ = MI_SEMAPHORE_WAIT |
4636 MI_SEMAPHORE_GLOBAL_GTT |
4637 MI_SEMAPHORE_POLL |
4638 MI_SEMAPHORE_SAD_NEQ_SDD;
4639 *cs++ = 0;
4640 *cs++ = offset;
4641 *cs++ = 0;
4642
4643 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
4644 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
4645 *cs++ = offset + idx * sizeof(u32);
4646 *cs++ = 0;
4647
4648 intel_ring_advance(rq, cs);
4649
4650 rq->sched.attr.priority = I915_PRIORITY_MASK;
4651 err = 0;
4652err:
4653 i915_request_get(rq);
4654 i915_request_add(rq);
4655 if (err) {
4656 i915_request_put(rq);
4657 return ERR_PTR(err);
4658 }
4659
4660 return rq;
4661}
4662
4663struct lrc_timestamp {
4664 struct intel_engine_cs *engine;
4665 struct intel_context *ce[2];
4666 u32 poison;
4667};
4668
4669static bool timestamp_advanced(u32 start, u32 end)
4670{
4671 return (s32)(end - start) > 0;
4672}
4673
4674static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
4675{
4676 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
4677 struct i915_request *rq;
4678 u32 timestamp;
4679 int err = 0;
4680
4681 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
4682 rq = create_timestamp(arg->ce[0], slot, 1);
4683 if (IS_ERR(rq))
4684 return PTR_ERR(rq);
4685
4686 err = wait_for_submit(rq->engine, rq, HZ / 2);
4687 if (err)
4688 goto err;
4689
4690 if (preempt) {
4691 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
4692 err = emit_semaphore_signal(arg->ce[1], slot);
4693 if (err)
4694 goto err;
4695 } else {
4696 slot[0] = 1;
4697 wmb();
4698 }
4699
b0158b91
CW		 4700 /* And wait for switch to kernel (to save our context to memory) */
4701 err = context_flush(arg->ce[0], HZ / 2);
4702 if (err)
e7aa531e 4703 goto err;
e7aa531e
CW		 4704
4705 if (!timestamp_advanced(arg->poison, slot[1])) {
4706 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
4707 arg->engine->name, preempt ? "preempt" : "simple",
4708 arg->poison, slot[1]);
4709 err = -EINVAL;
4710 }
4711
4712 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
4713 if (!timestamp_advanced(slot[1], timestamp)) {
4714 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
4715 arg->engine->name, preempt ? "preempt" : "simple",
4716 slot[1], timestamp);
4717 err = -EINVAL;
4718 }
4719
4720err:
4721 memset32(slot, -1, 4);
4722 i915_request_put(rq);
4723 return err;
4724}
4725
4726static int live_lrc_timestamp(void *arg)
4727{
9b234d26 4728 struct lrc_timestamp data = {};
e7aa531e
CW		 4729 struct intel_gt *gt = arg;
4730 enum intel_engine_id id;
e7aa531e
CW		 4731 const u32 poison[] = {
4732 0,
4733 S32_MAX,
4734 (u32)S32_MAX + 1,
4735 U32_MAX,
4736 };
4737
4738 /*
4739 * We want to verify that the timestamp is saved and restore across
4740 * context switches and is monotonic.
4741 *
4742 * So we do this with a little bit of LRC poisoning to check various
4743 * boundary conditions, and see what happens if we preempt the context
4744 * with a second request (carrying more poison into the timestamp).
4745 */
4746
4747 for_each_engine(data.engine, gt, id) {
4748 unsigned long heartbeat;
4749 int i, err = 0;
4750
4751 engine_heartbeat_disable(data.engine, &heartbeat);
4752
4753 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
4754 struct intel_context *tmp;
4755
4756 tmp = intel_context_create(data.engine);
4757 if (IS_ERR(tmp)) {
4758 err = PTR_ERR(tmp);
4759 goto err;
4760 }
4761
4762 err = intel_context_pin(tmp);
4763 if (err) {
4764 intel_context_put(tmp);
4765 goto err;
4766 }
4767
4768 data.ce[i] = tmp;
4769 }
4770
4771 for (i = 0; i < ARRAY_SIZE(poison); i++) {
4772 data.poison = poison[i];
4773
4774 err = __lrc_timestamp(&data, false);
4775 if (err)
4776 break;
4777
4778 err = __lrc_timestamp(&data, true);
4779 if (err)
4780 break;
4781 }
4782
4783err:
4784 engine_heartbeat_enable(data.engine, heartbeat);
4785 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
4786 if (!data.ce[i])
4787 break;
4788
4789 intel_context_unpin(data.ce[i]);
4790 intel_context_put(data.ce[i]);
4791 }
4792
4793 if (igt_flush_test(gt->i915))
4794 err = -EIO;
4795 if (err)
4796 return err;
4797 }
4798
4799 return 0;
4800}
4801
efb69b98
CW		 4802static struct i915_vma *
4803create_user_vma(struct i915_address_space *vm, unsigned long size)
4804{
4805 struct drm_i915_gem_object *obj;
4806 struct i915_vma *vma;
4807 int err;
4808
4809 obj = i915_gem_object_create_internal(vm->i915, size);
4810 if (IS_ERR(obj))
4811 return ERR_CAST(obj);
4812
4813 vma = i915_vma_instance(obj, vm, NULL);
4814 if (IS_ERR(vma)) {
4815 i915_gem_object_put(obj);
4816 return vma;
4817 }
4818
4819 err = i915_vma_pin(vma, 0, 0, PIN_USER);
4820 if (err) {
4821 i915_gem_object_put(obj);
4822 return ERR_PTR(err);
4823 }
4824
4825 return vma;
4826}
4827
4828static struct i915_vma *
4829store_context(struct intel_context *ce, struct i915_vma *scratch)
4830{
4831 struct i915_vma *batch;
4832 u32 dw, x, *cs, *hw;
4833
4834 batch = create_user_vma(ce->vm, SZ_64K);
4835 if (IS_ERR(batch))
4836 return batch;
4837
4838 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
4839 if (IS_ERR(cs)) {
4840 i915_vma_put(batch);
4841 return ERR_CAST(cs);
4842 }
4843
4844 x = 0;
4845 dw = 0;
4846 hw = ce->engine->pinned_default_state;
4847 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
4848 do {
4849 u32 len = hw[dw] & 0x7f;
4850
4851 if (hw[dw] == 0) {
4852 dw++;
4853 continue;
4854 }
4855
4856 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
4857 dw += len + 2;
4858 continue;
4859 }
4860
4861 dw++;
4862 len = (len + 1) / 2;
4863 while (len--) {
4864 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
4865 *cs++ = hw[dw];
4866 *cs++ = lower_32_bits(scratch->node.start + x);
4867 *cs++ = upper_32_bits(scratch->node.start + x);
4868
4869 dw += 2;
4870 x += 4;
4871 }
4872 } while (dw < PAGE_SIZE / sizeof(u32) &&
4873 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
4874
4875 *cs++ = MI_BATCH_BUFFER_END;
4876
4877 i915_gem_object_flush_map(batch->obj);
4878 i915_gem_object_unpin_map(batch->obj);
4879
4880 return batch;
4881}
4882
4883static int move_to_active(struct i915_request *rq,
4884 struct i915_vma *vma,
4885 unsigned int flags)
4886{
4887 int err;
4888
4889 i915_vma_lock(vma);
4890 err = i915_request_await_object(rq, vma->obj, flags);
4891 if (!err)
4892 err = i915_vma_move_to_active(vma, rq, flags);
4893 i915_vma_unlock(vma);
4894
4895 return err;
4896}
4897
4898static struct i915_request *
4899record_registers(struct intel_context *ce,
4900 struct i915_vma *before,
4901 struct i915_vma *after,
4902 u32 *sema)
4903{
4904 struct i915_vma *b_before, *b_after;
4905 struct i915_request *rq;
4906 u32 *cs;
4907 int err;
4908
4909 b_before = store_context(ce, before);
4910 if (IS_ERR(b_before))
4911 return ERR_CAST(b_before);
4912
4913 b_after = store_context(ce, after);
4914 if (IS_ERR(b_after)) {
4915 rq = ERR_CAST(b_after);
4916 goto err_before;
4917 }
4918
4919 rq = intel_context_create_request(ce);
4920 if (IS_ERR(rq))
4921 goto err_after;
4922
4923 err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
4924 if (err)
4925 goto err_rq;
4926
4927 err = move_to_active(rq, b_before, 0);
4928 if (err)
4929 goto err_rq;
4930
4931 err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
4932 if (err)
4933 goto err_rq;
4934
4935 err = move_to_active(rq, b_after, 0);
4936 if (err)
4937 goto err_rq;
4938
4939 cs = intel_ring_begin(rq, 14);
4940 if (IS_ERR(cs)) {
4941 err = PTR_ERR(cs);
4942 goto err_rq;
4943 }
4944
4945 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
4946 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
4947 *cs++ = lower_32_bits(b_before->node.start);
4948 *cs++ = upper_32_bits(b_before->node.start);
4949
4950 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
4951 *cs++ = MI_SEMAPHORE_WAIT |
4952 MI_SEMAPHORE_GLOBAL_GTT |
4953 MI_SEMAPHORE_POLL |
4954 MI_SEMAPHORE_SAD_NEQ_SDD;
4955 *cs++ = 0;
4956 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
4957 offset_in_page(sema);
4958 *cs++ = 0;
4959 *cs++ = MI_NOOP;
4960
4961 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
4962 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
4963 *cs++ = lower_32_bits(b_after->node.start);
4964 *cs++ = upper_32_bits(b_after->node.start);
4965
4966 intel_ring_advance(rq, cs);
4967
4968 WRITE_ONCE(*sema, 0);
4969 i915_request_get(rq);
4970 i915_request_add(rq);
4971err_after:
4972 i915_vma_put(b_after);
4973err_before:
4974 i915_vma_put(b_before);
4975 return rq;
4976
4977err_rq:
4978 i915_request_add(rq);
4979 rq = ERR_PTR(err);
4980 goto err_after;
4981}
4982
4983static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
4984{
4985 struct i915_vma *batch;
4986 u32 dw, *cs, *hw;
4987
4988 batch = create_user_vma(ce->vm, SZ_64K);
4989 if (IS_ERR(batch))
4990 return batch;
4991
4992 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
4993 if (IS_ERR(cs)) {
4994 i915_vma_put(batch);
4995 return ERR_CAST(cs);
4996 }
4997
4998 dw = 0;
4999 hw = ce->engine->pinned_default_state;
5000 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
5001 do {
5002 u32 len = hw[dw] & 0x7f;
5003
5004 if (hw[dw] == 0) {
5005 dw++;
5006 continue;
5007 }
5008
5009 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5010 dw += len + 2;
5011 continue;
5012 }
5013
5014 dw++;
5015 len = (len + 1) / 2;
5016 *cs++ = MI_LOAD_REGISTER_IMM(len);
5017 while (len--) {
5018 *cs++ = hw[dw];
5019 *cs++ = poison;
5020 dw += 2;
5021 }
5022 } while (dw < PAGE_SIZE / sizeof(u32) &&
5023 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5024
5025 *cs++ = MI_BATCH_BUFFER_END;
5026
5027 i915_gem_object_flush_map(batch->obj);
5028 i915_gem_object_unpin_map(batch->obj);
5029
5030 return batch;
5031}
5032
5033static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
5034{
5035 struct i915_request *rq;
5036 struct i915_vma *batch;
5037 u32 *cs;
5038 int err;
5039
5040 batch = load_context(ce, poison);
5041 if (IS_ERR(batch))
5042 return PTR_ERR(batch);
5043
5044 rq = intel_context_create_request(ce);
5045 if (IS_ERR(rq)) {
5046 err = PTR_ERR(rq);
5047 goto err_batch;
5048 }
5049
5050 err = move_to_active(rq, batch, 0);
5051 if (err)
5052 goto err_rq;
5053
5054 cs = intel_ring_begin(rq, 8);
5055 if (IS_ERR(cs)) {
5056 err = PTR_ERR(cs);
5057 goto err_rq;
5058 }
5059
5060 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
5061 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
5062 *cs++ = lower_32_bits(batch->node.start);
5063 *cs++ = upper_32_bits(batch->node.start);
5064
5065 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
5066 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
5067 offset_in_page(sema);
5068 *cs++ = 0;
5069 *cs++ = 1;
5070
5071 intel_ring_advance(rq, cs);
5072
5073 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
5074err_rq:
5075 i915_request_add(rq);
5076err_batch:
5077 i915_vma_put(batch);
5078 return err;
5079}
5080
5081static bool is_moving(u32 a, u32 b)
5082{
5083 return a != b;
5084}
5085
5086static int compare_isolation(struct intel_engine_cs *engine,
5087 struct i915_vma *ref[2],
5088 struct i915_vma *result[2],
5089 struct intel_context *ce,
5090 u32 poison)
5091{
5092 u32 x, dw, *hw, *lrc;
5093 u32 *A[2], *B[2];
5094 int err = 0;
5095
5096 A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
5097 if (IS_ERR(A[0]))
5098 return PTR_ERR(A[0]);
5099
5100 A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
5101 if (IS_ERR(A[1])) {
5102 err = PTR_ERR(A[1]);
5103 goto err_A0;
5104 }
5105
5106 B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
5107 if (IS_ERR(B[0])) {
5108 err = PTR_ERR(B[0]);
5109 goto err_A1;
5110 }
5111
5112 B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
5113 if (IS_ERR(B[1])) {
5114 err = PTR_ERR(B[1]);
5115 goto err_B0;
5116 }
5117
5118 lrc = i915_gem_object_pin_map(ce->state->obj,
5119 i915_coherent_map_type(engine->i915));
5120 if (IS_ERR(lrc)) {
5121 err = PTR_ERR(lrc);
5122 goto err_B1;
5123 }
5124 lrc += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
5125
5126 x = 0;
5127 dw = 0;
5128 hw = engine->pinned_default_state;
5129 hw += LRC_STATE_PN * PAGE_SIZE / sizeof(*hw);
5130 do {
5131 u32 len = hw[dw] & 0x7f;
5132
5133 if (hw[dw] == 0) {
5134 dw++;
5135 continue;
5136 }
5137
5138 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
5139 dw += len + 2;
5140 continue;
5141 }
5142
5143 dw++;
5144 len = (len + 1) / 2;
5145 while (len--) {
5146 if (!is_moving(A[0][x], A[1][x]) &&
5147 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
5148 switch (hw[dw] & 4095) {
5149 case 0x30: /* RING_HEAD */
5150 case 0x34: /* RING_TAIL */
5151 break;
5152
5153 default:
5154 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
5155 engine->name, dw,
5156 hw[dw], hw[dw + 1],
5157 A[0][x], B[0][x], B[1][x],
5158 poison, lrc[dw + 1]);
5159 err = -EINVAL;
5160 break;
5161 }
5162 }
5163 dw += 2;
5164 x++;
5165 }
5166 } while (dw < PAGE_SIZE / sizeof(u32) &&
5167 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
5168
5169 i915_gem_object_unpin_map(ce->state->obj);
5170err_B1:
5171 i915_gem_object_unpin_map(result[1]->obj);
5172err_B0:
5173 i915_gem_object_unpin_map(result[0]->obj);
5174err_A1:
5175 i915_gem_object_unpin_map(ref[1]->obj);
5176err_A0:
5177 i915_gem_object_unpin_map(ref[0]->obj);
5178 return err;
5179}
5180
5181static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
5182{
5183 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
5184 struct i915_vma *ref[2], *result[2];
5185 struct intel_context *A, *B;
5186 struct i915_request *rq;
5187 int err;
5188
5189 A = intel_context_create(engine);
5190 if (IS_ERR(A))
5191 return PTR_ERR(A);
5192
5193 B = intel_context_create(engine);
5194 if (IS_ERR(B)) {
5195 err = PTR_ERR(B);
5196 goto err_A;
5197 }
5198
5199 ref[0] = create_user_vma(A->vm, SZ_64K);
5200 if (IS_ERR(ref[0])) {
5201 err = PTR_ERR(ref[0]);
5202 goto err_B;
5203 }
5204
5205 ref[1] = create_user_vma(A->vm, SZ_64K);
5206 if (IS_ERR(ref[1])) {
5207 err = PTR_ERR(ref[1]);
5208 goto err_ref0;
5209 }
5210
5211 rq = record_registers(A, ref[0], ref[1], sema);
5212 if (IS_ERR(rq)) {
5213 err = PTR_ERR(rq);
5214 goto err_ref1;
5215 }
5216
5217 WRITE_ONCE(*sema, 1);
5218 wmb();
5219
5220 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5221 i915_request_put(rq);
5222 err = -ETIME;
5223 goto err_ref1;
5224 }
5225 i915_request_put(rq);
5226
5227 result[0] = create_user_vma(A->vm, SZ_64K);
5228 if (IS_ERR(result[0])) {
5229 err = PTR_ERR(result[0]);
5230 goto err_ref1;
5231 }
5232
5233 result[1] = create_user_vma(A->vm, SZ_64K);
5234 if (IS_ERR(result[1])) {
5235 err = PTR_ERR(result[1]);
5236 goto err_result0;
5237 }
5238
5239 rq = record_registers(A, result[0], result[1], sema);
5240 if (IS_ERR(rq)) {
5241 err = PTR_ERR(rq);
5242 goto err_result1;
5243 }
5244
5245 err = poison_registers(B, poison, sema);
5246 if (err) {
5247 WRITE_ONCE(*sema, -1);
5248 i915_request_put(rq);
5249 goto err_result1;
5250 }
5251
5252 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
5253 i915_request_put(rq);
5254 err = -ETIME;
5255 goto err_result1;
5256 }
5257 i915_request_put(rq);
5258
5259 err = compare_isolation(engine, ref, result, A, poison);
5260
5261err_result1:
5262 i915_vma_put(result[1]);
5263err_result0:
5264 i915_vma_put(result[0]);
5265err_ref1:
5266 i915_vma_put(ref[1]);
5267err_ref0:
5268 i915_vma_put(ref[0]);
5269err_B:
5270 intel_context_put(B);
5271err_A:
5272 intel_context_put(A);
5273 return err;
5274}
5275
5276static bool skip_isolation(const struct intel_engine_cs *engine)
5277{
5278 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
5279 return true;
5280
5281 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
5282 return true;
5283
5284 return false;
5285}
5286
5287static int live_lrc_isolation(void *arg)
5288{
5289 struct intel_gt *gt = arg;
5290 struct intel_engine_cs *engine;
5291 enum intel_engine_id id;
5292 const u32 poison[] = {
5293 STACK_MAGIC,
5294 0x3a3a3a3a,
bb4328f6 5295 0x5c5c5c5c,
efb69b98 5296 0xffffffff,
bb4328f6 5297 0xffff0000,
efb69b98
CW		 5298 };
5299
5300 /*
5301 * Our goal is try and verify that per-context state cannot be
5302 * tampered with by another non-privileged client.
5303 *
5304 * We take the list of context registers from the LRI in the default
5305 * context image and attempt to modify that list from a remote context.
5306 */
5307
5308 for_each_engine(engine, gt, id) {
5309 int err = 0;
5310 int i;
5311
5312 /* Just don't even ask */
5313 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
5314 skip_isolation(engine))
5315 continue;
5316
5317 intel_engine_pm_get(engine);
5318 if (engine->pinned_default_state) {
5319 for (i = 0; i < ARRAY_SIZE(poison); i++) {
5320 err = __lrc_isolation(engine, poison[i]);
5321 if (err)
5322 break;
bb4328f6
CW		 5323
5324 err = __lrc_isolation(engine, ~poison[i]);
5325 if (err)
5326 break;
efb69b98
CW		 5327 }
5328 }
5329 intel_engine_pm_put(engine);
5330 if (igt_flush_test(gt->i915))
5331 err = -EIO;
5332 if (err)
5333 return err;
5334 }
5335
5336 return 0;
5337}
5338
24eba7a9
CW		 5339static void garbage_reset(struct intel_engine_cs *engine,
5340 struct i915_request *rq)
5341{
5342 const unsigned int bit = I915_RESET_ENGINE + engine->id;
5343 unsigned long *lock = &engine->gt->reset.flags;
5344
5345 if (test_and_set_bit(bit, lock))
5346 return;
5347
5348 tasklet_disable(&engine->execlists.tasklet);
5349
5350 if (!rq->fence.error)
5351 intel_engine_reset(engine, NULL);
5352
5353 tasklet_enable(&engine->execlists.tasklet);
5354 clear_and_wake_up_bit(bit, lock);
5355}
5356
5357static struct i915_request *garbage(struct intel_context *ce,
5358 struct rnd_state *prng)
5359{
5360 struct i915_request *rq;
5361 int err;
5362
5363 err = intel_context_pin(ce);
5364 if (err)
5365 return ERR_PTR(err);
5366
5367 prandom_bytes_state(prng,
5368 ce->lrc_reg_state,
5369 ce->engine->context_size -
5370 LRC_STATE_PN * PAGE_SIZE);
5371
5372 rq = intel_context_create_request(ce);
5373 if (IS_ERR(rq)) {
5374 err = PTR_ERR(rq);
5375 goto err_unpin;
5376 }
5377
5378 i915_request_get(rq);
5379 i915_request_add(rq);
5380 return rq;
5381
5382err_unpin:
5383 intel_context_unpin(ce);
5384 return ERR_PTR(err);
5385}
5386
5387static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
5388{
5389 struct intel_context *ce;
5390 struct i915_request *hang;
5391 int err = 0;
5392
5393 ce = intel_context_create(engine);
5394 if (IS_ERR(ce))
5395 return PTR_ERR(ce);
5396
5397 hang = garbage(ce, prng);
5398 if (IS_ERR(hang)) {
5399 err = PTR_ERR(hang);
5400 goto err_ce;
5401 }
5402
5403 if (wait_for_submit(engine, hang, HZ / 2)) {
5404 i915_request_put(hang);
5405 err = -ETIME;
5406 goto err_ce;
5407 }
5408
5409 intel_context_set_banned(ce);
5410 garbage_reset(engine, hang);
5411
5412 intel_engine_flush_submission(engine);
5413 if (!hang->fence.error) {
5414 i915_request_put(hang);
5415 pr_err("%s: corrupted context was not reset\n",
5416 engine->name);
5417 err = -EINVAL;
5418 goto err_ce;
5419 }
5420
5421 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
5422 pr_err("%s: corrupted context did not recover\n",
5423 engine->name);
5424 i915_request_put(hang);
5425 err = -EIO;
5426 goto err_ce;
5427 }
5428 i915_request_put(hang);
5429
5430err_ce:
5431 intel_context_put(ce);
5432 return err;
5433}
5434
5435static int live_lrc_garbage(void *arg)
5436{
5437 struct intel_gt *gt = arg;
5438 struct intel_engine_cs *engine;
5439 enum intel_engine_id id;
5440
5441 /*
5442 * Verify that we can recover if one context state is completely
5443 * corrupted.
5444 */
5445
5446 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
5447 return 0;
5448
5449 for_each_engine(engine, gt, id) {
5450 I915_RND_STATE(prng);
5451 int err = 0, i;
5452
5453 if (!intel_has_reset_engine(engine->gt))
5454 continue;
5455
5456 intel_engine_pm_get(engine);
5457 for (i = 0; i < 3; i++) {
5458 err = __lrc_garbage(engine, &prng);
5459 if (err)
5460 break;
5461 }
5462 intel_engine_pm_put(engine);
5463
5464 if (igt_flush_test(gt->i915))
5465 err = -EIO;
5466 if (err)
5467 return err;
5468 }
5469
5470 return 0;
5471}
5472
1883a0a4
TU		 5473static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
5474{
5475 struct intel_context *ce;
5476 struct i915_request *rq;
5477 IGT_TIMEOUT(end_time);
5478 int err;
5479
5480 ce = intel_context_create(engine);
5481 if (IS_ERR(ce))
5482 return PTR_ERR(ce);
5483
5484 ce->runtime.num_underflow = 0;
5485 ce->runtime.max_underflow = 0;
5486
5487 do {
5488 unsigned int loop = 1024;
5489
5490 while (loop) {
5491 rq = intel_context_create_request(ce);
5492 if (IS_ERR(rq)) {
5493 err = PTR_ERR(rq);
5494 goto err_rq;
5495 }
5496
5497 if (--loop == 0)
5498 i915_request_get(rq);
5499
5500 i915_request_add(rq);
5501 }
5502
5503 if (__igt_timeout(end_time, NULL))
5504 break;
5505
5506 i915_request_put(rq);
5507 } while (1);
5508
5509 err = i915_request_wait(rq, 0, HZ / 5);
5510 if (err < 0) {
5511 pr_err("%s: request not completed!\n", engine->name);
5512 goto err_wait;
5513 }
5514
5515 igt_flush_test(engine->i915);
5516
5517 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
5518 engine->name,
5519 intel_context_get_total_runtime_ns(ce),
5520 intel_context_get_avg_runtime_ns(ce));
5521
5522 err = 0;
5523 if (ce->runtime.num_underflow) {
5524 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
5525 engine->name,
5526 ce->runtime.num_underflow,
5527 ce->runtime.max_underflow);
5528 GEM_TRACE_DUMP();
5529 err = -EOVERFLOW;
5530 }
5531
5532err_wait:
5533 i915_request_put(rq);
5534err_rq:
5535 intel_context_put(ce);
5536 return err;
5537}
5538
5539static int live_pphwsp_runtime(void *arg)
5540{
5541 struct intel_gt *gt = arg;
5542 struct intel_engine_cs *engine;
5543 enum intel_engine_id id;
5544 int err = 0;
5545
5546 /*
5547 * Check that cumulative context runtime as stored in the pphwsp[16]
5548 * is monotonic.
5549 */
5550
5551 for_each_engine(engine, gt, id) {
5552 err = __live_pphwsp_runtime(engine);
5553 if (err)
5554 break;
5555 }
5556
5557 if (igt_flush_test(gt->i915))
5558 err = -EIO;
5559
5560 return err;
5561}
5562
7dc56af5
CW		 5563int intel_lrc_live_selftests(struct drm_i915_private *i915)
5564{
5565 static const struct i915_subtest tests[] = {
5566 SUBTEST(live_lrc_layout),
35865aef 5567 SUBTEST(live_lrc_fixed),
9c27462c 5568 SUBTEST(live_lrc_state),
bd3d1f86 5569 SUBTEST(live_lrc_gpr),
efb69b98 5570 SUBTEST(live_lrc_isolation),
e7aa531e 5571 SUBTEST(live_lrc_timestamp),
24eba7a9 5572 SUBTEST(live_lrc_garbage),
1883a0a4 5573 SUBTEST(live_pphwsp_runtime),
7dc56af5
CW		 5574 };
5575
5576 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
5577 return 0;
5578
5579 return intel_gt_live_subtests(tests, &i915->gt);
5580}
A mirror of Linus' kernel repository