]>
Commit | Line | Data |
---|---|---|
e4734057 ZW |
1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
21 | * SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Zhi Wang <zhi.a.wang@intel.com> | |
25 | * | |
26 | * Contributors: | |
27 | * Ping Gao <ping.a.gao@intel.com> | |
28 | * Tina Zhang <tina.zhang@intel.com> | |
29 | * Chanbin Du <changbin.du@intel.com> | |
30 | * Min He <min.he@intel.com> | |
31 | * Bing Niu <bing.niu@intel.com> | |
32 | * Zhenyu Wang <zhenyuw@linux.intel.com> | |
33 | * | |
34 | */ | |
35 | ||
e4734057 ZW |
36 | #include <linux/kthread.h> |
37 | ||
10be98a7 CW |
38 | #include "gem/i915_gem_pm.h" |
39 | #include "gt/intel_context.h" | |
2871ea85 | 40 | #include "gt/intel_ring.h" |
10be98a7 | 41 | |
feddf6e8 | 42 | #include "i915_drv.h" |
e6ba7648 | 43 | #include "i915_gem_gtt.h" |
feddf6e8 ZW |
44 | #include "gvt.h" |
45 | ||
e4734057 ZW |
46 | #define RING_CTX_OFF(x) \ |
47 | offsetof(struct execlist_ring_context, x) | |
48 | ||
999ccb40 CD |
49 | static void set_context_pdp_root_pointer( |
50 | struct execlist_ring_context *ring_context, | |
e4734057 ZW |
51 | u32 pdp[8]) |
52 | { | |
e4734057 ZW |
53 | int i; |
54 | ||
55 | for (i = 0; i < 8; i++) | |
1417fad7 | 56 | ring_context->pdps[i].val = pdp[7 - i]; |
e4734057 ZW |
57 | } |
58 | ||
b20c0d5c ZW |
59 | static void update_shadow_pdps(struct intel_vgpu_workload *workload) |
60 | { | |
b20c0d5c | 61 | struct drm_i915_gem_object *ctx_obj = |
9f3ccd40 | 62 | workload->req->context->state->obj; |
b20c0d5c ZW |
63 | struct execlist_ring_context *shadow_ring_context; |
64 | struct page *page; | |
65 | ||
66 | if (WARN_ON(!workload->shadow_mm)) | |
67 | return; | |
68 | ||
69 | if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount))) | |
70 | return; | |
71 | ||
72 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); | |
73 | shadow_ring_context = kmap(page); | |
74 | set_context_pdp_root_pointer(shadow_ring_context, | |
75 | (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps); | |
76 | kunmap(page); | |
77 | } | |
78 | ||
fa3dd623 MH |
79 | /* |
80 | * when populating shadow ctx from guest, we should not overrride oa related | |
81 | * registers, so that they will not be overlapped by guest oa configs. Thus | |
82 | * made it possible to capture oa data from host for both host and guests. | |
83 | */ | |
84 | static void sr_oa_regs(struct intel_vgpu_workload *workload, | |
85 | u32 *reg_state, bool save) | |
86 | { | |
a61ac1e7 | 87 | struct drm_i915_private *dev_priv = workload->vgpu->gvt->gt->i915; |
a37f08a8 UNR |
88 | u32 ctx_oactxctrl = dev_priv->perf.ctx_oactxctrl_offset; |
89 | u32 ctx_flexeu0 = dev_priv->perf.ctx_flexeu0_offset; | |
fa3dd623 MH |
90 | int i = 0; |
91 | u32 flex_mmio[] = { | |
92 | i915_mmio_reg_offset(EU_PERF_CNTL0), | |
93 | i915_mmio_reg_offset(EU_PERF_CNTL1), | |
94 | i915_mmio_reg_offset(EU_PERF_CNTL2), | |
95 | i915_mmio_reg_offset(EU_PERF_CNTL3), | |
96 | i915_mmio_reg_offset(EU_PERF_CNTL4), | |
97 | i915_mmio_reg_offset(EU_PERF_CNTL5), | |
98 | i915_mmio_reg_offset(EU_PERF_CNTL6), | |
99 | }; | |
100 | ||
8fde4107 | 101 | if (workload->engine->id != RCS0) |
fa3dd623 MH |
102 | return; |
103 | ||
104 | if (save) { | |
105 | workload->oactxctrl = reg_state[ctx_oactxctrl + 1]; | |
106 | ||
107 | for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { | |
108 | u32 state_offset = ctx_flexeu0 + i * 2; | |
109 | ||
110 | workload->flex_mmio[i] = reg_state[state_offset + 1]; | |
111 | } | |
112 | } else { | |
113 | reg_state[ctx_oactxctrl] = | |
114 | i915_mmio_reg_offset(GEN8_OACTXCONTROL); | |
115 | reg_state[ctx_oactxctrl + 1] = workload->oactxctrl; | |
116 | ||
117 | for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { | |
118 | u32 state_offset = ctx_flexeu0 + i * 2; | |
119 | u32 mmio = flex_mmio[i]; | |
120 | ||
121 | reg_state[state_offset] = mmio; | |
122 | reg_state[state_offset + 1] = workload->flex_mmio[i]; | |
123 | } | |
124 | } | |
125 | } | |
126 | ||
e4734057 ZW |
127 | static int populate_shadow_context(struct intel_vgpu_workload *workload) |
128 | { | |
129 | struct intel_vgpu *vgpu = workload->vgpu; | |
130 | struct intel_gvt *gvt = vgpu->gvt; | |
e4734057 | 131 | struct drm_i915_gem_object *ctx_obj = |
9f3ccd40 | 132 | workload->req->context->state->obj; |
e4734057 ZW |
133 | struct execlist_ring_context *shadow_ring_context; |
134 | struct page *page; | |
135 | void *dst; | |
136 | unsigned long context_gpa, context_page_num; | |
137 | int i; | |
138 | ||
e4734057 | 139 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); |
c754936f | 140 | shadow_ring_context = kmap(page); |
e4734057 | 141 | |
fa3dd623 | 142 | sr_oa_regs(workload, (u32 *)shadow_ring_context, true); |
e4734057 ZW |
143 | #define COPY_REG(name) \ |
144 | intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ | |
145 | + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) | |
d8303075 ZW |
146 | #define COPY_REG_MASKED(name) {\ |
147 | intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ | |
148 | + RING_CTX_OFF(name.val),\ | |
149 | &shadow_ring_context->name.val, 4);\ | |
150 | shadow_ring_context->name.val |= 0xffff << 16;\ | |
151 | } | |
e4734057 | 152 | |
d8303075 | 153 | COPY_REG_MASKED(ctx_ctrl); |
e4734057 ZW |
154 | COPY_REG(ctx_timestamp); |
155 | ||
8fde4107 | 156 | if (workload->engine->id == RCS0) { |
e4734057 ZW |
157 | COPY_REG(bb_per_ctx_ptr); |
158 | COPY_REG(rcs_indirect_ctx); | |
159 | COPY_REG(rcs_indirect_ctx_offset); | |
160 | } | |
161 | #undef COPY_REG | |
d8303075 | 162 | #undef COPY_REG_MASKED |
e4734057 | 163 | |
e4734057 ZW |
164 | intel_gvt_hypervisor_read_gpa(vgpu, |
165 | workload->ring_context_gpa + | |
166 | sizeof(*shadow_ring_context), | |
167 | (void *)shadow_ring_context + | |
168 | sizeof(*shadow_ring_context), | |
9556e118 | 169 | I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); |
e4734057 | 170 | |
fa3dd623 | 171 | sr_oa_regs(workload, (u32 *)shadow_ring_context, false); |
c754936f | 172 | kunmap(page); |
8bfa02c8 ZY |
173 | |
174 | if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val)) | |
175 | return 0; | |
176 | ||
8fde4107 CW |
177 | gvt_dbg_sched("ring %s workload lrca %x", |
178 | workload->engine->name, | |
179 | workload->ctx_desc.lrca); | |
8bfa02c8 | 180 | |
8fde4107 | 181 | context_page_num = workload->engine->context_size; |
8bfa02c8 ZY |
182 | context_page_num = context_page_num >> PAGE_SHIFT; |
183 | ||
a61ac1e7 | 184 | if (IS_BROADWELL(gvt->gt->i915) && workload->engine->id == RCS0) |
8bfa02c8 ZY |
185 | context_page_num = 19; |
186 | ||
187 | i = 2; | |
188 | while (i < context_page_num) { | |
189 | context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, | |
190 | (u32)((workload->ctx_desc.lrca + i) << | |
191 | I915_GTT_PAGE_SHIFT)); | |
192 | if (context_gpa == INTEL_GVT_INVALID_ADDR) { | |
193 | gvt_vgpu_err("Invalid guest context descriptor\n"); | |
194 | return -EFAULT; | |
195 | } | |
196 | ||
9f379407 | 197 | page = i915_gem_object_get_page(ctx_obj, i); |
8bfa02c8 ZY |
198 | dst = kmap(page); |
199 | intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, | |
200 | I915_GTT_PAGE_SIZE); | |
201 | kunmap(page); | |
202 | i++; | |
203 | } | |
e4734057 ZW |
204 | return 0; |
205 | } | |
206 | ||
9f3ccd40 | 207 | static inline bool is_gvt_request(struct i915_request *rq) |
bc2d4b62 | 208 | { |
9f3ccd40 | 209 | return intel_context_force_single_submission(rq->context); |
bc2d4b62 CD |
210 | } |
211 | ||
8fde4107 CW |
212 | static void save_ring_hw_state(struct intel_vgpu *vgpu, |
213 | const struct intel_engine_cs *engine) | |
295764cd | 214 | { |
8fde4107 | 215 | struct intel_uncore *uncore = engine->uncore; |
295764cd XZ |
216 | i915_reg_t reg; |
217 | ||
8fde4107 CW |
218 | reg = RING_INSTDONE(engine->mmio_base); |
219 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = | |
220 | intel_uncore_read(uncore, reg); | |
221 | ||
222 | reg = RING_ACTHD(engine->mmio_base); | |
223 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = | |
224 | intel_uncore_read(uncore, reg); | |
225 | ||
226 | reg = RING_ACTHD_UDW(engine->mmio_base); | |
227 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = | |
228 | intel_uncore_read(uncore, reg); | |
295764cd XZ |
229 | } |
230 | ||
e4734057 ZW |
231 | static int shadow_context_status_change(struct notifier_block *nb, |
232 | unsigned long action, void *data) | |
233 | { | |
8fde4107 | 234 | struct i915_request *rq = data; |
3fc03069 | 235 | struct intel_gvt *gvt = container_of(nb, struct intel_gvt, |
8fde4107 | 236 | shadow_ctx_notifier_block[rq->engine->id]); |
3fc03069 | 237 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; |
8fde4107 | 238 | enum intel_engine_id ring_id = rq->engine->id; |
0e86cc9c | 239 | struct intel_vgpu_workload *workload; |
679fd3eb | 240 | unsigned long flags; |
0e86cc9c | 241 | |
8fde4107 | 242 | if (!is_gvt_request(rq)) { |
679fd3eb | 243 | spin_lock_irqsave(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
244 | if (action == INTEL_CONTEXT_SCHEDULE_IN && |
245 | scheduler->engine_owner[ring_id]) { | |
246 | /* Switch ring from vGPU to host. */ | |
247 | intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], | |
8fde4107 | 248 | NULL, rq->engine); |
0e86cc9c CD |
249 | scheduler->engine_owner[ring_id] = NULL; |
250 | } | |
679fd3eb | 251 | spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
252 | |
253 | return NOTIFY_OK; | |
254 | } | |
e4734057 | 255 | |
0e86cc9c CD |
256 | workload = scheduler->current_workload[ring_id]; |
257 | if (unlikely(!workload)) | |
9272f73f CD |
258 | return NOTIFY_OK; |
259 | ||
e4734057 ZW |
260 | switch (action) { |
261 | case INTEL_CONTEXT_SCHEDULE_IN: | |
679fd3eb | 262 | spin_lock_irqsave(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
263 | if (workload->vgpu != scheduler->engine_owner[ring_id]) { |
264 | /* Switch ring from host to vGPU or vGPU to vGPU. */ | |
265 | intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], | |
8fde4107 | 266 | workload->vgpu, rq->engine); |
0e86cc9c CD |
267 | scheduler->engine_owner[ring_id] = workload->vgpu; |
268 | } else | |
269 | gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n", | |
270 | ring_id, workload->vgpu->id); | |
679fd3eb | 271 | spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); |
e4734057 ZW |
272 | atomic_set(&workload->shadow_ctx_active, 1); |
273 | break; | |
274 | case INTEL_CONTEXT_SCHEDULE_OUT: | |
8fde4107 | 275 | save_ring_hw_state(workload->vgpu, rq->engine); |
e4734057 ZW |
276 | atomic_set(&workload->shadow_ctx_active, 0); |
277 | break; | |
da5f99ea | 278 | case INTEL_CONTEXT_SCHEDULE_PREEMPTED: |
8fde4107 | 279 | save_ring_hw_state(workload->vgpu, rq->engine); |
da5f99ea | 280 | break; |
e4734057 ZW |
281 | default: |
282 | WARN_ON(1); | |
283 | return NOTIFY_OK; | |
284 | } | |
285 | wake_up(&workload->shadow_ctx_status_wq); | |
286 | return NOTIFY_OK; | |
287 | } | |
288 | ||
251d46b0 CW |
289 | static void |
290 | shadow_context_descriptor_update(struct intel_context *ce, | |
291 | struct intel_vgpu_workload *workload) | |
9dfb8e5b | 292 | { |
53b2622e | 293 | u64 desc = ce->lrc.desc; |
9dfb8e5b | 294 | |
251d46b0 CW |
295 | /* |
296 | * Update bits 0-11 of the context descriptor which includes flags | |
9dfb8e5b KL |
297 | * like GEN8_CTX_* cached in desc_template |
298 | */ | |
eb0ff807 TZ |
299 | desc &= ~(0x3ull << GEN8_CTX_ADDRESSING_MODE_SHIFT); |
300 | desc |= (u64)workload->ctx_desc.addressing_mode << | |
251d46b0 CW |
301 | GEN8_CTX_ADDRESSING_MODE_SHIFT; |
302 | ||
53b2622e | 303 | ce->lrc.desc = desc; |
9dfb8e5b KL |
304 | } |
305 | ||
0a53bc07 | 306 | static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) |
307 | { | |
308 | struct intel_vgpu *vgpu = workload->vgpu; | |
1fc44d9b | 309 | struct i915_request *req = workload->req; |
0a53bc07 | 310 | void *shadow_ring_buffer_va; |
311 | u32 *cs; | |
a8c2d5ab | 312 | int err; |
cd7e61b9 | 313 | |
9f3ccd40 | 314 | if (IS_GEN(req->i915, 9) && is_inhibit_context(req->context)) |
cd7e61b9 | 315 | intel_vgpu_restore_inhibit_context(vgpu, req); |
0a53bc07 | 316 | |
a8c2d5ab W |
317 | /* |
318 | * To track whether a request has started on HW, we can emit a | |
319 | * breadcrumb at the beginning of the request and check its | |
320 | * timeline's HWSP to see if the breadcrumb has advanced past the | |
321 | * start of this request. Actually, the request must have the | |
322 | * init_breadcrumb if its timeline set has_init_bread_crumb, or the | |
323 | * scheduler might get a wrong state of it during reset. Since the | |
324 | * requests from gvt always set the has_init_breadcrumb flag, here | |
325 | * need to do the emit_init_breadcrumb for all the requests. | |
326 | */ | |
327 | if (req->engine->emit_init_breadcrumb) { | |
328 | err = req->engine->emit_init_breadcrumb(req); | |
329 | if (err) { | |
330 | gvt_vgpu_err("fail to emit init breadcrumb\n"); | |
331 | return err; | |
332 | } | |
333 | } | |
334 | ||
0a53bc07 | 335 | /* allocate shadow ring buffer */ |
336 | cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32)); | |
337 | if (IS_ERR(cs)) { | |
338 | gvt_vgpu_err("fail to alloc size =%ld shadow ring buffer\n", | |
339 | workload->rb_len); | |
340 | return PTR_ERR(cs); | |
341 | } | |
342 | ||
343 | shadow_ring_buffer_va = workload->shadow_ring_buffer_va; | |
344 | ||
345 | /* get shadow ring buffer va */ | |
346 | workload->shadow_ring_buffer_va = cs; | |
347 | ||
348 | memcpy(cs, shadow_ring_buffer_va, | |
349 | workload->rb_len); | |
350 | ||
351 | cs += workload->rb_len / sizeof(u32); | |
352 | intel_ring_advance(workload->req, cs); | |
353 | ||
354 | return 0; | |
355 | } | |
356 | ||
7b302556 | 357 | static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
a3cfdca9 | 358 | { |
359 | if (!wa_ctx->indirect_ctx.obj) | |
360 | return; | |
361 | ||
362 | i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); | |
363 | i915_gem_object_put(wa_ctx->indirect_ctx.obj); | |
0f755512 WL |
364 | |
365 | wa_ctx->indirect_ctx.obj = NULL; | |
366 | wa_ctx->indirect_ctx.shadow_va = NULL; | |
a3cfdca9 | 367 | } |
368 | ||
41874148 | 369 | static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, |
9f3ccd40 | 370 | struct intel_context *ce) |
4f15665c XZ |
371 | { |
372 | struct intel_vgpu_mm *mm = workload->shadow_mm; | |
9f3ccd40 | 373 | struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ce->vm); |
4f15665c XZ |
374 | int i = 0; |
375 | ||
4f15665c | 376 | if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { |
b5b7bef9 | 377 | px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0]; |
4f15665c XZ |
378 | } else { |
379 | for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { | |
b5b7bef9 MK |
380 | struct i915_page_directory * const pd = |
381 | i915_pd_entry(ppgtt->pd, i); | |
72a7a992 ZW |
382 | /* skip now as current i915 ppgtt alloc won't allocate |
383 | top level pdp for non 4-level table, won't impact | |
384 | shadow ppgtt. */ | |
385 | if (!pd) | |
386 | break; | |
b5b7bef9 | 387 | px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i]; |
4f15665c XZ |
388 | } |
389 | } | |
4f15665c XZ |
390 | } |
391 | ||
f0e99437 ZW |
392 | static int |
393 | intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) | |
394 | { | |
395 | struct intel_vgpu *vgpu = workload->vgpu; | |
396 | struct intel_vgpu_submission *s = &vgpu->submission; | |
f0e99437 | 397 | struct i915_request *rq; |
f0e99437 | 398 | |
f0e99437 | 399 | if (workload->req) |
251d46b0 | 400 | return 0; |
f0e99437 | 401 | |
8fde4107 | 402 | rq = i915_request_create(s->shadow[workload->engine->id]); |
f0e99437 ZW |
403 | if (IS_ERR(rq)) { |
404 | gvt_vgpu_err("fail to allocate gem request\n"); | |
251d46b0 | 405 | return PTR_ERR(rq); |
f0e99437 | 406 | } |
251d46b0 | 407 | |
f0e99437 | 408 | workload->req = i915_request_get(rq); |
251d46b0 | 409 | return 0; |
f0e99437 ZW |
410 | } |
411 | ||
89ea20b9 PG |
412 | /** |
413 | * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and | |
414 | * shadow it as well, include ringbuffer,wa_ctx and ctx. | |
415 | * @workload: an abstract entity for each execlist submission. | |
416 | * | |
417 | * This function is called before the workload submitting to i915, to make | |
418 | * sure the content of the workload is valid. | |
419 | */ | |
420 | int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) | |
e4734057 | 421 | { |
1406a14b ZW |
422 | struct intel_vgpu *vgpu = workload->vgpu; |
423 | struct intel_vgpu_submission *s = &vgpu->submission; | |
e4734057 ZW |
424 | int ret; |
425 | ||
952f89f0 | 426 | lockdep_assert_held(&vgpu->vgpu_lock); |
87e919d7 | 427 | |
f0e99437 | 428 | if (workload->shadow) |
d0302e74 | 429 | return 0; |
e4734057 | 430 | |
8fde4107 CW |
431 | if (!test_and_set_bit(workload->engine->id, s->shadow_ctx_desc_updated)) |
432 | shadow_context_descriptor_update(s->shadow[workload->engine->id], | |
251d46b0 | 433 | workload); |
3cd23b82 | 434 | |
89ea20b9 | 435 | ret = intel_gvt_scan_and_shadow_ringbuffer(workload); |
be1da707 | 436 | if (ret) |
251d46b0 | 437 | return ret; |
be1da707 | 438 | |
8fde4107 CW |
439 | if (workload->engine->id == RCS0 && |
440 | workload->wa_ctx.indirect_ctx.size) { | |
17f1b1a6 TZ |
441 | ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx); |
442 | if (ret) | |
1fc44d9b | 443 | goto err_shadow; |
17f1b1a6 | 444 | } |
be1da707 | 445 | |
f0e99437 | 446 | workload->shadow = true; |
6bb2a2af | 447 | return 0; |
8fde4107 | 448 | |
f2880e04 | 449 | err_shadow: |
450 | release_shadow_wa_ctx(&workload->wa_ctx); | |
89ea20b9 PG |
451 | return ret; |
452 | } | |
453 | ||
f52c380a ZW |
454 | static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload); |
455 | ||
d8235b5e ZW |
456 | static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) |
457 | { | |
458 | struct intel_gvt *gvt = workload->vgpu->gvt; | |
459 | const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; | |
f52c380a ZW |
460 | struct intel_vgpu_shadow_bb *bb; |
461 | int ret; | |
462 | ||
463 | list_for_each_entry(bb, &workload->shadow_bb, list) { | |
ef75c685 | 464 | /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va |
465 | * is only updated into ring_scan_buffer, not real ring address | |
466 | * allocated in later copy_workload_to_ring_buffer. pls be noted | |
467 | * shadow_ring_buffer_va is now pointed to real ring buffer va | |
468 | * in copy_workload_to_ring_buffer. | |
469 | */ | |
470 | ||
471 | if (bb->bb_offset) | |
472 | bb->bb_start_cmd_va = workload->shadow_ring_buffer_va | |
473 | + bb->bb_offset; | |
474 | ||
96bebe39 ZY |
475 | if (bb->ppgtt) { |
476 | /* for non-priv bb, scan&shadow is only for | |
477 | * debugging purpose, so the content of shadow bb | |
478 | * is the same as original bb. Therefore, | |
479 | * here, rather than switch to shadow bb's gma | |
480 | * address, we directly use original batch buffer's | |
481 | * gma address, and send original bb to hardware | |
482 | * directly | |
483 | */ | |
484 | if (bb->clflush & CLFLUSH_AFTER) { | |
485 | drm_clflush_virt_range(bb->va, | |
486 | bb->obj->base.size); | |
487 | bb->clflush &= ~CLFLUSH_AFTER; | |
488 | } | |
f0e4a063 | 489 | i915_gem_object_finish_access(bb->obj); |
96bebe39 ZY |
490 | bb->accessing = false; |
491 | ||
492 | } else { | |
493 | bb->vma = i915_gem_object_ggtt_pin(bb->obj, | |
494 | NULL, 0, 0, 0); | |
495 | if (IS_ERR(bb->vma)) { | |
496 | ret = PTR_ERR(bb->vma); | |
497 | goto err; | |
498 | } | |
499 | ||
500 | /* relocate shadow batch buffer */ | |
501 | bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma); | |
502 | if (gmadr_bytes == 8) | |
503 | bb->bb_start_cmd_va[2] = 0; | |
504 | ||
505 | /* No one is going to touch shadow bb from now on. */ | |
506 | if (bb->clflush & CLFLUSH_AFTER) { | |
507 | drm_clflush_virt_range(bb->va, | |
508 | bb->obj->base.size); | |
509 | bb->clflush &= ~CLFLUSH_AFTER; | |
510 | } | |
511 | ||
512 | ret = i915_gem_object_set_to_gtt_domain(bb->obj, | |
6951e589 | 513 | false); |
96bebe39 ZY |
514 | if (ret) |
515 | goto err; | |
516 | ||
a5236978 CW |
517 | ret = i915_vma_move_to_active(bb->vma, |
518 | workload->req, | |
519 | 0); | |
520 | if (ret) | |
521 | goto err; | |
6951e589 CW |
522 | |
523 | i915_gem_object_finish_access(bb->obj); | |
524 | bb->accessing = false; | |
f52c380a | 525 | } |
d8235b5e ZW |
526 | } |
527 | return 0; | |
f52c380a ZW |
528 | err: |
529 | release_shadow_batch_buffer(workload); | |
530 | return ret; | |
d8235b5e ZW |
531 | } |
532 | ||
1fc44d9b | 533 | static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
d8235b5e | 534 | { |
1fc44d9b CW |
535 | struct intel_vgpu_workload *workload = |
536 | container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx); | |
537 | struct i915_request *rq = workload->req; | |
538 | struct execlist_ring_context *shadow_ring_context = | |
9f3ccd40 | 539 | (struct execlist_ring_context *)rq->context->lrc_reg_state; |
d8235b5e ZW |
540 | |
541 | shadow_ring_context->bb_per_ctx_ptr.val = | |
542 | (shadow_ring_context->bb_per_ctx_ptr.val & | |
543 | (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; | |
544 | shadow_ring_context->rcs_indirect_ctx.val = | |
545 | (shadow_ring_context->rcs_indirect_ctx.val & | |
546 | (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; | |
d8235b5e ZW |
547 | } |
548 | ||
549 | static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) | |
550 | { | |
551 | struct i915_vma *vma; | |
552 | unsigned char *per_ctx_va = | |
553 | (unsigned char *)wa_ctx->indirect_ctx.shadow_va + | |
554 | wa_ctx->indirect_ctx.size; | |
555 | ||
556 | if (wa_ctx->indirect_ctx.size == 0) | |
557 | return 0; | |
558 | ||
559 | vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, | |
560 | 0, CACHELINE_BYTES, 0); | |
561 | if (IS_ERR(vma)) | |
562 | return PTR_ERR(vma); | |
563 | ||
564 | /* FIXME: we are not tracking our pinned VMA leaving it | |
565 | * up to the core to fix up the stray pin_count upon | |
566 | * free. | |
567 | */ | |
568 | ||
569 | wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); | |
570 | ||
571 | wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); | |
572 | memset(per_ctx_va, 0, CACHELINE_BYTES); | |
573 | ||
574 | update_wa_ctx_2_shadow_ctx(wa_ctx); | |
575 | return 0; | |
576 | } | |
577 | ||
4a532256 WL |
578 | static void update_vreg_in_ctx(struct intel_vgpu_workload *workload) |
579 | { | |
8fde4107 CW |
580 | vgpu_vreg_t(workload->vgpu, RING_START(workload->engine->mmio_base)) = |
581 | workload->rb_start; | |
4a532256 WL |
582 | } |
583 | ||
d8235b5e ZW |
584 | static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) |
585 | { | |
f52c380a ZW |
586 | struct intel_vgpu_shadow_bb *bb, *pos; |
587 | ||
588 | if (list_empty(&workload->shadow_bb)) | |
589 | return; | |
590 | ||
591 | bb = list_first_entry(&workload->shadow_bb, | |
592 | struct intel_vgpu_shadow_bb, list); | |
593 | ||
f52c380a ZW |
594 | list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) { |
595 | if (bb->obj) { | |
596 | if (bb->accessing) | |
f0e4a063 | 597 | i915_gem_object_finish_access(bb->obj); |
f52c380a ZW |
598 | |
599 | if (bb->va && !IS_ERR(bb->va)) | |
600 | i915_gem_object_unpin_map(bb->obj); | |
601 | ||
602 | if (bb->vma && !IS_ERR(bb->vma)) { | |
603 | i915_vma_unpin(bb->vma); | |
604 | i915_vma_close(bb->vma); | |
605 | } | |
c017cf6b | 606 | i915_gem_object_put(bb->obj); |
d8235b5e | 607 | } |
f52c380a ZW |
608 | list_del(&bb->list); |
609 | kfree(bb); | |
d8235b5e ZW |
610 | } |
611 | } | |
612 | ||
497aa3f5 ZW |
613 | static int prepare_workload(struct intel_vgpu_workload *workload) |
614 | { | |
d8235b5e | 615 | struct intel_vgpu *vgpu = workload->vgpu; |
41874148 | 616 | struct intel_vgpu_submission *s = &vgpu->submission; |
497aa3f5 ZW |
617 | int ret = 0; |
618 | ||
d8235b5e ZW |
619 | ret = intel_vgpu_pin_mm(workload->shadow_mm); |
620 | if (ret) { | |
621 | gvt_vgpu_err("fail to vgpu pin mm\n"); | |
622 | return ret; | |
623 | } | |
624 | ||
41874148 CX |
625 | if (workload->shadow_mm->type != INTEL_GVT_MM_PPGTT || |
626 | !workload->shadow_mm->ppgtt_mm.shadowed) { | |
627 | gvt_vgpu_err("workload shadow ppgtt isn't ready\n"); | |
628 | return -EINVAL; | |
629 | } | |
630 | ||
b20c0d5c ZW |
631 | update_shadow_pdps(workload); |
632 | ||
8fde4107 | 633 | set_context_ppgtt_from_shadow(workload, s->shadow[workload->engine->id]); |
41874148 | 634 | |
d8235b5e ZW |
635 | ret = intel_vgpu_sync_oos_pages(workload->vgpu); |
636 | if (ret) { | |
637 | gvt_vgpu_err("fail to vgpu sync oos pages\n"); | |
638 | goto err_unpin_mm; | |
639 | } | |
640 | ||
641 | ret = intel_vgpu_flush_post_shadow(workload->vgpu); | |
642 | if (ret) { | |
643 | gvt_vgpu_err("fail to flush post shadow\n"); | |
644 | goto err_unpin_mm; | |
645 | } | |
646 | ||
6bb2a2af | 647 | ret = copy_workload_to_ring_buffer(workload); |
f2880e04 | 648 | if (ret) { |
649 | gvt_vgpu_err("fail to generate request\n"); | |
650 | goto err_unpin_mm; | |
651 | } | |
652 | ||
d8235b5e ZW |
653 | ret = prepare_shadow_batch_buffer(workload); |
654 | if (ret) { | |
655 | gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n"); | |
656 | goto err_unpin_mm; | |
657 | } | |
658 | ||
659 | ret = prepare_shadow_wa_ctx(&workload->wa_ctx); | |
660 | if (ret) { | |
661 | gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n"); | |
662 | goto err_shadow_batch; | |
663 | } | |
664 | ||
665 | if (workload->prepare) { | |
497aa3f5 | 666 | ret = workload->prepare(workload); |
d8235b5e ZW |
667 | if (ret) |
668 | goto err_shadow_wa_ctx; | |
669 | } | |
497aa3f5 | 670 | |
d8235b5e ZW |
671 | return 0; |
672 | err_shadow_wa_ctx: | |
673 | release_shadow_wa_ctx(&workload->wa_ctx); | |
674 | err_shadow_batch: | |
675 | release_shadow_batch_buffer(workload); | |
676 | err_unpin_mm: | |
677 | intel_vgpu_unpin_mm(workload->shadow_mm); | |
497aa3f5 ZW |
678 | return ret; |
679 | } | |
680 | ||
89ea20b9 PG |
681 | static int dispatch_workload(struct intel_vgpu_workload *workload) |
682 | { | |
1406a14b | 683 | struct intel_vgpu *vgpu = workload->vgpu; |
f552e7bd | 684 | struct i915_request *rq; |
1fc44d9b | 685 | int ret; |
89ea20b9 | 686 | |
8fde4107 CW |
687 | gvt_dbg_sched("ring id %s prepare to dispatch workload %p\n", |
688 | workload->engine->name, workload); | |
89ea20b9 | 689 | |
f25a49ab | 690 | mutex_lock(&vgpu->vgpu_lock); |
89ea20b9 | 691 | |
f0e99437 ZW |
692 | ret = intel_gvt_workload_req_alloc(workload); |
693 | if (ret) | |
694 | goto err_req; | |
695 | ||
89ea20b9 | 696 | ret = intel_gvt_scan_and_shadow_workload(workload); |
e4734057 | 697 | if (ret) |
90d27a1b | 698 | goto out; |
e4734057 | 699 | |
f0e99437 ZW |
700 | ret = populate_shadow_context(workload); |
701 | if (ret) { | |
702 | release_shadow_wa_ctx(&workload->wa_ctx); | |
703 | goto out; | |
704 | } | |
e4734057 | 705 | |
f0e99437 | 706 | ret = prepare_workload(workload); |
90d27a1b | 707 | out: |
f552e7bd ZW |
708 | if (ret) { |
709 | /* We might still need to add request with | |
710 | * clean ctx to retire it properly.. | |
711 | */ | |
712 | rq = fetch_and_zero(&workload->req); | |
713 | i915_request_put(rq); | |
714 | } | |
715 | ||
89ea20b9 | 716 | if (!IS_ERR_OR_NULL(workload->req)) { |
8fde4107 CW |
717 | gvt_dbg_sched("ring id %s submit workload to i915 %p\n", |
718 | workload->engine->name, workload->req); | |
e61e0f51 | 719 | i915_request_add(workload->req); |
89ea20b9 PG |
720 | workload->dispatched = true; |
721 | } | |
f0e99437 ZW |
722 | err_req: |
723 | if (ret) | |
724 | workload->status = ret; | |
f25a49ab | 725 | mutex_unlock(&vgpu->vgpu_lock); |
e4734057 ZW |
726 | return ret; |
727 | } | |
728 | ||
8fde4107 CW |
729 | static struct intel_vgpu_workload * |
730 | pick_next_workload(struct intel_gvt *gvt, struct intel_engine_cs *engine) | |
e4734057 ZW |
731 | { |
732 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
733 | struct intel_vgpu_workload *workload = NULL; | |
734 | ||
9a512e23 | 735 | mutex_lock(&gvt->sched_lock); |
e4734057 ZW |
736 | |
737 | /* | |
738 | * no current vgpu / will be scheduled out / no workload | |
739 | * bail out | |
740 | */ | |
741 | if (!scheduler->current_vgpu) { | |
8fde4107 | 742 | gvt_dbg_sched("ring %s stop - no current vgpu\n", engine->name); |
e4734057 ZW |
743 | goto out; |
744 | } | |
745 | ||
746 | if (scheduler->need_reschedule) { | |
8fde4107 | 747 | gvt_dbg_sched("ring %s stop - will reschedule\n", engine->name); |
e4734057 ZW |
748 | goto out; |
749 | } | |
750 | ||
9f498477 | 751 | if (!scheduler->current_vgpu->active || |
8fde4107 | 752 | list_empty(workload_q_head(scheduler->current_vgpu, engine))) |
e4734057 | 753 | goto out; |
e4734057 ZW |
754 | |
755 | /* | |
756 | * still have current workload, maybe the workload disptacher | |
757 | * fail to submit it for some reason, resubmit it. | |
758 | */ | |
8fde4107 CW |
759 | if (scheduler->current_workload[engine->id]) { |
760 | workload = scheduler->current_workload[engine->id]; | |
761 | gvt_dbg_sched("ring %s still have current workload %p\n", | |
762 | engine->name, workload); | |
e4734057 ZW |
763 | goto out; |
764 | } | |
765 | ||
766 | /* | |
767 | * pick a workload as current workload | |
768 | * once current workload is set, schedule policy routines | |
769 | * will wait the current workload is finished when trying to | |
770 | * schedule out a vgpu. | |
771 | */ | |
8fde4107 CW |
772 | scheduler->current_workload[engine->id] = |
773 | list_first_entry(workload_q_head(scheduler->current_vgpu, | |
774 | engine), | |
775 | struct intel_vgpu_workload, list); | |
e4734057 | 776 | |
8fde4107 | 777 | workload = scheduler->current_workload[engine->id]; |
e4734057 | 778 | |
8fde4107 | 779 | gvt_dbg_sched("ring %s pick new workload %p\n", engine->name, workload); |
e4734057 | 780 | |
1406a14b | 781 | atomic_inc(&workload->vgpu->submission.running_workload_num); |
e4734057 | 782 | out: |
9a512e23 | 783 | mutex_unlock(&gvt->sched_lock); |
e4734057 ZW |
784 | return workload; |
785 | } | |
786 | ||
787 | static void update_guest_context(struct intel_vgpu_workload *workload) | |
788 | { | |
1fc44d9b | 789 | struct i915_request *rq = workload->req; |
e4734057 | 790 | struct intel_vgpu *vgpu = workload->vgpu; |
9f3ccd40 | 791 | struct drm_i915_gem_object *ctx_obj = rq->context->state->obj; |
e4734057 ZW |
792 | struct execlist_ring_context *shadow_ring_context; |
793 | struct page *page; | |
794 | void *src; | |
795 | unsigned long context_gpa, context_page_num; | |
796 | int i; | |
15e7f52a XZ |
797 | u32 ring_base; |
798 | u32 head, tail; | |
799 | u16 wrap_count; | |
e4734057 | 800 | |
1fc44d9b CW |
801 | gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id, |
802 | workload->ctx_desc.lrca); | |
e4734057 | 803 | |
15e7f52a XZ |
804 | head = workload->rb_head; |
805 | tail = workload->rb_tail; | |
806 | wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF; | |
807 | ||
808 | if (tail < head) { | |
809 | if (wrap_count == RB_HEAD_WRAP_CNT_MAX) | |
810 | wrap_count = 0; | |
811 | else | |
812 | wrap_count += 1; | |
813 | } | |
814 | ||
815 | head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail; | |
816 | ||
8fde4107 | 817 | ring_base = rq->engine->mmio_base; |
15e7f52a XZ |
818 | vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail; |
819 | vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head; | |
820 | ||
1fc44d9b | 821 | context_page_num = rq->engine->context_size; |
e4734057 ZW |
822 | context_page_num = context_page_num >> PAGE_SHIFT; |
823 | ||
8fde4107 | 824 | if (IS_BROADWELL(rq->i915) && rq->engine->id == RCS0) |
e4734057 ZW |
825 | context_page_num = 19; |
826 | ||
827 | i = 2; | |
828 | ||
829 | while (i < context_page_num) { | |
830 | context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, | |
831 | (u32)((workload->ctx_desc.lrca + i) << | |
9556e118 | 832 | I915_GTT_PAGE_SHIFT)); |
e4734057 | 833 | if (context_gpa == INTEL_GVT_INVALID_ADDR) { |
695fbc08 | 834 | gvt_vgpu_err("invalid guest context descriptor\n"); |
e4734057 ZW |
835 | return; |
836 | } | |
837 | ||
9f379407 | 838 | page = i915_gem_object_get_page(ctx_obj, i); |
c754936f | 839 | src = kmap(page); |
e4734057 | 840 | intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, |
9556e118 | 841 | I915_GTT_PAGE_SIZE); |
c754936f | 842 | kunmap(page); |
e4734057 ZW |
843 | i++; |
844 | } | |
845 | ||
846 | intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + | |
847 | RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4); | |
848 | ||
849 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); | |
c754936f | 850 | shadow_ring_context = kmap(page); |
e4734057 ZW |
851 | |
852 | #define COPY_REG(name) \ | |
853 | intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \ | |
854 | RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) | |
855 | ||
856 | COPY_REG(ctx_ctrl); | |
857 | COPY_REG(ctx_timestamp); | |
858 | ||
859 | #undef COPY_REG | |
860 | ||
861 | intel_gvt_hypervisor_write_gpa(vgpu, | |
862 | workload->ring_context_gpa + | |
863 | sizeof(*shadow_ring_context), | |
864 | (void *)shadow_ring_context + | |
865 | sizeof(*shadow_ring_context), | |
9556e118 | 866 | I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); |
e4734057 | 867 | |
c754936f | 868 | kunmap(page); |
e4734057 ZW |
869 | } |
870 | ||
f9090d4c | 871 | void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, |
3a891a62 | 872 | intel_engine_mask_t engine_mask) |
e2c43c01 ZW |
873 | { |
874 | struct intel_vgpu_submission *s = &vgpu->submission; | |
a61ac1e7 | 875 | struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915; |
e2c43c01 ZW |
876 | struct intel_engine_cs *engine; |
877 | struct intel_vgpu_workload *pos, *n; | |
3a891a62 | 878 | intel_engine_mask_t tmp; |
e2c43c01 ZW |
879 | |
880 | /* free the unsubmited workloads in the queues. */ | |
a50134b1 | 881 | for_each_engine_masked(engine, &dev_priv->gt, engine_mask, tmp) { |
e2c43c01 ZW |
882 | list_for_each_entry_safe(pos, n, |
883 | &s->workload_q_head[engine->id], list) { | |
884 | list_del_init(&pos->list); | |
885 | intel_vgpu_destroy_workload(pos); | |
886 | } | |
887 | clear_bit(engine->id, s->shadow_ctx_desc_updated); | |
888 | } | |
889 | } | |
890 | ||
e4734057 ZW |
891 | static void complete_current_workload(struct intel_gvt *gvt, int ring_id) |
892 | { | |
893 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
1406a14b ZW |
894 | struct intel_vgpu_workload *workload = |
895 | scheduler->current_workload[ring_id]; | |
896 | struct intel_vgpu *vgpu = workload->vgpu; | |
897 | struct intel_vgpu_submission *s = &vgpu->submission; | |
6bb2a2af | 898 | struct i915_request *rq = workload->req; |
be1da707 | 899 | int event; |
e4734057 | 900 | |
f25a49ab | 901 | mutex_lock(&vgpu->vgpu_lock); |
9a512e23 | 902 | mutex_lock(&gvt->sched_lock); |
e4734057 | 903 | |
8f1117ab CD |
904 | /* For the workload w/ request, needs to wait for the context |
905 | * switch to make sure request is completed. | |
906 | * For the workload w/o request, directly complete the workload. | |
907 | */ | |
1fc44d9b | 908 | if (rq) { |
e4734057 ZW |
909 | wait_event(workload->shadow_ctx_status_wq, |
910 | !atomic_read(&workload->shadow_ctx_active)); | |
911 | ||
0cf5ec41 CD |
912 | /* If this request caused GPU hang, req->fence.error will |
913 | * be set to -EIO. Use -EIO to set workload status so | |
914 | * that when this request caused GPU hang, didn't trigger | |
915 | * context switch interrupt to guest. | |
916 | */ | |
917 | if (likely(workload->status == -EINPROGRESS)) { | |
918 | if (workload->req->fence.error == -EIO) | |
919 | workload->status = -EIO; | |
920 | else | |
921 | workload->status = 0; | |
922 | } | |
923 | ||
8a68d464 CW |
924 | if (!workload->status && |
925 | !(vgpu->resetting_eng & BIT(ring_id))) { | |
8f1117ab | 926 | update_guest_context(workload); |
be1da707 | 927 | |
8f1117ab CD |
928 | for_each_set_bit(event, workload->pending_events, |
929 | INTEL_GVT_EVENT_MAX) | |
930 | intel_vgpu_trigger_virtual_event(vgpu, event); | |
931 | } | |
1fc44d9b | 932 | |
6bb2a2af | 933 | i915_request_put(fetch_and_zero(&workload->req)); |
e4734057 ZW |
934 | } |
935 | ||
936 | gvt_dbg_sched("ring id %d complete workload %p status %d\n", | |
937 | ring_id, workload, workload->status); | |
938 | ||
939 | scheduler->current_workload[ring_id] = NULL; | |
940 | ||
e4734057 | 941 | list_del_init(&workload->list); |
d8235b5e | 942 | |
8a68d464 | 943 | if (workload->status || vgpu->resetting_eng & BIT(ring_id)) { |
e2c43c01 ZW |
944 | /* if workload->status is not successful means HW GPU |
945 | * has occurred GPU hang or something wrong with i915/GVT, | |
946 | * and GVT won't inject context switch interrupt to guest. | |
947 | * So this error is a vGPU hang actually to the guest. | |
948 | * According to this we should emunlate a vGPU hang. If | |
949 | * there are pending workloads which are already submitted | |
950 | * from guest, we should clean them up like HW GPU does. | |
951 | * | |
952 | * if it is in middle of engine resetting, the pending | |
953 | * workloads won't be submitted to HW GPU and will be | |
954 | * cleaned up during the resetting process later, so doing | |
955 | * the workload clean up here doesn't have any impact. | |
956 | **/ | |
8a68d464 | 957 | intel_vgpu_clean_workloads(vgpu, BIT(ring_id)); |
e2c43c01 ZW |
958 | } |
959 | ||
e4734057 ZW |
960 | workload->complete(workload); |
961 | ||
1406a14b | 962 | atomic_dec(&s->running_workload_num); |
e4734057 | 963 | wake_up(&scheduler->workload_complete_wq); |
f100daec PG |
964 | |
965 | if (gvt->scheduler.need_reschedule) | |
966 | intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED); | |
967 | ||
9a512e23 | 968 | mutex_unlock(&gvt->sched_lock); |
f25a49ab | 969 | mutex_unlock(&vgpu->vgpu_lock); |
e4734057 ZW |
970 | } |
971 | ||
8fde4107 | 972 | static int workload_thread(void *arg) |
e4734057 | 973 | { |
8fde4107 CW |
974 | struct intel_engine_cs *engine = arg; |
975 | const bool need_force_wake = INTEL_GEN(engine->i915) >= 9; | |
976 | struct intel_gvt *gvt = engine->i915->gvt; | |
e4734057 ZW |
977 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; |
978 | struct intel_vgpu_workload *workload = NULL; | |
695fbc08 | 979 | struct intel_vgpu *vgpu = NULL; |
e4734057 | 980 | int ret; |
e45d7b7f | 981 | DEFINE_WAIT_FUNC(wait, woken_wake_function); |
e4734057 | 982 | |
8fde4107 | 983 | gvt_dbg_core("workload thread for ring %s started\n", engine->name); |
e4734057 ZW |
984 | |
985 | while (!kthread_should_stop()) { | |
8fde4107 CW |
986 | intel_wakeref_t wakeref; |
987 | ||
988 | add_wait_queue(&scheduler->waitq[engine->id], &wait); | |
e45d7b7f | 989 | do { |
8fde4107 | 990 | workload = pick_next_workload(gvt, engine); |
e45d7b7f CD |
991 | if (workload) |
992 | break; | |
993 | wait_woken(&wait, TASK_INTERRUPTIBLE, | |
994 | MAX_SCHEDULE_TIMEOUT); | |
995 | } while (!kthread_should_stop()); | |
8fde4107 | 996 | remove_wait_queue(&scheduler->waitq[engine->id], &wait); |
e45d7b7f CD |
997 | |
998 | if (!workload) | |
e4734057 ZW |
999 | break; |
1000 | ||
8fde4107 CW |
1001 | gvt_dbg_sched("ring %s next workload %p vgpu %d\n", |
1002 | engine->name, workload, | |
1003 | workload->vgpu->id); | |
e4734057 | 1004 | |
8fde4107 | 1005 | wakeref = intel_runtime_pm_get(engine->uncore->rpm); |
ef5b0b44 | 1006 | |
8fde4107 CW |
1007 | gvt_dbg_sched("ring %s will dispatch workload %p\n", |
1008 | engine->name, workload); | |
e4734057 ZW |
1009 | |
1010 | if (need_force_wake) | |
8fde4107 CW |
1011 | intel_uncore_forcewake_get(engine->uncore, |
1012 | FORCEWAKE_ALL); | |
4a532256 WL |
1013 | /* |
1014 | * Update the vReg of the vGPU which submitted this | |
1015 | * workload. The vGPU may use these registers for checking | |
1016 | * the context state. The value comes from GPU commands | |
1017 | * in this workload. | |
1018 | */ | |
1019 | update_vreg_in_ctx(workload); | |
e4734057 ZW |
1020 | |
1021 | ret = dispatch_workload(workload); | |
66bbc3b2 | 1022 | |
e4734057 | 1023 | if (ret) { |
695fbc08 TZ |
1024 | vgpu = workload->vgpu; |
1025 | gvt_vgpu_err("fail to dispatch workload, skip\n"); | |
e4734057 ZW |
1026 | goto complete; |
1027 | } | |
1028 | ||
8fde4107 CW |
1029 | gvt_dbg_sched("ring %s wait workload %p\n", |
1030 | engine->name, workload); | |
e61e0f51 | 1031 | i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT); |
e4734057 ZW |
1032 | |
1033 | complete: | |
3ce3274b | 1034 | gvt_dbg_sched("will complete workload %p, status: %d\n", |
8fde4107 | 1035 | workload, workload->status); |
e4734057 | 1036 | |
8fde4107 | 1037 | complete_current_workload(gvt, engine->id); |
2e51ef32 | 1038 | |
e4734057 | 1039 | if (need_force_wake) |
8fde4107 CW |
1040 | intel_uncore_forcewake_put(engine->uncore, |
1041 | FORCEWAKE_ALL); | |
e4734057 | 1042 | |
8fde4107 | 1043 | intel_runtime_pm_put(engine->uncore->rpm, wakeref); |
6d763035 | 1044 | if (ret && (vgpu_is_vm_unhealthy(ret))) |
e011c6ce | 1045 | enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); |
e4734057 ZW |
1046 | } |
1047 | return 0; | |
1048 | } | |
1049 | ||
1050 | void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu) | |
1051 | { | |
1406a14b | 1052 | struct intel_vgpu_submission *s = &vgpu->submission; |
e4734057 ZW |
1053 | struct intel_gvt *gvt = vgpu->gvt; |
1054 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
1055 | ||
1406a14b | 1056 | if (atomic_read(&s->running_workload_num)) { |
e4734057 ZW |
1057 | gvt_dbg_sched("wait vgpu idle\n"); |
1058 | ||
1059 | wait_event(scheduler->workload_complete_wq, | |
1406a14b | 1060 | !atomic_read(&s->running_workload_num)); |
e4734057 ZW |
1061 | } |
1062 | } | |
1063 | ||
1064 | void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt) | |
1065 | { | |
1066 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
3fc03069 CD |
1067 | struct intel_engine_cs *engine; |
1068 | enum intel_engine_id i; | |
e4734057 ZW |
1069 | |
1070 | gvt_dbg_core("clean workload scheduler\n"); | |
1071 | ||
a61ac1e7 | 1072 | for_each_engine(engine, gvt->gt, i) { |
3fc03069 CD |
1073 | atomic_notifier_chain_unregister( |
1074 | &engine->context_status_notifier, | |
1075 | &gvt->shadow_ctx_notifier_block[i]); | |
1076 | kthread_stop(scheduler->thread[i]); | |
e4734057 ZW |
1077 | } |
1078 | } | |
1079 | ||
1080 | int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt) | |
1081 | { | |
1082 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
3fc03069 CD |
1083 | struct intel_engine_cs *engine; |
1084 | enum intel_engine_id i; | |
e4734057 | 1085 | int ret; |
e4734057 ZW |
1086 | |
1087 | gvt_dbg_core("init workload scheduler\n"); | |
1088 | ||
1089 | init_waitqueue_head(&scheduler->workload_complete_wq); | |
1090 | ||
a61ac1e7 | 1091 | for_each_engine(engine, gvt->gt, i) { |
e4734057 ZW |
1092 | init_waitqueue_head(&scheduler->waitq[i]); |
1093 | ||
8fde4107 CW |
1094 | scheduler->thread[i] = kthread_run(workload_thread, engine, |
1095 | "gvt:%s", engine->name); | |
e4734057 ZW |
1096 | if (IS_ERR(scheduler->thread[i])) { |
1097 | gvt_err("fail to create workload thread\n"); | |
1098 | ret = PTR_ERR(scheduler->thread[i]); | |
1099 | goto err; | |
1100 | } | |
3fc03069 CD |
1101 | |
1102 | gvt->shadow_ctx_notifier_block[i].notifier_call = | |
1103 | shadow_context_status_change; | |
1104 | atomic_notifier_chain_register(&engine->context_status_notifier, | |
1105 | &gvt->shadow_ctx_notifier_block[i]); | |
e4734057 | 1106 | } |
8fde4107 | 1107 | |
e4734057 | 1108 | return 0; |
8fde4107 | 1109 | |
e4734057 ZW |
1110 | err: |
1111 | intel_gvt_clean_workload_scheduler(gvt); | |
e4734057 ZW |
1112 | return ret; |
1113 | } | |
1114 | ||
f39a89b8 | 1115 | static void |
251d46b0 | 1116 | i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s, |
ab53497b | 1117 | struct i915_ppgtt *ppgtt) |
f39a89b8 | 1118 | { |
f39a89b8 XZ |
1119 | int i; |
1120 | ||
251d46b0 | 1121 | if (i915_vm_is_4lvl(&ppgtt->vm)) { |
b5b7bef9 | 1122 | px_dma(ppgtt->pd) = s->i915_context_pml4; |
a9fe9ca4 | 1123 | } else { |
b5b7bef9 MK |
1124 | for (i = 0; i < GEN8_3LVL_PDPES; i++) { |
1125 | struct i915_page_directory * const pd = | |
1126 | i915_pd_entry(ppgtt->pd, i); | |
1127 | ||
1128 | px_dma(pd) = s->i915_context_pdps[i]; | |
1129 | } | |
f39a89b8 XZ |
1130 | } |
1131 | } | |
1132 | ||
874b6a91 ZW |
1133 | /** |
1134 | * intel_vgpu_clean_submission - free submission-related resource for vGPU | |
1135 | * @vgpu: a vGPU | |
1136 | * | |
1137 | * This function is called when a vGPU is being destroyed. | |
1138 | * | |
1139 | */ | |
1140 | void intel_vgpu_clean_submission(struct intel_vgpu *vgpu) | |
e4734057 | 1141 | { |
1406a14b | 1142 | struct intel_vgpu_submission *s = &vgpu->submission; |
251d46b0 CW |
1143 | struct intel_engine_cs *engine; |
1144 | enum intel_engine_id id; | |
1406a14b | 1145 | |
7569a06d | 1146 | intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); |
251d46b0 | 1147 | |
f5d974f9 | 1148 | i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(s->shadow[0]->vm)); |
a61ac1e7 | 1149 | for_each_engine(engine, vgpu->gvt->gt, id) |
251d46b0 CW |
1150 | intel_context_unpin(s->shadow[id]); |
1151 | ||
1406a14b | 1152 | kmem_cache_destroy(s->workloads); |
e4734057 ZW |
1153 | } |
1154 | ||
06bb372f ZW |
1155 | |
1156 | /** | |
1157 | * intel_vgpu_reset_submission - reset submission-related resource for vGPU | |
1158 | * @vgpu: a vGPU | |
1159 | * @engine_mask: engines expected to be reset | |
1160 | * | |
1161 | * This function is called when a vGPU is being destroyed. | |
1162 | * | |
1163 | */ | |
1164 | void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, | |
3a891a62 | 1165 | intel_engine_mask_t engine_mask) |
06bb372f ZW |
1166 | { |
1167 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1168 | ||
1169 | if (!s->active) | |
1170 | return; | |
1171 | ||
f9090d4c | 1172 | intel_vgpu_clean_workloads(vgpu, engine_mask); |
06bb372f ZW |
1173 | s->ops->reset(vgpu, engine_mask); |
1174 | } | |
1175 | ||
f39a89b8 | 1176 | static void |
251d46b0 | 1177 | i915_context_ppgtt_root_save(struct intel_vgpu_submission *s, |
ab53497b | 1178 | struct i915_ppgtt *ppgtt) |
f39a89b8 | 1179 | { |
f39a89b8 XZ |
1180 | int i; |
1181 | ||
251d46b0 | 1182 | if (i915_vm_is_4lvl(&ppgtt->vm)) { |
b5b7bef9 | 1183 | s->i915_context_pml4 = px_dma(ppgtt->pd); |
251d46b0 | 1184 | } else { |
b5b7bef9 MK |
1185 | for (i = 0; i < GEN8_3LVL_PDPES; i++) { |
1186 | struct i915_page_directory * const pd = | |
1187 | i915_pd_entry(ppgtt->pd, i); | |
1188 | ||
1189 | s->i915_context_pdps[i] = px_dma(pd); | |
1190 | } | |
f39a89b8 XZ |
1191 | } |
1192 | } | |
1193 | ||
874b6a91 ZW |
1194 | /** |
1195 | * intel_vgpu_setup_submission - setup submission-related resource for vGPU | |
1196 | * @vgpu: a vGPU | |
1197 | * | |
1198 | * This function is called when a vGPU is being created. | |
1199 | * | |
1200 | * Returns: | |
1201 | * Zero on success, negative error code if failed. | |
1202 | * | |
1203 | */ | |
1204 | int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) | |
e4734057 | 1205 | { |
a61ac1e7 | 1206 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
1406a14b | 1207 | struct intel_vgpu_submission *s = &vgpu->submission; |
9a9829e9 | 1208 | struct intel_engine_cs *engine; |
a4e7ccda | 1209 | struct i915_ppgtt *ppgtt; |
251d46b0 | 1210 | enum intel_engine_id i; |
9a9829e9 | 1211 | int ret; |
e4734057 | 1212 | |
2c86e55d | 1213 | ppgtt = i915_ppgtt_create(&i915->gt); |
e6ba7648 CW |
1214 | if (IS_ERR(ppgtt)) |
1215 | return PTR_ERR(ppgtt); | |
72e27775 | 1216 | |
a4e7ccda | 1217 | i915_context_ppgtt_root_save(s, ppgtt); |
e4734057 | 1218 | |
a61ac1e7 | 1219 | for_each_engine(engine, vgpu->gvt->gt, i) { |
251d46b0 CW |
1220 | struct intel_context *ce; |
1221 | ||
1222 | INIT_LIST_HEAD(&s->workload_q_head[i]); | |
1223 | s->shadow[i] = ERR_PTR(-EINVAL); | |
1224 | ||
e6ba7648 | 1225 | ce = intel_context_create(engine); |
251d46b0 CW |
1226 | if (IS_ERR(ce)) { |
1227 | ret = PTR_ERR(ce); | |
1228 | goto out_shadow_ctx; | |
1229 | } | |
e4734057 | 1230 | |
e6ba7648 CW |
1231 | i915_vm_put(ce->vm); |
1232 | ce->vm = i915_vm_get(&ppgtt->vm); | |
9f3ccd40 CW |
1233 | intel_context_set_single_submission(ce); |
1234 | ||
065273f7 | 1235 | /* Max ring buffer size */ |
202c98e7 | 1236 | if (!intel_uc_wants_guc_submission(&engine->gt->uc)) { |
48ae397b CW |
1237 | const unsigned int ring_size = 512 * SZ_4K; |
1238 | ||
1239 | ce->ring = __intel_context_ring_size(ring_size); | |
1240 | } | |
1241 | ||
fa9f6681 CW |
1242 | ret = intel_context_pin(ce); |
1243 | intel_context_put(ce); | |
1244 | if (ret) | |
1245 | goto out_shadow_ctx; | |
1246 | ||
251d46b0 CW |
1247 | s->shadow[i] = ce; |
1248 | } | |
f39a89b8 | 1249 | |
1406a14b | 1250 | bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES); |
9dfb8e5b | 1251 | |
850555d1 ZW |
1252 | s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload", |
1253 | sizeof(struct intel_vgpu_workload), 0, | |
1254 | SLAB_HWCACHE_ALIGN, | |
1255 | offsetof(struct intel_vgpu_workload, rb_tail), | |
1256 | sizeof_field(struct intel_vgpu_workload, rb_tail), | |
1257 | NULL); | |
9a9829e9 | 1258 | |
1406a14b | 1259 | if (!s->workloads) { |
9a9829e9 ZW |
1260 | ret = -ENOMEM; |
1261 | goto out_shadow_ctx; | |
1262 | } | |
1263 | ||
1406a14b | 1264 | atomic_set(&s->running_workload_num, 0); |
91d5d854 | 1265 | bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES); |
9a9829e9 | 1266 | |
a4e7ccda | 1267 | i915_vm_put(&ppgtt->vm); |
e4734057 | 1268 | return 0; |
9a9829e9 ZW |
1269 | |
1270 | out_shadow_ctx: | |
a4e7ccda | 1271 | i915_context_ppgtt_root_restore(s, ppgtt); |
a61ac1e7 | 1272 | for_each_engine(engine, vgpu->gvt->gt, i) { |
251d46b0 CW |
1273 | if (IS_ERR(s->shadow[i])) |
1274 | break; | |
1275 | ||
1276 | intel_context_unpin(s->shadow[i]); | |
38775829 | 1277 | intel_context_put(s->shadow[i]); |
251d46b0 | 1278 | } |
a4e7ccda | 1279 | i915_vm_put(&ppgtt->vm); |
9a9829e9 | 1280 | return ret; |
e4734057 | 1281 | } |
21527a8d | 1282 | |
ad1d3636 ZW |
1283 | /** |
1284 | * intel_vgpu_select_submission_ops - select virtual submission interface | |
1285 | * @vgpu: a vGPU | |
a752b070 | 1286 | * @engine_mask: either ALL_ENGINES or target engine mask |
ad1d3636 ZW |
1287 | * @interface: expected vGPU virtual submission interface |
1288 | * | |
1289 | * This function is called when guest configures submission interface. | |
1290 | * | |
1291 | * Returns: | |
1292 | * Zero on success, negative error code if failed. | |
1293 | * | |
1294 | */ | |
1295 | int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, | |
3a891a62 | 1296 | intel_engine_mask_t engine_mask, |
ad1d3636 ZW |
1297 | unsigned int interface) |
1298 | { | |
a61ac1e7 | 1299 | struct drm_i915_private *i915 = vgpu->gvt->gt->i915; |
ad1d3636 ZW |
1300 | struct intel_vgpu_submission *s = &vgpu->submission; |
1301 | const struct intel_vgpu_submission_ops *ops[] = { | |
1302 | [INTEL_VGPU_EXECLIST_SUBMISSION] = | |
1303 | &intel_vgpu_execlist_submission_ops, | |
1304 | }; | |
1305 | int ret; | |
1306 | ||
12d58619 | 1307 | if (drm_WARN_ON(&i915->drm, interface >= ARRAY_SIZE(ops))) |
ad1d3636 ZW |
1308 | return -EINVAL; |
1309 | ||
12d58619 PB |
1310 | if (drm_WARN_ON(&i915->drm, |
1311 | interface == 0 && engine_mask != ALL_ENGINES)) | |
9212b13f WL |
1312 | return -EINVAL; |
1313 | ||
1314 | if (s->active) | |
7569a06d | 1315 | s->ops->clean(vgpu, engine_mask); |
ad1d3636 ZW |
1316 | |
1317 | if (interface == 0) { | |
1318 | s->ops = NULL; | |
1319 | s->virtual_submission_interface = 0; | |
9212b13f WL |
1320 | s->active = false; |
1321 | gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id); | |
ad1d3636 ZW |
1322 | return 0; |
1323 | } | |
1324 | ||
7569a06d | 1325 | ret = ops[interface]->init(vgpu, engine_mask); |
ad1d3636 ZW |
1326 | if (ret) |
1327 | return ret; | |
1328 | ||
1329 | s->ops = ops[interface]; | |
1330 | s->virtual_submission_interface = interface; | |
1331 | s->active = true; | |
1332 | ||
1333 | gvt_dbg_core("vgpu%d: activate ops [ %s ]\n", | |
1334 | vgpu->id, s->ops->name); | |
1335 | ||
1336 | return 0; | |
1337 | } | |
1338 | ||
21527a8d ZW |
1339 | /** |
1340 | * intel_vgpu_destroy_workload - destroy a vGPU workload | |
a752b070 | 1341 | * @workload: workload to destroy |
21527a8d ZW |
1342 | * |
1343 | * This function is called when destroy a vGPU workload. | |
1344 | * | |
1345 | */ | |
1346 | void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) | |
1347 | { | |
1348 | struct intel_vgpu_submission *s = &workload->vgpu->submission; | |
1349 | ||
0f755512 WL |
1350 | release_shadow_batch_buffer(workload); |
1351 | release_shadow_wa_ctx(&workload->wa_ctx); | |
1352 | ||
21527a8d | 1353 | if (workload->shadow_mm) |
1bc25851 | 1354 | intel_vgpu_mm_put(workload->shadow_mm); |
21527a8d ZW |
1355 | |
1356 | kmem_cache_free(s->workloads, workload); | |
1357 | } | |
1358 | ||
6d763035 ZW |
1359 | static struct intel_vgpu_workload * |
1360 | alloc_workload(struct intel_vgpu *vgpu) | |
1361 | { | |
1362 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1363 | struct intel_vgpu_workload *workload; | |
1364 | ||
1365 | workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL); | |
1366 | if (!workload) | |
1367 | return ERR_PTR(-ENOMEM); | |
1368 | ||
1369 | INIT_LIST_HEAD(&workload->list); | |
1370 | INIT_LIST_HEAD(&workload->shadow_bb); | |
1371 | ||
1372 | init_waitqueue_head(&workload->shadow_ctx_status_wq); | |
1373 | atomic_set(&workload->shadow_ctx_active, 0); | |
1374 | ||
1375 | workload->status = -EINPROGRESS; | |
6d763035 ZW |
1376 | workload->vgpu = vgpu; |
1377 | ||
1378 | return workload; | |
1379 | } | |
1380 | ||
1381 | #define RING_CTX_OFF(x) \ | |
1382 | offsetof(struct execlist_ring_context, x) | |
1383 | ||
1384 | static void read_guest_pdps(struct intel_vgpu *vgpu, | |
1385 | u64 ring_context_gpa, u32 pdp[8]) | |
1386 | { | |
1387 | u64 gpa; | |
1388 | int i; | |
1389 | ||
1417fad7 | 1390 | gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val); |
6d763035 ZW |
1391 | |
1392 | for (i = 0; i < 8; i++) | |
1393 | intel_gvt_hypervisor_read_gpa(vgpu, | |
1394 | gpa + i * 8, &pdp[7 - i], 4); | |
1395 | } | |
1396 | ||
1397 | static int prepare_mm(struct intel_vgpu_workload *workload) | |
1398 | { | |
1399 | struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; | |
1400 | struct intel_vgpu_mm *mm; | |
1401 | struct intel_vgpu *vgpu = workload->vgpu; | |
0cf8f58d | 1402 | enum intel_gvt_gtt_type root_entry_type; |
ede9d0cf | 1403 | u64 pdps[GVT_RING_CTX_NR_PDPS]; |
6d763035 | 1404 | |
ede9d0cf CD |
1405 | switch (desc->addressing_mode) { |
1406 | case 1: /* legacy 32-bit */ | |
1407 | root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; | |
1408 | break; | |
1409 | case 3: /* legacy 64-bit */ | |
1410 | root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY; | |
1411 | break; | |
1412 | default: | |
6d763035 ZW |
1413 | gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); |
1414 | return -EINVAL; | |
1415 | } | |
1416 | ||
ede9d0cf | 1417 | read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps); |
6d763035 | 1418 | |
e6e9c46f CD |
1419 | mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps); |
1420 | if (IS_ERR(mm)) | |
1421 | return PTR_ERR(mm); | |
1422 | ||
6d763035 ZW |
1423 | workload->shadow_mm = mm; |
1424 | return 0; | |
1425 | } | |
1426 | ||
1427 | #define same_context(a, b) (((a)->context_id == (b)->context_id) && \ | |
1428 | ((a)->lrca == (b)->lrca)) | |
1429 | ||
21527a8d ZW |
1430 | /** |
1431 | * intel_vgpu_create_workload - create a vGPU workload | |
1432 | * @vgpu: a vGPU | |
8fde4107 | 1433 | * @engine: the engine |
6d763035 | 1434 | * @desc: a guest context descriptor |
21527a8d ZW |
1435 | * |
1436 | * This function is called when creating a vGPU workload. | |
1437 | * | |
1438 | * Returns: | |
1439 | * struct intel_vgpu_workload * on success, negative error code in | |
1440 | * pointer if failed. | |
1441 | * | |
1442 | */ | |
1443 | struct intel_vgpu_workload * | |
8fde4107 CW |
1444 | intel_vgpu_create_workload(struct intel_vgpu *vgpu, |
1445 | const struct intel_engine_cs *engine, | |
6d763035 | 1446 | struct execlist_ctx_descriptor_format *desc) |
21527a8d ZW |
1447 | { |
1448 | struct intel_vgpu_submission *s = &vgpu->submission; | |
8fde4107 | 1449 | struct list_head *q = workload_q_head(vgpu, engine); |
0a3242bd | 1450 | struct intel_vgpu_workload *last_workload = NULL; |
6d763035 | 1451 | struct intel_vgpu_workload *workload = NULL; |
6d763035 ZW |
1452 | u64 ring_context_gpa; |
1453 | u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; | |
15e7f52a | 1454 | u32 guest_head; |
6d763035 | 1455 | int ret; |
21527a8d | 1456 | |
6d763035 | 1457 | ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, |
9556e118 | 1458 | (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT)); |
6d763035 ZW |
1459 | if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { |
1460 | gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); | |
1461 | return ERR_PTR(-EINVAL); | |
1462 | } | |
21527a8d | 1463 | |
6d763035 ZW |
1464 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1465 | RING_CTX_OFF(ring_header.val), &head, 4); | |
21527a8d | 1466 | |
6d763035 ZW |
1467 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1468 | RING_CTX_OFF(ring_tail.val), &tail, 4); | |
21527a8d | 1469 | |
15e7f52a XZ |
1470 | guest_head = head; |
1471 | ||
6d763035 ZW |
1472 | head &= RB_HEAD_OFF_MASK; |
1473 | tail &= RB_TAIL_OFF_MASK; | |
1474 | ||
0a3242bd XZ |
1475 | list_for_each_entry_reverse(last_workload, q, list) { |
1476 | ||
1477 | if (same_context(&last_workload->ctx_desc, desc)) { | |
8fde4107 CW |
1478 | gvt_dbg_el("ring %s cur workload == last\n", |
1479 | engine->name); | |
0a3242bd | 1480 | gvt_dbg_el("ctx head %x real head %lx\n", head, |
8fde4107 | 1481 | last_workload->rb_tail); |
0a3242bd XZ |
1482 | /* |
1483 | * cannot use guest context head pointer here, | |
1484 | * as it might not be updated at this time | |
1485 | */ | |
1486 | head = last_workload->rb_tail; | |
1487 | break; | |
1488 | } | |
6d763035 ZW |
1489 | } |
1490 | ||
8fde4107 | 1491 | gvt_dbg_el("ring %s begin a new workload\n", engine->name); |
6d763035 ZW |
1492 | |
1493 | /* record some ring buffer register values for scan and shadow */ | |
1494 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1495 | RING_CTX_OFF(rb_start.val), &start, 4); | |
1496 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1497 | RING_CTX_OFF(rb_ctrl.val), &ctl, 4); | |
1498 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1499 | RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); | |
1500 | ||
2089a76a XZ |
1501 | if (!intel_gvt_ggtt_validate_range(vgpu, start, |
1502 | _RING_CTL_BUF_SIZE(ctl))) { | |
1503 | gvt_vgpu_err("context contain invalid rb at: 0x%x\n", start); | |
1504 | return ERR_PTR(-EINVAL); | |
1505 | } | |
1506 | ||
6d763035 ZW |
1507 | workload = alloc_workload(vgpu); |
1508 | if (IS_ERR(workload)) | |
1509 | return workload; | |
1510 | ||
8fde4107 | 1511 | workload->engine = engine; |
6d763035 ZW |
1512 | workload->ctx_desc = *desc; |
1513 | workload->ring_context_gpa = ring_context_gpa; | |
1514 | workload->rb_head = head; | |
15e7f52a | 1515 | workload->guest_rb_head = guest_head; |
6d763035 ZW |
1516 | workload->rb_tail = tail; |
1517 | workload->rb_start = start; | |
1518 | workload->rb_ctl = ctl; | |
1519 | ||
8fde4107 | 1520 | if (engine->id == RCS0) { |
6d763035 ZW |
1521 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1522 | RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); | |
1523 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1524 | RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); | |
1525 | ||
1526 | workload->wa_ctx.indirect_ctx.guest_gma = | |
1527 | indirect_ctx & INDIRECT_CTX_ADDR_MASK; | |
1528 | workload->wa_ctx.indirect_ctx.size = | |
1529 | (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * | |
1530 | CACHELINE_BYTES; | |
2089a76a XZ |
1531 | |
1532 | if (workload->wa_ctx.indirect_ctx.size != 0) { | |
1533 | if (!intel_gvt_ggtt_validate_range(vgpu, | |
1534 | workload->wa_ctx.indirect_ctx.guest_gma, | |
1535 | workload->wa_ctx.indirect_ctx.size)) { | |
2089a76a XZ |
1536 | gvt_vgpu_err("invalid wa_ctx at: 0x%lx\n", |
1537 | workload->wa_ctx.indirect_ctx.guest_gma); | |
eac4471d | 1538 | kmem_cache_free(s->workloads, workload); |
2089a76a XZ |
1539 | return ERR_PTR(-EINVAL); |
1540 | } | |
1541 | } | |
1542 | ||
6d763035 ZW |
1543 | workload->wa_ctx.per_ctx.guest_gma = |
1544 | per_ctx & PER_CTX_ADDR_MASK; | |
1545 | workload->wa_ctx.per_ctx.valid = per_ctx & 1; | |
2089a76a XZ |
1546 | if (workload->wa_ctx.per_ctx.valid) { |
1547 | if (!intel_gvt_ggtt_validate_range(vgpu, | |
1548 | workload->wa_ctx.per_ctx.guest_gma, | |
1549 | CACHELINE_BYTES)) { | |
2089a76a XZ |
1550 | gvt_vgpu_err("invalid per_ctx at: 0x%lx\n", |
1551 | workload->wa_ctx.per_ctx.guest_gma); | |
eac4471d | 1552 | kmem_cache_free(s->workloads, workload); |
2089a76a XZ |
1553 | return ERR_PTR(-EINVAL); |
1554 | } | |
1555 | } | |
6d763035 ZW |
1556 | } |
1557 | ||
8fde4107 CW |
1558 | gvt_dbg_el("workload %p ring %s head %x tail %x start %x ctl %x\n", |
1559 | workload, engine->name, head, tail, start, ctl); | |
6d763035 ZW |
1560 | |
1561 | ret = prepare_mm(workload); | |
1562 | if (ret) { | |
1563 | kmem_cache_free(s->workloads, workload); | |
1564 | return ERR_PTR(ret); | |
1565 | } | |
1566 | ||
1567 | /* Only scan and shadow the first workload in the queue | |
1568 | * as there is only one pre-allocated buf-obj for shadow. | |
1569 | */ | |
8fde4107 CW |
1570 | if (list_empty(q)) { |
1571 | intel_wakeref_t wakeref; | |
1572 | ||
1573 | with_intel_runtime_pm(engine->gt->uncore->rpm, wakeref) | |
1574 | ret = intel_gvt_scan_and_shadow_workload(workload); | |
6d763035 ZW |
1575 | } |
1576 | ||
dade58ed YZ |
1577 | if (ret) { |
1578 | if (vgpu_is_vm_unhealthy(ret)) | |
1579 | enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); | |
6d763035 ZW |
1580 | intel_vgpu_destroy_workload(workload); |
1581 | return ERR_PTR(ret); | |
1582 | } | |
21527a8d ZW |
1583 | |
1584 | return workload; | |
1585 | } | |
59a716c6 CD |
1586 | |
1587 | /** | |
1588 | * intel_vgpu_queue_workload - Qeue a vGPU workload | |
1589 | * @workload: the workload to queue in | |
1590 | */ | |
1591 | void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload) | |
1592 | { | |
1593 | list_add_tail(&workload->list, | |
8fde4107 | 1594 | workload_q_head(workload->vgpu, workload->engine)); |
c130456c | 1595 | intel_gvt_kick_schedule(workload->vgpu->gvt); |
8fde4107 | 1596 | wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->engine->id]); |
59a716c6 | 1597 | } |