]>
Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
a8c21a54 T |
2 | /* |
3 | * Copyright (C) 2015 Etnaviv Project | |
a8c21a54 T |
4 | */ |
5 | ||
6eae41fe | 6 | #include <drm/drm_file.h> |
9ad59fea | 7 | #include <linux/dma-fence-array.h> |
6eae41fe | 8 | #include <linux/file.h> |
2e737e52 | 9 | #include <linux/pm_runtime.h> |
52791eee | 10 | #include <linux/dma-resv.h> |
9ad59fea | 11 | #include <linux/sync_file.h> |
6eae41fe SR |
12 | #include <linux/uaccess.h> |
13 | #include <linux/vmalloc.h> | |
14 | ||
ea1f5729 | 15 | #include "etnaviv_cmdbuf.h" |
a8c21a54 T |
16 | #include "etnaviv_drv.h" |
17 | #include "etnaviv_gpu.h" | |
18 | #include "etnaviv_gem.h" | |
c8e4a7fd | 19 | #include "etnaviv_perfmon.h" |
e93b6dee | 20 | #include "etnaviv_sched.h" |
a8c21a54 T |
21 | |
22 | /* | |
23 | * Cmdstream submission: | |
24 | */ | |
25 | ||
26 | #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) | |
27 | /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ | |
28 | #define BO_LOCKED 0x4000 | |
29 | #define BO_PINNED 0x2000 | |
30 | ||
a8c21a54 | 31 | static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, |
ef146c00 | 32 | struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs) |
a8c21a54 T |
33 | { |
34 | struct etnaviv_gem_submit *submit; | |
ef146c00 | 35 | size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit)); |
a8c21a54 | 36 | |
c5283723 LS |
37 | submit = kzalloc(sz, GFP_KERNEL); |
38 | if (!submit) | |
39 | return NULL; | |
a8c21a54 | 40 | |
ef146c00 LS |
41 | submit->pmrs = kcalloc(nr_pmrs, sizeof(struct etnaviv_perfmon_request), |
42 | GFP_KERNEL); | |
43 | if (!submit->pmrs) { | |
44 | kfree(submit); | |
45 | return NULL; | |
46 | } | |
47 | submit->nr_pmrs = nr_pmrs; | |
48 | ||
c5283723 | 49 | submit->gpu = gpu; |
e0329e6c | 50 | kref_init(&submit->refcount); |
a8c21a54 | 51 | |
a8c21a54 T |
52 | return submit; |
53 | } | |
54 | ||
55 | static int submit_lookup_objects(struct etnaviv_gem_submit *submit, | |
56 | struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, | |
57 | unsigned nr_bos) | |
58 | { | |
59 | struct drm_etnaviv_gem_submit_bo *bo; | |
60 | unsigned i; | |
61 | int ret = 0; | |
62 | ||
63 | spin_lock(&file->table_lock); | |
64 | ||
65 | for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { | |
66 | struct drm_gem_object *obj; | |
67 | ||
68 | if (bo->flags & BO_INVALID_FLAGS) { | |
69 | DRM_ERROR("invalid flags: %x\n", bo->flags); | |
70 | ret = -EINVAL; | |
71 | goto out_unlock; | |
72 | } | |
73 | ||
74 | submit->bos[i].flags = bo->flags; | |
088880dd LS |
75 | if (submit->flags & ETNA_SUBMIT_SOFTPIN) { |
76 | if (bo->presumed < ETNAVIV_SOFTPIN_START_ADDRESS) { | |
77 | DRM_ERROR("invalid softpin address\n"); | |
78 | ret = -EINVAL; | |
79 | goto out_unlock; | |
80 | } | |
81 | submit->bos[i].va = bo->presumed; | |
82 | } | |
a8c21a54 T |
83 | |
84 | /* normally use drm_gem_object_lookup(), but for bulk lookup | |
85 | * all under single table_lock just hit object_idr directly: | |
86 | */ | |
87 | obj = idr_find(&file->object_idr, bo->handle); | |
88 | if (!obj) { | |
89 | DRM_ERROR("invalid handle %u at index %u\n", | |
90 | bo->handle, i); | |
91 | ret = -EINVAL; | |
92 | goto out_unlock; | |
93 | } | |
94 | ||
95 | /* | |
96 | * Take a refcount on the object. The file table lock | |
97 | * prevents the object_idr's refcount on this being dropped. | |
98 | */ | |
23d1dd03 | 99 | drm_gem_object_get(obj); |
a8c21a54 T |
100 | |
101 | submit->bos[i].obj = to_etnaviv_bo(obj); | |
102 | } | |
103 | ||
104 | out_unlock: | |
105 | submit->nr_bos = i; | |
106 | spin_unlock(&file->table_lock); | |
107 | ||
108 | return ret; | |
109 | } | |
110 | ||
111 | static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) | |
112 | { | |
113 | if (submit->bos[i].flags & BO_LOCKED) { | |
fa238ea1 | 114 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
a8c21a54 | 115 | |
caa2a778 | 116 | dma_resv_unlock(obj->resv); |
a8c21a54 T |
117 | submit->bos[i].flags &= ~BO_LOCKED; |
118 | } | |
119 | } | |
120 | ||
08301d73 LS |
121 | static int submit_lock_objects(struct etnaviv_gem_submit *submit, |
122 | struct ww_acquire_ctx *ticket) | |
a8c21a54 T |
123 | { |
124 | int contended, slow_locked = -1, i, ret = 0; | |
125 | ||
126 | retry: | |
127 | for (i = 0; i < submit->nr_bos; i++) { | |
fa238ea1 | 128 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
a8c21a54 T |
129 | |
130 | if (slow_locked == i) | |
131 | slow_locked = -1; | |
132 | ||
133 | contended = i; | |
134 | ||
135 | if (!(submit->bos[i].flags & BO_LOCKED)) { | |
caa2a778 | 136 | ret = dma_resv_lock_interruptible(obj->resv, ticket); |
a8c21a54 T |
137 | if (ret == -EALREADY) |
138 | DRM_ERROR("BO at index %u already on submit list\n", | |
139 | i); | |
140 | if (ret) | |
141 | goto fail; | |
142 | submit->bos[i].flags |= BO_LOCKED; | |
143 | } | |
144 | } | |
145 | ||
08301d73 | 146 | ww_acquire_done(ticket); |
a8c21a54 T |
147 | |
148 | return 0; | |
149 | ||
150 | fail: | |
151 | for (; i >= 0; i--) | |
152 | submit_unlock_object(submit, i); | |
153 | ||
154 | if (slow_locked > 0) | |
155 | submit_unlock_object(submit, slow_locked); | |
156 | ||
157 | if (ret == -EDEADLK) { | |
fa238ea1 | 158 | struct drm_gem_object *obj; |
a8c21a54 | 159 | |
fa238ea1 | 160 | obj = &submit->bos[contended].obj->base; |
a8c21a54 T |
161 | |
162 | /* we lost out in a seqno race, lock and retry.. */ | |
caa2a778 | 163 | ret = dma_resv_lock_slow_interruptible(obj->resv, ticket); |
a8c21a54 T |
164 | if (!ret) { |
165 | submit->bos[contended].flags |= BO_LOCKED; | |
166 | slow_locked = contended; | |
167 | goto retry; | |
168 | } | |
169 | } | |
170 | ||
171 | return ret; | |
172 | } | |
173 | ||
683da226 | 174 | static int submit_fence_sync(struct etnaviv_gem_submit *submit) |
a8c21a54 | 175 | { |
a8c21a54 T |
176 | int i, ret = 0; |
177 | ||
178 | for (i = 0; i < submit->nr_bos; i++) { | |
683da226 | 179 | struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; |
52791eee | 180 | struct dma_resv *robj = bo->obj->base.resv; |
a8c21a54 | 181 | |
683da226 | 182 | if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) { |
52791eee | 183 | ret = dma_resv_reserve_shared(robj, 1); |
683da226 LS |
184 | if (ret) |
185 | return ret; | |
186 | } | |
187 | ||
188 | if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT) | |
189 | continue; | |
190 | ||
191 | if (bo->flags & ETNA_SUBMIT_BO_WRITE) { | |
52791eee | 192 | ret = dma_resv_get_fences_rcu(robj, &bo->excl, |
683da226 LS |
193 | &bo->nr_shared, |
194 | &bo->shared); | |
195 | if (ret) | |
196 | return ret; | |
197 | } else { | |
52791eee | 198 | bo->excl = dma_resv_get_excl_rcu(robj); |
683da226 | 199 | } |
a8c21a54 | 200 | |
9efabd73 LS |
201 | } |
202 | ||
a8c21a54 T |
203 | return ret; |
204 | } | |
205 | ||
0236efe9 LS |
206 | static void submit_attach_object_fences(struct etnaviv_gem_submit *submit) |
207 | { | |
208 | int i; | |
209 | ||
210 | for (i = 0; i < submit->nr_bos; i++) { | |
fa238ea1 | 211 | struct drm_gem_object *obj = &submit->bos[i].obj->base; |
0236efe9 LS |
212 | |
213 | if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) | |
52791eee | 214 | dma_resv_add_excl_fence(obj->resv, |
10009ea2 | 215 | submit->out_fence); |
0236efe9 | 216 | else |
52791eee | 217 | dma_resv_add_shared_fence(obj->resv, |
10009ea2 | 218 | submit->out_fence); |
0236efe9 LS |
219 | |
220 | submit_unlock_object(submit, i); | |
221 | } | |
222 | } | |
223 | ||
a8c21a54 T |
224 | static int submit_pin_objects(struct etnaviv_gem_submit *submit) |
225 | { | |
226 | int i, ret = 0; | |
227 | ||
228 | for (i = 0; i < submit->nr_bos; i++) { | |
229 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
b6325f40 | 230 | struct etnaviv_vram_mapping *mapping; |
a8c21a54 | 231 | |
b6325f40 | 232 | mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base, |
088880dd LS |
233 | submit->mmu_context, |
234 | submit->bos[i].va); | |
b6325f40 RK |
235 | if (IS_ERR(mapping)) { |
236 | ret = PTR_ERR(mapping); | |
a8c21a54 | 237 | break; |
b6325f40 | 238 | } |
088880dd LS |
239 | |
240 | if ((submit->flags & ETNA_SUBMIT_SOFTPIN) && | |
ad99cb5e DC |
241 | submit->bos[i].va != mapping->iova) { |
242 | etnaviv_gem_mapping_unreference(mapping); | |
088880dd | 243 | return -EINVAL; |
ad99cb5e | 244 | } |
088880dd | 245 | |
5b223e94 | 246 | atomic_inc(&etnaviv_obj->gpu_active); |
a8c21a54 T |
247 | |
248 | submit->bos[i].flags |= BO_PINNED; | |
b6325f40 | 249 | submit->bos[i].mapping = mapping; |
a8c21a54 T |
250 | } |
251 | ||
252 | return ret; | |
253 | } | |
254 | ||
255 | static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, | |
8779aa8f | 256 | struct etnaviv_gem_submit_bo **bo) |
a8c21a54 T |
257 | { |
258 | if (idx >= submit->nr_bos) { | |
259 | DRM_ERROR("invalid buffer index: %u (out of %u)\n", | |
260 | idx, submit->nr_bos); | |
261 | return -EINVAL; | |
262 | } | |
263 | ||
8779aa8f | 264 | *bo = &submit->bos[idx]; |
a8c21a54 T |
265 | |
266 | return 0; | |
267 | } | |
268 | ||
269 | /* process the reloc's and patch up the cmdstream as needed: */ | |
270 | static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, | |
271 | u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, | |
272 | u32 nr_relocs) | |
273 | { | |
274 | u32 i, last_offset = 0; | |
275 | u32 *ptr = stream; | |
276 | int ret; | |
277 | ||
088880dd LS |
278 | /* Submits using softpin don't blend with relocs */ |
279 | if ((submit->flags & ETNA_SUBMIT_SOFTPIN) && nr_relocs != 0) | |
280 | return -EINVAL; | |
281 | ||
a8c21a54 T |
282 | for (i = 0; i < nr_relocs; i++) { |
283 | const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; | |
8779aa8f RK |
284 | struct etnaviv_gem_submit_bo *bo; |
285 | u32 off; | |
a8c21a54 T |
286 | |
287 | if (unlikely(r->flags)) { | |
288 | DRM_ERROR("invalid reloc flags\n"); | |
289 | return -EINVAL; | |
290 | } | |
291 | ||
292 | if (r->submit_offset % 4) { | |
293 | DRM_ERROR("non-aligned reloc offset: %u\n", | |
294 | r->submit_offset); | |
295 | return -EINVAL; | |
296 | } | |
297 | ||
298 | /* offset in dwords: */ | |
299 | off = r->submit_offset / 4; | |
300 | ||
301 | if ((off >= size ) || | |
302 | (off < last_offset)) { | |
303 | DRM_ERROR("invalid offset %u at reloc %u\n", off, i); | |
304 | return -EINVAL; | |
305 | } | |
306 | ||
8779aa8f | 307 | ret = submit_bo(submit, r->reloc_idx, &bo); |
a8c21a54 T |
308 | if (ret) |
309 | return ret; | |
310 | ||
d6f756e0 WL |
311 | if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) { |
312 | DRM_ERROR("relocation %u outside object\n", i); | |
a8c21a54 T |
313 | return -EINVAL; |
314 | } | |
315 | ||
8779aa8f | 316 | ptr[off] = bo->mapping->iova + r->reloc_offset; |
a8c21a54 T |
317 | |
318 | last_offset = off; | |
319 | } | |
320 | ||
321 | return 0; | |
322 | } | |
323 | ||
c8e4a7fd | 324 | static int submit_perfmon_validate(struct etnaviv_gem_submit *submit, |
ef146c00 | 325 | u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs) |
c8e4a7fd CG |
326 | { |
327 | u32 i; | |
328 | ||
ef146c00 | 329 | for (i = 0; i < submit->nr_pmrs; i++) { |
c8e4a7fd CG |
330 | const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i; |
331 | struct etnaviv_gem_submit_bo *bo; | |
332 | int ret; | |
333 | ||
334 | ret = submit_bo(submit, r->read_idx, &bo); | |
335 | if (ret) | |
336 | return ret; | |
337 | ||
338 | /* at offset 0 a sequence number gets stored used for userspace sync */ | |
339 | if (r->read_offset == 0) { | |
340 | DRM_ERROR("perfmon request: offset is 0"); | |
341 | return -EINVAL; | |
342 | } | |
343 | ||
344 | if (r->read_offset >= bo->obj->base.size - sizeof(u32)) { | |
345 | DRM_ERROR("perfmon request: offset %u outside object", i); | |
346 | return -EINVAL; | |
347 | } | |
348 | ||
349 | if (r->flags & ~(ETNA_PM_PROCESS_PRE | ETNA_PM_PROCESS_POST)) { | |
350 | DRM_ERROR("perfmon request: flags are not valid"); | |
351 | return -EINVAL; | |
352 | } | |
353 | ||
ef146c00 | 354 | if (etnaviv_pm_req_validate(r, exec_state)) { |
c8e4a7fd CG |
355 | DRM_ERROR("perfmon request: domain or signal not valid"); |
356 | return -EINVAL; | |
357 | } | |
358 | ||
ef146c00 LS |
359 | submit->pmrs[i].flags = r->flags; |
360 | submit->pmrs[i].domain = r->domain; | |
361 | submit->pmrs[i].signal = r->signal; | |
362 | submit->pmrs[i].sequence = r->sequence; | |
363 | submit->pmrs[i].offset = r->read_offset; | |
364 | submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base); | |
c8e4a7fd CG |
365 | } |
366 | ||
367 | return 0; | |
368 | } | |
369 | ||
e0329e6c | 370 | static void submit_cleanup(struct kref *kref) |
a8c21a54 | 371 | { |
e0329e6c LS |
372 | struct etnaviv_gem_submit *submit = |
373 | container_of(kref, struct etnaviv_gem_submit, refcount); | |
a8c21a54 T |
374 | unsigned i; |
375 | ||
8bda1516 LS |
376 | if (submit->runtime_resumed) |
377 | pm_runtime_put_autosuspend(submit->gpu->dev); | |
378 | ||
2f9225db LS |
379 | if (submit->cmdbuf.suballoc) |
380 | etnaviv_cmdbuf_free(&submit->cmdbuf); | |
381 | ||
17e4660a LS |
382 | if (submit->mmu_context) |
383 | etnaviv_iommu_context_put(submit->mmu_context); | |
384 | ||
385 | if (submit->prev_mmu_context) | |
386 | etnaviv_iommu_context_put(submit->prev_mmu_context); | |
387 | ||
a8c21a54 T |
388 | for (i = 0; i < submit->nr_bos; i++) { |
389 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; | |
390 | ||
33a63e68 LS |
391 | /* unpin all objects */ |
392 | if (submit->bos[i].flags & BO_PINNED) { | |
393 | etnaviv_gem_mapping_unreference(submit->bos[i].mapping); | |
5b223e94 | 394 | atomic_dec(&etnaviv_obj->gpu_active); |
33a63e68 LS |
395 | submit->bos[i].mapping = NULL; |
396 | submit->bos[i].flags &= ~BO_PINNED; | |
397 | } | |
398 | ||
0236efe9 | 399 | /* if the GPU submit failed, objects might still be locked */ |
a8c21a54 | 400 | submit_unlock_object(submit, i); |
23d1dd03 | 401 | drm_gem_object_put_unlocked(&etnaviv_obj->base); |
a8c21a54 T |
402 | } |
403 | ||
5b223e94 LS |
404 | wake_up_all(&submit->gpu->fence_event); |
405 | ||
9efabd73 LS |
406 | if (submit->in_fence) |
407 | dma_fence_put(submit->in_fence); | |
e93b6dee LS |
408 | if (submit->out_fence) { |
409 | /* first remove from IDR, so fence can not be found anymore */ | |
a0780bb1 | 410 | mutex_lock(&submit->gpu->fence_lock); |
e93b6dee | 411 | idr_remove(&submit->gpu->fence_idr, submit->out_fence_id); |
a0780bb1 | 412 | mutex_unlock(&submit->gpu->fence_lock); |
10009ea2 | 413 | dma_fence_put(submit->out_fence); |
e93b6dee | 414 | } |
ef146c00 | 415 | kfree(submit->pmrs); |
a8c21a54 T |
416 | kfree(submit); |
417 | } | |
418 | ||
e0329e6c LS |
419 | void etnaviv_submit_put(struct etnaviv_gem_submit *submit) |
420 | { | |
421 | kref_put(&submit->refcount, submit_cleanup); | |
422 | } | |
423 | ||
a8c21a54 T |
424 | int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, |
425 | struct drm_file *file) | |
426 | { | |
e93b6dee | 427 | struct etnaviv_file_private *ctx = file->driver_priv; |
a8c21a54 T |
428 | struct etnaviv_drm_private *priv = dev->dev_private; |
429 | struct drm_etnaviv_gem_submit *args = data; | |
430 | struct drm_etnaviv_gem_submit_reloc *relocs; | |
c8e4a7fd | 431 | struct drm_etnaviv_gem_submit_pmr *pmrs; |
a8c21a54 T |
432 | struct drm_etnaviv_gem_submit_bo *bos; |
433 | struct etnaviv_gem_submit *submit; | |
a8c21a54 | 434 | struct etnaviv_gpu *gpu; |
78ec187f | 435 | struct sync_file *sync_file = NULL; |
08301d73 | 436 | struct ww_acquire_ctx ticket; |
78ec187f | 437 | int out_fence_fd = -1; |
a8c21a54 T |
438 | void *stream; |
439 | int ret; | |
440 | ||
441 | if (args->pipe >= ETNA_MAX_PIPES) | |
442 | return -EINVAL; | |
443 | ||
444 | gpu = priv->gpu[args->pipe]; | |
445 | if (!gpu) | |
446 | return -ENXIO; | |
447 | ||
448 | if (args->stream_size % 4) { | |
449 | DRM_ERROR("non-aligned cmdstream buffer size: %u\n", | |
450 | args->stream_size); | |
451 | return -EINVAL; | |
452 | } | |
453 | ||
454 | if (args->exec_state != ETNA_PIPE_3D && | |
455 | args->exec_state != ETNA_PIPE_2D && | |
456 | args->exec_state != ETNA_PIPE_VG) { | |
457 | DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); | |
458 | return -EINVAL; | |
459 | } | |
460 | ||
9ad59fea PZ |
461 | if (args->flags & ~ETNA_SUBMIT_FLAGS) { |
462 | DRM_ERROR("invalid flags: 0x%x\n", args->flags); | |
463 | return -EINVAL; | |
464 | } | |
465 | ||
088880dd LS |
466 | if ((args->flags & ETNA_SUBMIT_SOFTPIN) && |
467 | priv->mmu_global->version != ETNAVIV_IOMMU_V2) { | |
468 | DRM_ERROR("softpin requested on incompatible MMU\n"); | |
469 | return -EINVAL; | |
470 | } | |
471 | ||
a8c21a54 T |
472 | /* |
473 | * Copy the command submission and bo array to kernel space in | |
474 | * one go, and do this outside of any locks. | |
475 | */ | |
2098105e MH |
476 | bos = kvmalloc_array(args->nr_bos, sizeof(*bos), GFP_KERNEL); |
477 | relocs = kvmalloc_array(args->nr_relocs, sizeof(*relocs), GFP_KERNEL); | |
c8e4a7fd | 478 | pmrs = kvmalloc_array(args->nr_pmrs, sizeof(*pmrs), GFP_KERNEL); |
2098105e | 479 | stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL); |
2f9225db | 480 | if (!bos || !relocs || !pmrs || !stream) { |
a8c21a54 T |
481 | ret = -ENOMEM; |
482 | goto err_submit_cmds; | |
483 | } | |
484 | ||
3ed605bc | 485 | ret = copy_from_user(bos, u64_to_user_ptr(args->bos), |
a8c21a54 T |
486 | args->nr_bos * sizeof(*bos)); |
487 | if (ret) { | |
488 | ret = -EFAULT; | |
489 | goto err_submit_cmds; | |
490 | } | |
491 | ||
3ed605bc | 492 | ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs), |
a8c21a54 T |
493 | args->nr_relocs * sizeof(*relocs)); |
494 | if (ret) { | |
495 | ret = -EFAULT; | |
496 | goto err_submit_cmds; | |
497 | } | |
498 | ||
c8e4a7fd CG |
499 | ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs), |
500 | args->nr_pmrs * sizeof(*pmrs)); | |
501 | if (ret) { | |
502 | ret = -EFAULT; | |
503 | goto err_submit_cmds; | |
504 | } | |
c8e4a7fd | 505 | |
3ed605bc | 506 | ret = copy_from_user(stream, u64_to_user_ptr(args->stream), |
a8c21a54 T |
507 | args->stream_size); |
508 | if (ret) { | |
509 | ret = -EFAULT; | |
510 | goto err_submit_cmds; | |
511 | } | |
512 | ||
78ec187f PZ |
513 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
514 | out_fence_fd = get_unused_fd_flags(O_CLOEXEC); | |
515 | if (out_fence_fd < 0) { | |
516 | ret = out_fence_fd; | |
517 | goto err_submit_cmds; | |
518 | } | |
519 | } | |
520 | ||
08301d73 LS |
521 | ww_acquire_init(&ticket, &reservation_ww_class); |
522 | ||
ef146c00 | 523 | submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs); |
a8c21a54 T |
524 | if (!submit) { |
525 | ret = -ENOMEM; | |
08301d73 | 526 | goto err_submit_ww_acquire; |
a8c21a54 T |
527 | } |
528 | ||
bffe5db8 | 529 | ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &submit->cmdbuf, |
2f9225db LS |
530 | ALIGN(args->stream_size, 8) + 8); |
531 | if (ret) | |
532 | goto err_submit_objects; | |
533 | ||
72ac64b8 | 534 | submit->ctx = file->driver_priv; |
17e4660a LS |
535 | etnaviv_iommu_context_get(submit->ctx->mmu); |
536 | submit->mmu_context = submit->ctx->mmu; | |
797b0159 | 537 | submit->exec_state = args->exec_state; |
9ad59fea PZ |
538 | submit->flags = args->flags; |
539 | ||
a8c21a54 T |
540 | ret = submit_lookup_objects(submit, file, bos, args->nr_bos); |
541 | if (ret) | |
542 | goto err_submit_objects; | |
543 | ||
edb5ff07 LS |
544 | if ((priv->mmu_global->version != ETNAVIV_IOMMU_V2) && |
545 | !etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, | |
a8c21a54 T |
546 | relocs, args->nr_relocs)) { |
547 | ret = -EINVAL; | |
548 | goto err_submit_objects; | |
549 | } | |
550 | ||
9ad59fea | 551 | if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) { |
9efabd73 LS |
552 | submit->in_fence = sync_file_get_fence(args->fence_fd); |
553 | if (!submit->in_fence) { | |
9ad59fea PZ |
554 | ret = -EINVAL; |
555 | goto err_submit_objects; | |
556 | } | |
9ad59fea PZ |
557 | } |
558 | ||
a8c21a54 T |
559 | ret = submit_pin_objects(submit); |
560 | if (ret) | |
33a63e68 | 561 | goto err_submit_objects; |
a8c21a54 T |
562 | |
563 | ret = submit_reloc(submit, stream, args->stream_size / 4, | |
564 | relocs, args->nr_relocs); | |
565 | if (ret) | |
33a63e68 | 566 | goto err_submit_objects; |
a8c21a54 | 567 | |
ef146c00 | 568 | ret = submit_perfmon_validate(submit, args->exec_state, pmrs); |
c8e4a7fd | 569 | if (ret) |
33a63e68 | 570 | goto err_submit_objects; |
c8e4a7fd | 571 | |
2f9225db | 572 | memcpy(submit->cmdbuf.vaddr, stream, args->stream_size); |
a8c21a54 | 573 | |
e0580254 LS |
574 | ret = submit_lock_objects(submit, &ticket); |
575 | if (ret) | |
576 | goto err_submit_objects; | |
577 | ||
578 | ret = submit_fence_sync(submit); | |
579 | if (ret) | |
580 | goto err_submit_objects; | |
581 | ||
e93b6dee | 582 | ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit); |
5a642e6b | 583 | if (ret) |
33a63e68 | 584 | goto err_submit_objects; |
5a642e6b | 585 | |
0236efe9 LS |
586 | submit_attach_object_fences(submit); |
587 | ||
78ec187f PZ |
588 | if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { |
589 | /* | |
590 | * This can be improved: ideally we want to allocate the sync | |
591 | * file before kicking off the GPU job and just attach the | |
592 | * fence to the sync file here, eliminating the ENOMEM | |
593 | * possibility at this stage. | |
594 | */ | |
10009ea2 | 595 | sync_file = sync_file_create(submit->out_fence); |
78ec187f PZ |
596 | if (!sync_file) { |
597 | ret = -ENOMEM; | |
33a63e68 | 598 | goto err_submit_objects; |
78ec187f PZ |
599 | } |
600 | fd_install(out_fence_fd, sync_file->file); | |
601 | } | |
602 | ||
603 | args->fence_fd = out_fence_fd; | |
8bc4d885 | 604 | args->fence = submit->out_fence_id; |
a8c21a54 | 605 | |
a8c21a54 | 606 | err_submit_objects: |
e0329e6c | 607 | etnaviv_submit_put(submit); |
a8c21a54 | 608 | |
08301d73 LS |
609 | err_submit_ww_acquire: |
610 | ww_acquire_fini(&ticket); | |
611 | ||
a8c21a54 | 612 | err_submit_cmds: |
78ec187f PZ |
613 | if (ret && (out_fence_fd >= 0)) |
614 | put_unused_fd(out_fence_fd); | |
a8c21a54 | 615 | if (stream) |
2098105e | 616 | kvfree(stream); |
a8c21a54 | 617 | if (bos) |
2098105e | 618 | kvfree(bos); |
a8c21a54 | 619 | if (relocs) |
2098105e | 620 | kvfree(relocs); |
c8e4a7fd CG |
621 | if (pmrs) |
622 | kvfree(pmrs); | |
a8c21a54 T |
623 | |
624 | return ret; | |
625 | } |