1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2020 NVIDIA Corporation */
4 #include <linux/dma-fence-array.h>
5 #include <linux/dma-mapping.h>
6 #include <linux/file.h>
7 #include <linux/host1x.h>
8 #include <linux/iommu.h>
9 #include <linux/kref.h>
10 #include <linux/list.h>
11 #include <linux/nospec.h>
12 #include <linux/pm_runtime.h>
13 #include <linux/scatterlist.h>
14 #include <linux/slab.h>
15 #include <linux/sync_file.h>
17 #include <drm/drm_drv.h>
18 #include <drm/drm_file.h>
19 #include <drm/drm_syncobj.h>
26 #define SUBMIT_ERR(context, fmt, ...) \
27 dev_err_ratelimited(context->client->base.dev, \
28 "%s: job submission failed: " fmt "\n", \
29 current->comm, ##__VA_ARGS__)
32 struct host1x_bo base
;
38 dma_addr_t gather_data_dma
;
39 size_t gather_data_words
;
42 static struct host1x_bo
*gather_bo_get(struct host1x_bo
*host_bo
)
44 struct gather_bo
*bo
= container_of(host_bo
, struct gather_bo
, base
);
51 static void gather_bo_release(struct kref
*ref
)
53 struct gather_bo
*bo
= container_of(ref
, struct gather_bo
, ref
);
55 dma_free_attrs(bo
->dev
, bo
->gather_data_words
* 4, bo
->gather_data
, bo
->gather_data_dma
,
60 static void gather_bo_put(struct host1x_bo
*host_bo
)
62 struct gather_bo
*bo
= container_of(host_bo
, struct gather_bo
, base
);
64 kref_put(&bo
->ref
, gather_bo_release
);
67 static struct host1x_bo_mapping
*
68 gather_bo_pin(struct device
*dev
, struct host1x_bo
*bo
, enum dma_data_direction direction
)
70 struct gather_bo
*gather
= container_of(bo
, struct gather_bo
, base
);
71 struct host1x_bo_mapping
*map
;
74 map
= kzalloc(sizeof(*map
), GFP_KERNEL
);
76 return ERR_PTR(-ENOMEM
);
79 map
->bo
= host1x_bo_get(bo
);
80 map
->direction
= direction
;
83 map
->sgt
= kzalloc(sizeof(*map
->sgt
), GFP_KERNEL
);
89 err
= dma_get_sgtable(gather
->dev
, map
->sgt
, gather
->gather_data
, gather
->gather_data_dma
,
90 gather
->gather_data_words
* 4);
94 err
= dma_map_sgtable(dev
, map
->sgt
, direction
, 0);
98 map
->phys
= sg_dma_address(map
->sgt
->sgl
);
99 map
->size
= gather
->gather_data_words
* 4;
105 sg_free_table(map
->sgt
);
112 static void gather_bo_unpin(struct host1x_bo_mapping
*map
)
117 dma_unmap_sgtable(map
->dev
, map
->sgt
, map
->direction
, 0);
118 sg_free_table(map
->sgt
);
120 host1x_bo_put(map
->bo
);
125 static void *gather_bo_mmap(struct host1x_bo
*host_bo
)
127 struct gather_bo
*bo
= container_of(host_bo
, struct gather_bo
, base
);
129 return bo
->gather_data
;
132 static void gather_bo_munmap(struct host1x_bo
*host_bo
, void *addr
)
136 static const struct host1x_bo_ops gather_bo_ops
= {
137 .get
= gather_bo_get
,
138 .put
= gather_bo_put
,
139 .pin
= gather_bo_pin
,
140 .unpin
= gather_bo_unpin
,
141 .mmap
= gather_bo_mmap
,
142 .munmap
= gather_bo_munmap
,
145 static struct tegra_drm_mapping
*
146 tegra_drm_mapping_get(struct tegra_drm_context
*context
, u32 id
)
148 struct tegra_drm_mapping
*mapping
;
150 xa_lock(&context
->mappings
);
152 mapping
= xa_load(&context
->mappings
, id
);
154 kref_get(&mapping
->ref
);
156 xa_unlock(&context
->mappings
);
161 static void *alloc_copy_user_array(void __user
*from
, size_t count
, size_t size
)
166 if (check_mul_overflow(count
, size
, ©_len
))
167 return ERR_PTR(-EINVAL
);
169 if (copy_len
> 0x4000)
170 return ERR_PTR(-E2BIG
);
172 data
= vmemdup_user(from
, copy_len
);
174 return ERR_CAST(data
);
179 static int submit_copy_gather_data(struct gather_bo
**pbo
, struct device
*dev
,
180 struct tegra_drm_context
*context
,
181 struct drm_tegra_channel_submit
*args
)
183 struct gather_bo
*bo
;
186 if (args
->gather_data_words
== 0) {
187 SUBMIT_ERR(context
, "gather_data_words cannot be zero");
191 if (check_mul_overflow((size_t)args
->gather_data_words
, (size_t)4, ©_len
)) {
192 SUBMIT_ERR(context
, "gather_data_words is too large");
196 bo
= kzalloc(sizeof(*bo
), GFP_KERNEL
);
198 SUBMIT_ERR(context
, "failed to allocate memory for bo info");
202 host1x_bo_init(&bo
->base
, &gather_bo_ops
);
206 bo
->gather_data
= dma_alloc_attrs(dev
, copy_len
, &bo
->gather_data_dma
,
207 GFP_KERNEL
| __GFP_NOWARN
, 0);
208 if (!bo
->gather_data
) {
209 SUBMIT_ERR(context
, "failed to allocate memory for gather data");
214 if (copy_from_user(bo
->gather_data
, u64_to_user_ptr(args
->gather_data_ptr
), copy_len
)) {
215 SUBMIT_ERR(context
, "failed to copy gather data from userspace");
216 dma_free_attrs(dev
, copy_len
, bo
->gather_data
, bo
->gather_data_dma
, 0);
221 bo
->gather_data_words
= args
->gather_data_words
;
228 static int submit_write_reloc(struct tegra_drm_context
*context
, struct gather_bo
*bo
,
229 struct drm_tegra_submit_buf
*buf
, struct tegra_drm_mapping
*mapping
)
231 /* TODO check that target_offset is within bounds */
232 dma_addr_t iova
= mapping
->iova
+ buf
->reloc
.target_offset
;
235 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
236 if (buf
->flags
& DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT
)
240 written_ptr
= iova
>> buf
->reloc
.shift
;
242 if (buf
->reloc
.gather_offset_words
>= bo
->gather_data_words
) {
244 "relocation has too large gather offset (%u vs gather length %zu)",
245 buf
->reloc
.gather_offset_words
, bo
->gather_data_words
);
249 buf
->reloc
.gather_offset_words
= array_index_nospec(buf
->reloc
.gather_offset_words
,
250 bo
->gather_data_words
);
252 bo
->gather_data
[buf
->reloc
.gather_offset_words
] = written_ptr
;
257 static int submit_process_bufs(struct tegra_drm_context
*context
, struct gather_bo
*bo
,
258 struct drm_tegra_channel_submit
*args
,
259 struct tegra_drm_submit_data
*job_data
)
261 struct tegra_drm_used_mapping
*mappings
;
262 struct drm_tegra_submit_buf
*bufs
;
266 bufs
= alloc_copy_user_array(u64_to_user_ptr(args
->bufs_ptr
), args
->num_bufs
,
269 SUBMIT_ERR(context
, "failed to copy bufs array from userspace");
270 return PTR_ERR(bufs
);
273 mappings
= kcalloc(args
->num_bufs
, sizeof(*mappings
), GFP_KERNEL
);
275 SUBMIT_ERR(context
, "failed to allocate memory for mapping info");
280 for (i
= 0; i
< args
->num_bufs
; i
++) {
281 struct drm_tegra_submit_buf
*buf
= &bufs
[i
];
282 struct tegra_drm_mapping
*mapping
;
284 if (buf
->flags
& ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT
) {
285 SUBMIT_ERR(context
, "invalid flag specified for buffer");
290 mapping
= tegra_drm_mapping_get(context
, buf
->mapping
);
292 SUBMIT_ERR(context
, "invalid mapping ID '%u' for buffer", buf
->mapping
);
297 err
= submit_write_reloc(context
, bo
, buf
, mapping
);
299 tegra_drm_mapping_put(mapping
);
303 mappings
[i
].mapping
= mapping
;
304 mappings
[i
].flags
= buf
->flags
;
307 job_data
->used_mappings
= mappings
;
308 job_data
->num_used_mappings
= i
;
316 tegra_drm_mapping_put(mappings
[i
].mapping
);
319 job_data
->used_mappings
= NULL
;
327 static int submit_get_syncpt(struct tegra_drm_context
*context
, struct host1x_job
*job
,
328 struct xarray
*syncpoints
, struct drm_tegra_channel_submit
*args
)
330 struct host1x_syncpt
*sp
;
332 if (args
->syncpt
.flags
) {
333 SUBMIT_ERR(context
, "invalid flag specified for syncpt");
337 /* Syncpt ref will be dropped on job release */
338 sp
= xa_load(syncpoints
, args
->syncpt
.id
);
340 SUBMIT_ERR(context
, "syncpoint specified in syncpt was not allocated");
344 job
->syncpt
= host1x_syncpt_get(sp
);
345 job
->syncpt_incrs
= args
->syncpt
.increments
;
350 static int submit_job_add_gather(struct host1x_job
*job
, struct tegra_drm_context
*context
,
351 struct drm_tegra_submit_cmd_gather_uptr
*cmd
,
352 struct gather_bo
*bo
, u32
*offset
,
353 struct tegra_drm_submit_data
*job_data
,
358 if (cmd
->reserved
[0] || cmd
->reserved
[1] || cmd
->reserved
[2]) {
359 SUBMIT_ERR(context
, "non-zero reserved field in GATHER_UPTR command");
363 /* Check for maximum gather size */
364 if (cmd
->words
> 16383) {
365 SUBMIT_ERR(context
, "too many words in GATHER_UPTR command");
369 if (check_add_overflow(*offset
, cmd
->words
, &next_offset
)) {
370 SUBMIT_ERR(context
, "too many total words in job");
374 if (next_offset
> bo
->gather_data_words
) {
375 SUBMIT_ERR(context
, "GATHER_UPTR command overflows gather data");
379 if (tegra_drm_fw_validate(context
->client
, bo
->gather_data
, *offset
,
380 cmd
->words
, job_data
, class)) {
381 SUBMIT_ERR(context
, "job was rejected by firewall");
385 host1x_job_add_gather(job
, &bo
->base
, cmd
->words
, *offset
* 4);
387 *offset
= next_offset
;
392 static struct host1x_job
*
393 submit_create_job(struct tegra_drm_context
*context
, struct gather_bo
*bo
,
394 struct drm_tegra_channel_submit
*args
, struct tegra_drm_submit_data
*job_data
,
395 struct xarray
*syncpoints
)
397 struct drm_tegra_submit_cmd
*cmds
;
398 u32 i
, gather_offset
= 0, class;
399 struct host1x_job
*job
;
402 /* Set initial class for firewall. */
403 class = context
->client
->base
.class;
405 cmds
= alloc_copy_user_array(u64_to_user_ptr(args
->cmds_ptr
), args
->num_cmds
,
408 SUBMIT_ERR(context
, "failed to copy cmds array from userspace");
409 return ERR_CAST(cmds
);
412 job
= host1x_job_alloc(context
->channel
, args
->num_cmds
, 0, true);
414 SUBMIT_ERR(context
, "failed to allocate memory for job");
415 job
= ERR_PTR(-ENOMEM
);
419 err
= submit_get_syncpt(context
, job
, syncpoints
, args
);
423 job
->client
= &context
->client
->base
;
424 job
->class = context
->client
->base
.class;
425 job
->serialize
= true;
427 for (i
= 0; i
< args
->num_cmds
; i
++) {
428 struct drm_tegra_submit_cmd
*cmd
= &cmds
[i
];
431 SUBMIT_ERR(context
, "unknown flags given for cmd");
436 if (cmd
->type
== DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR
) {
437 err
= submit_job_add_gather(job
, context
, &cmd
->gather_uptr
, bo
,
438 &gather_offset
, job_data
, &class);
441 } else if (cmd
->type
== DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT
) {
442 if (cmd
->wait_syncpt
.reserved
[0] || cmd
->wait_syncpt
.reserved
[1]) {
443 SUBMIT_ERR(context
, "non-zero reserved value");
448 host1x_job_add_wait(job
, cmd
->wait_syncpt
.id
, cmd
->wait_syncpt
.value
,
450 } else if (cmd
->type
== DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE
) {
451 if (cmd
->wait_syncpt
.reserved
[0] || cmd
->wait_syncpt
.reserved
[1]) {
452 SUBMIT_ERR(context
, "non-zero reserved value");
457 if (cmd
->wait_syncpt
.id
!= args
->syncpt
.id
) {
458 SUBMIT_ERR(context
, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job");
463 host1x_job_add_wait(job
, cmd
->wait_syncpt
.id
, cmd
->wait_syncpt
.value
,
466 SUBMIT_ERR(context
, "unknown cmd type");
472 if (gather_offset
== 0) {
473 SUBMIT_ERR(context
, "job must have at least one gather");
490 static void release_job(struct host1x_job
*job
)
492 struct tegra_drm_client
*client
= container_of(job
->client
, struct tegra_drm_client
, base
);
493 struct tegra_drm_submit_data
*job_data
= job
->user_data
;
496 if (job
->memory_context
)
497 host1x_memory_context_put(job
->memory_context
);
499 for (i
= 0; i
< job_data
->num_used_mappings
; i
++)
500 tegra_drm_mapping_put(job_data
->used_mappings
[i
].mapping
);
502 kfree(job_data
->used_mappings
);
505 pm_runtime_mark_last_busy(client
->base
.dev
);
506 pm_runtime_put_autosuspend(client
->base
.dev
);
509 int tegra_drm_ioctl_channel_submit(struct drm_device
*drm
, void *data
,
510 struct drm_file
*file
)
512 struct tegra_drm_file
*fpriv
= file
->driver_priv
;
513 struct drm_tegra_channel_submit
*args
= data
;
514 struct tegra_drm_submit_data
*job_data
;
515 struct drm_syncobj
*syncobj
= NULL
;
516 struct tegra_drm_context
*context
;
517 struct host1x_job
*job
;
518 struct gather_bo
*bo
;
522 mutex_lock(&fpriv
->lock
);
524 context
= xa_load(&fpriv
->contexts
, args
->context
);
526 mutex_unlock(&fpriv
->lock
);
527 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__
,
528 current
->comm
, args
->context
);
532 if (args
->syncobj_in
) {
533 struct dma_fence
*fence
;
535 err
= drm_syncobj_find_fence(file
, args
->syncobj_in
, 0, 0, &fence
);
537 SUBMIT_ERR(context
, "invalid syncobj_in '%#x'", args
->syncobj_in
);
541 err
= dma_fence_wait_timeout(fence
, true, msecs_to_jiffies(10000));
542 dma_fence_put(fence
);
544 SUBMIT_ERR(context
, "wait for syncobj_in timed out");
549 if (args
->syncobj_out
) {
550 syncobj
= drm_syncobj_find(file
, args
->syncobj_out
);
552 SUBMIT_ERR(context
, "invalid syncobj_out '%#x'", args
->syncobj_out
);
558 /* Allocate gather BO and copy gather words in. */
559 err
= submit_copy_gather_data(&bo
, drm
->dev
, context
, args
);
563 job_data
= kzalloc(sizeof(*job_data
), GFP_KERNEL
);
565 SUBMIT_ERR(context
, "failed to allocate memory for job data");
570 /* Get data buffer mappings and do relocation patching. */
571 err
= submit_process_bufs(context
, bo
, args
, job_data
);
575 /* Allocate host1x_job and add gathers and waits to it. */
576 job
= submit_create_job(context
, bo
, args
, job_data
, &fpriv
->syncpoints
);
582 /* Map gather data for Host1x. */
583 err
= host1x_job_pin(job
, context
->client
->base
.dev
);
585 SUBMIT_ERR(context
, "failed to pin job: %d", err
);
589 if (context
->client
->ops
->get_streamid_offset
) {
590 err
= context
->client
->ops
->get_streamid_offset(
591 context
->client
, &job
->engine_streamid_offset
);
593 SUBMIT_ERR(context
, "failed to get streamid offset: %d", err
);
598 if (context
->memory_context
&& context
->client
->ops
->can_use_memory_ctx
) {
601 err
= context
->client
->ops
->can_use_memory_ctx(context
->client
, &supported
);
603 SUBMIT_ERR(context
, "failed to detect if engine can use memory context: %d", err
);
608 job
->memory_context
= context
->memory_context
;
609 host1x_memory_context_get(job
->memory_context
);
611 } else if (context
->client
->ops
->get_streamid_offset
) {
613 * Job submission will need to temporarily change stream ID,
614 * so need to tell it what to change it back to.
616 if (!tegra_dev_iommu_get_stream_id(context
->client
->base
.dev
,
617 &job
->engine_fallback_streamid
))
618 job
->engine_fallback_streamid
= TEGRA_STREAM_ID_BYPASS
;
622 err
= pm_runtime_resume_and_get(context
->client
->base
.dev
);
624 SUBMIT_ERR(context
, "could not power up engine: %d", err
);
625 goto put_memory_context
;
628 job
->user_data
= job_data
;
629 job
->release
= release_job
;
630 job
->timeout
= 10000;
633 * job_data is now part of job reference counting, so don't release
638 /* Submit job to hardware. */
639 err
= host1x_job_submit(job
);
641 SUBMIT_ERR(context
, "host1x job submission failed: %d", err
);
645 /* Return postfences to userspace and add fences to DMA reservations. */
646 args
->syncpt
.value
= job
->syncpt_end
;
649 struct dma_fence
*fence
= host1x_fence_create(job
->syncpt
, job
->syncpt_end
, true);
651 err
= PTR_ERR(fence
);
652 SUBMIT_ERR(context
, "failed to create postfence: %d", err
);
655 drm_syncobj_replace_fence(syncobj
, fence
);
661 if (job
->memory_context
)
662 host1x_memory_context_put(job
->memory_context
);
664 host1x_job_unpin(job
);
668 if (job_data
&& job_data
->used_mappings
) {
669 for (i
= 0; i
< job_data
->num_used_mappings
; i
++)
670 tegra_drm_mapping_put(job_data
->used_mappings
[i
].mapping
);
672 kfree(job_data
->used_mappings
);
677 gather_bo_put(&bo
->base
);
680 drm_syncobj_put(syncobj
);
682 mutex_unlock(&fpriv
->lock
);