2 * Copyright © 2012 Red Hat
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Dave Airlie <airlied@redhat.com>
25 * Rob Clark <rob.clark@linaro.org>
29 #include <linux/export.h>
30 #include <linux/dma-buf.h>
31 #include <linux/rbtree.h>
32 #include <linux/module.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_file.h>
37 #include <drm/drm_framebuffer.h>
38 #include <drm/drm_gem.h>
39 #include <drm/drm_prime.h>
41 #include "drm_internal.h"
43 MODULE_IMPORT_NS(DMA_BUF
);
46 * DOC: overview and lifetime rules
48 * Similar to GEM global names, PRIME file descriptors are also used to share
49 * buffer objects across processes. They offer additional security: as file
50 * descriptors must be explicitly sent over UNIX domain sockets to be shared
51 * between applications, they can't be guessed like the globally unique GEM
54 * Drivers that support the PRIME API implement the drm_gem_object_funcs.export
55 * and &drm_driver.gem_prime_import hooks. &dma_buf_ops implementations for
56 * drivers are all individually exported for drivers which need to overwrite
57 * or reimplement some of them.
59 * Reference Counting for GEM Drivers
60 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
62 * On the export the &dma_buf holds a reference to the exported buffer object,
63 * usually a &drm_gem_object. It takes this reference in the PRIME_HANDLE_TO_FD
64 * IOCTL, when it first calls &drm_gem_object_funcs.export
65 * and stores the exporting GEM object in the &dma_buf.priv field. This
66 * reference needs to be released when the final reference to the &dma_buf
67 * itself is dropped and its &dma_buf_ops.release function is called. For
68 * GEM-based drivers, the &dma_buf should be exported using
69 * drm_gem_dmabuf_export() and then released by drm_gem_dmabuf_release().
71 * Thus the chain of references always flows in one direction, avoiding loops:
72 * importing GEM object -> dma-buf -> exported GEM bo. A further complication
73 * are the lookup caches for import and export. These are required to guarantee
74 * that any given object will always have only one unique userspace handle. This
75 * is required to allow userspace to detect duplicated imports, since some GEM
76 * drivers do fail command submissions if a given buffer object is listed more
77 * than once. These import and export caches in &drm_prime_file_private only
78 * retain a weak reference, which is cleaned up when the corresponding object is
81 * Self-importing: If userspace is using PRIME as a replacement for flink then
82 * it will get a fd->handle request for a GEM object that it created. Drivers
83 * should detect this situation and return back the underlying object from the
84 * dma-buf private. For GEM based drivers this is handled in
85 * drm_gem_prime_import() already.
88 struct drm_prime_member
{
89 struct dma_buf
*dma_buf
;
92 struct rb_node dmabuf_rb
;
93 struct rb_node handle_rb
;
96 static int drm_prime_add_buf_handle(struct drm_prime_file_private
*prime_fpriv
,
97 struct dma_buf
*dma_buf
, uint32_t handle
)
99 struct drm_prime_member
*member
;
100 struct rb_node
**p
, *rb
;
102 member
= kmalloc(sizeof(*member
), GFP_KERNEL
);
106 get_dma_buf(dma_buf
);
107 member
->dma_buf
= dma_buf
;
108 member
->handle
= handle
;
111 p
= &prime_fpriv
->dmabufs
.rb_node
;
113 struct drm_prime_member
*pos
;
116 pos
= rb_entry(rb
, struct drm_prime_member
, dmabuf_rb
);
117 if (dma_buf
> pos
->dma_buf
)
122 rb_link_node(&member
->dmabuf_rb
, rb
, p
);
123 rb_insert_color(&member
->dmabuf_rb
, &prime_fpriv
->dmabufs
);
126 p
= &prime_fpriv
->handles
.rb_node
;
128 struct drm_prime_member
*pos
;
131 pos
= rb_entry(rb
, struct drm_prime_member
, handle_rb
);
132 if (handle
> pos
->handle
)
137 rb_link_node(&member
->handle_rb
, rb
, p
);
138 rb_insert_color(&member
->handle_rb
, &prime_fpriv
->handles
);
143 static struct dma_buf
*drm_prime_lookup_buf_by_handle(struct drm_prime_file_private
*prime_fpriv
,
148 rb
= prime_fpriv
->handles
.rb_node
;
150 struct drm_prime_member
*member
;
152 member
= rb_entry(rb
, struct drm_prime_member
, handle_rb
);
153 if (member
->handle
== handle
)
154 return member
->dma_buf
;
155 else if (member
->handle
< handle
)
164 static int drm_prime_lookup_buf_handle(struct drm_prime_file_private
*prime_fpriv
,
165 struct dma_buf
*dma_buf
,
170 rb
= prime_fpriv
->dmabufs
.rb_node
;
172 struct drm_prime_member
*member
;
174 member
= rb_entry(rb
, struct drm_prime_member
, dmabuf_rb
);
175 if (member
->dma_buf
== dma_buf
) {
176 *handle
= member
->handle
;
178 } else if (member
->dma_buf
< dma_buf
) {
188 void drm_prime_remove_buf_handle(struct drm_prime_file_private
*prime_fpriv
,
193 mutex_lock(&prime_fpriv
->lock
);
195 rb
= prime_fpriv
->handles
.rb_node
;
197 struct drm_prime_member
*member
;
199 member
= rb_entry(rb
, struct drm_prime_member
, handle_rb
);
200 if (member
->handle
== handle
) {
201 rb_erase(&member
->handle_rb
, &prime_fpriv
->handles
);
202 rb_erase(&member
->dmabuf_rb
, &prime_fpriv
->dmabufs
);
204 dma_buf_put(member
->dma_buf
);
207 } else if (member
->handle
< handle
) {
214 mutex_unlock(&prime_fpriv
->lock
);
217 void drm_prime_init_file_private(struct drm_prime_file_private
*prime_fpriv
)
219 mutex_init(&prime_fpriv
->lock
);
220 prime_fpriv
->dmabufs
= RB_ROOT
;
221 prime_fpriv
->handles
= RB_ROOT
;
224 void drm_prime_destroy_file_private(struct drm_prime_file_private
*prime_fpriv
)
226 /* by now drm_gem_release should've made sure the list is empty */
227 WARN_ON(!RB_EMPTY_ROOT(&prime_fpriv
->dmabufs
));
231 * drm_gem_dmabuf_export - &dma_buf export implementation for GEM
232 * @dev: parent device for the exported dmabuf
233 * @exp_info: the export information used by dma_buf_export()
235 * This wraps dma_buf_export() for use by generic GEM drivers that are using
236 * drm_gem_dmabuf_release(). In addition to calling dma_buf_export(), we take
237 * a reference to the &drm_device and the exported &drm_gem_object (stored in
238 * &dma_buf_export_info.priv) which is released by drm_gem_dmabuf_release().
240 * Returns the new dmabuf.
242 struct dma_buf
*drm_gem_dmabuf_export(struct drm_device
*dev
,
243 struct dma_buf_export_info
*exp_info
)
245 struct drm_gem_object
*obj
= exp_info
->priv
;
246 struct dma_buf
*dma_buf
;
248 dma_buf
= dma_buf_export(exp_info
);
253 drm_gem_object_get(obj
);
254 dma_buf
->file
->f_mapping
= obj
->dev
->anon_inode
->i_mapping
;
258 EXPORT_SYMBOL(drm_gem_dmabuf_export
);
261 * drm_gem_dmabuf_release - &dma_buf release implementation for GEM
262 * @dma_buf: buffer to be released
264 * Generic release function for dma_bufs exported as PRIME buffers. GEM drivers
265 * must use this in their &dma_buf_ops structure as the release callback.
266 * drm_gem_dmabuf_release() should be used in conjunction with
267 * drm_gem_dmabuf_export().
269 void drm_gem_dmabuf_release(struct dma_buf
*dma_buf
)
271 struct drm_gem_object
*obj
= dma_buf
->priv
;
272 struct drm_device
*dev
= obj
->dev
;
274 /* drop the reference on the export fd holds */
275 drm_gem_object_put(obj
);
279 EXPORT_SYMBOL(drm_gem_dmabuf_release
);
282 * drm_gem_prime_fd_to_handle - PRIME import function for GEM drivers
283 * @dev: drm_device to import into
284 * @file_priv: drm file-private structure
285 * @prime_fd: fd id of the dma-buf which should be imported
286 * @handle: pointer to storage for the handle of the imported buffer object
288 * This is the PRIME import function which must be used mandatorily by GEM
289 * drivers to ensure correct lifetime management of the underlying GEM object.
290 * The actual importing of GEM object from the dma-buf is done through the
291 * &drm_driver.gem_prime_import driver callback.
293 * Returns 0 on success or a negative error code on failure.
295 int drm_gem_prime_fd_to_handle(struct drm_device
*dev
,
296 struct drm_file
*file_priv
, int prime_fd
,
299 struct dma_buf
*dma_buf
;
300 struct drm_gem_object
*obj
;
303 dma_buf
= dma_buf_get(prime_fd
);
305 return PTR_ERR(dma_buf
);
307 mutex_lock(&file_priv
->prime
.lock
);
309 ret
= drm_prime_lookup_buf_handle(&file_priv
->prime
,
314 /* never seen this one, need to import */
315 mutex_lock(&dev
->object_name_lock
);
316 if (dev
->driver
->gem_prime_import
)
317 obj
= dev
->driver
->gem_prime_import(dev
, dma_buf
);
319 obj
= drm_gem_prime_import(dev
, dma_buf
);
326 WARN_ON(obj
->dma_buf
!= dma_buf
);
328 obj
->dma_buf
= dma_buf
;
329 get_dma_buf(dma_buf
);
332 /* _handle_create_tail unconditionally unlocks dev->object_name_lock. */
333 ret
= drm_gem_handle_create_tail(file_priv
, obj
, handle
);
334 drm_gem_object_put(obj
);
338 ret
= drm_prime_add_buf_handle(&file_priv
->prime
,
340 mutex_unlock(&file_priv
->prime
.lock
);
344 dma_buf_put(dma_buf
);
349 /* hmm, if driver attached, we are relying on the free-object path
350 * to detach.. which seems ok..
352 drm_gem_handle_delete(file_priv
, *handle
);
353 dma_buf_put(dma_buf
);
357 mutex_unlock(&dev
->object_name_lock
);
359 mutex_unlock(&file_priv
->prime
.lock
);
360 dma_buf_put(dma_buf
);
363 EXPORT_SYMBOL(drm_gem_prime_fd_to_handle
);
365 int drm_prime_fd_to_handle_ioctl(struct drm_device
*dev
, void *data
,
366 struct drm_file
*file_priv
)
368 struct drm_prime_handle
*args
= data
;
370 if (dev
->driver
->prime_fd_to_handle
) {
371 return dev
->driver
->prime_fd_to_handle(dev
, file_priv
, args
->fd
,
375 return drm_gem_prime_fd_to_handle(dev
, file_priv
, args
->fd
, &args
->handle
);
378 static struct dma_buf
*export_and_register_object(struct drm_device
*dev
,
379 struct drm_gem_object
*obj
,
382 struct dma_buf
*dmabuf
;
384 /* prevent races with concurrent gem_close. */
385 if (obj
->handle_count
== 0) {
386 dmabuf
= ERR_PTR(-ENOENT
);
390 if (obj
->funcs
&& obj
->funcs
->export
)
391 dmabuf
= obj
->funcs
->export(obj
, flags
);
393 dmabuf
= drm_gem_prime_export(obj
, flags
);
394 if (IS_ERR(dmabuf
)) {
395 /* normally the created dma-buf takes ownership of the ref,
396 * but if that fails then drop the ref
402 * Note that callers do not need to clean up the export cache
403 * since the check for obj->handle_count guarantees that someone
406 obj
->dma_buf
= dmabuf
;
407 get_dma_buf(obj
->dma_buf
);
413 * drm_gem_prime_handle_to_fd - PRIME export function for GEM drivers
414 * @dev: dev to export the buffer from
415 * @file_priv: drm file-private structure
416 * @handle: buffer handle to export
417 * @flags: flags like DRM_CLOEXEC
418 * @prime_fd: pointer to storage for the fd id of the create dma-buf
420 * This is the PRIME export function which must be used mandatorily by GEM
421 * drivers to ensure correct lifetime management of the underlying GEM object.
422 * The actual exporting from GEM object to a dma-buf is done through the
423 * &drm_gem_object_funcs.export callback.
425 int drm_gem_prime_handle_to_fd(struct drm_device
*dev
,
426 struct drm_file
*file_priv
, uint32_t handle
,
430 struct drm_gem_object
*obj
;
432 struct dma_buf
*dmabuf
;
434 mutex_lock(&file_priv
->prime
.lock
);
435 obj
= drm_gem_object_lookup(file_priv
, handle
);
441 dmabuf
= drm_prime_lookup_buf_by_handle(&file_priv
->prime
, handle
);
444 goto out_have_handle
;
447 mutex_lock(&dev
->object_name_lock
);
448 /* re-export the original imported object */
449 if (obj
->import_attach
) {
450 dmabuf
= obj
->import_attach
->dmabuf
;
456 get_dma_buf(obj
->dma_buf
);
457 dmabuf
= obj
->dma_buf
;
461 dmabuf
= export_and_register_object(dev
, obj
, flags
);
462 if (IS_ERR(dmabuf
)) {
463 /* normally the created dma-buf takes ownership of the ref,
464 * but if that fails then drop the ref
466 ret
= PTR_ERR(dmabuf
);
467 mutex_unlock(&dev
->object_name_lock
);
473 * If we've exported this buffer then cheat and add it to the import list
474 * so we get the correct handle back. We must do this under the
475 * protection of dev->object_name_lock to ensure that a racing gem close
476 * ioctl doesn't miss to remove this buffer handle from the cache.
478 ret
= drm_prime_add_buf_handle(&file_priv
->prime
,
480 mutex_unlock(&dev
->object_name_lock
);
482 goto fail_put_dmabuf
;
485 ret
= dma_buf_fd(dmabuf
, flags
);
487 * We must _not_ remove the buffer from the handle cache since the newly
488 * created dma buf is already linked in the global obj->dma_buf pointer,
489 * and that is invariant as long as a userspace gem handle exists.
490 * Closing the handle will clean out the cache anyway, so we don't leak.
493 goto fail_put_dmabuf
;
504 drm_gem_object_put(obj
);
506 mutex_unlock(&file_priv
->prime
.lock
);
510 EXPORT_SYMBOL(drm_gem_prime_handle_to_fd
);
512 int drm_prime_handle_to_fd_ioctl(struct drm_device
*dev
, void *data
,
513 struct drm_file
*file_priv
)
515 struct drm_prime_handle
*args
= data
;
517 /* check flags are valid */
518 if (args
->flags
& ~(DRM_CLOEXEC
| DRM_RDWR
))
521 if (dev
->driver
->prime_handle_to_fd
) {
522 return dev
->driver
->prime_handle_to_fd(dev
, file_priv
,
523 args
->handle
, args
->flags
,
526 return drm_gem_prime_handle_to_fd(dev
, file_priv
, args
->handle
,
527 args
->flags
, &args
->fd
);
533 * Drivers can implement &drm_gem_object_funcs.export and
534 * &drm_driver.gem_prime_import in terms of simpler APIs by using the helper
535 * functions drm_gem_prime_export() and drm_gem_prime_import(). These functions
536 * implement dma-buf support in terms of some lower-level helpers, which are
537 * again exported for drivers to use individually:
542 * Optional pinning of buffers is handled at dma-buf attach and detach time in
543 * drm_gem_map_attach() and drm_gem_map_detach(). Backing storage itself is
544 * handled by drm_gem_map_dma_buf() and drm_gem_unmap_dma_buf(), which relies on
545 * &drm_gem_object_funcs.get_sg_table. If &drm_gem_object_funcs.get_sg_table is
546 * unimplemented, exports into another device are rejected.
548 * For kernel-internal access there's drm_gem_dmabuf_vmap() and
549 * drm_gem_dmabuf_vunmap(). Userspace mmap support is provided by
550 * drm_gem_dmabuf_mmap().
552 * Note that these export helpers can only be used if the underlying backing
553 * storage is fully coherent and either permanently pinned, or it is safe to pin
556 * FIXME: The underlying helper functions are named rather inconsistently.
561 * Importing dma-bufs using drm_gem_prime_import() relies on
562 * &drm_driver.gem_prime_import_sg_table.
564 * Note that similarly to the export helpers this permanently pins the
565 * underlying backing storage. Which is ok for scanout, but is not the best
566 * option for sharing lots of buffers for rendering.
570 * drm_gem_map_attach - dma_buf attach implementation for GEM
571 * @dma_buf: buffer to attach device to
572 * @attach: buffer attachment data
574 * Calls &drm_gem_object_funcs.pin for device specific handling. This can be
575 * used as the &dma_buf_ops.attach callback. Must be used together with
576 * drm_gem_map_detach().
578 * Returns 0 on success, negative error code on failure.
580 int drm_gem_map_attach(struct dma_buf
*dma_buf
,
581 struct dma_buf_attachment
*attach
)
583 struct drm_gem_object
*obj
= dma_buf
->priv
;
585 if (!obj
->funcs
->get_sg_table
)
588 return drm_gem_pin(obj
);
590 EXPORT_SYMBOL(drm_gem_map_attach
);
593 * drm_gem_map_detach - dma_buf detach implementation for GEM
594 * @dma_buf: buffer to detach from
595 * @attach: attachment to be detached
597 * Calls &drm_gem_object_funcs.pin for device specific handling. Cleans up
598 * &dma_buf_attachment from drm_gem_map_attach(). This can be used as the
599 * &dma_buf_ops.detach callback.
601 void drm_gem_map_detach(struct dma_buf
*dma_buf
,
602 struct dma_buf_attachment
*attach
)
604 struct drm_gem_object
*obj
= dma_buf
->priv
;
608 EXPORT_SYMBOL(drm_gem_map_detach
);
611 * drm_gem_map_dma_buf - map_dma_buf implementation for GEM
612 * @attach: attachment whose scatterlist is to be returned
613 * @dir: direction of DMA transfer
615 * Calls &drm_gem_object_funcs.get_sg_table and then maps the scatterlist. This
616 * can be used as the &dma_buf_ops.map_dma_buf callback. Should be used together
617 * with drm_gem_unmap_dma_buf().
619 * Returns:sg_table containing the scatterlist to be returned; returns ERR_PTR
620 * on error. May return -EINTR if it is interrupted by a signal.
622 struct sg_table
*drm_gem_map_dma_buf(struct dma_buf_attachment
*attach
,
623 enum dma_data_direction dir
)
625 struct drm_gem_object
*obj
= attach
->dmabuf
->priv
;
626 struct sg_table
*sgt
;
629 if (WARN_ON(dir
== DMA_NONE
))
630 return ERR_PTR(-EINVAL
);
632 if (WARN_ON(!obj
->funcs
->get_sg_table
))
633 return ERR_PTR(-ENOSYS
);
635 sgt
= obj
->funcs
->get_sg_table(obj
);
639 ret
= dma_map_sgtable(attach
->dev
, sgt
, dir
,
640 DMA_ATTR_SKIP_CPU_SYNC
);
649 EXPORT_SYMBOL(drm_gem_map_dma_buf
);
652 * drm_gem_unmap_dma_buf - unmap_dma_buf implementation for GEM
653 * @attach: attachment to unmap buffer from
654 * @sgt: scatterlist info of the buffer to unmap
655 * @dir: direction of DMA transfer
657 * This can be used as the &dma_buf_ops.unmap_dma_buf callback.
659 void drm_gem_unmap_dma_buf(struct dma_buf_attachment
*attach
,
660 struct sg_table
*sgt
,
661 enum dma_data_direction dir
)
666 dma_unmap_sgtable(attach
->dev
, sgt
, dir
, DMA_ATTR_SKIP_CPU_SYNC
);
670 EXPORT_SYMBOL(drm_gem_unmap_dma_buf
);
673 * drm_gem_dmabuf_vmap - dma_buf vmap implementation for GEM
674 * @dma_buf: buffer to be mapped
675 * @map: the virtual address of the buffer
677 * Sets up a kernel virtual mapping. This can be used as the &dma_buf_ops.vmap
678 * callback. Calls into &drm_gem_object_funcs.vmap for device specific handling.
679 * The kernel virtual address is returned in map.
681 * Returns 0 on success or a negative errno code otherwise.
683 int drm_gem_dmabuf_vmap(struct dma_buf
*dma_buf
, struct iosys_map
*map
)
685 struct drm_gem_object
*obj
= dma_buf
->priv
;
687 return drm_gem_vmap(obj
, map
);
689 EXPORT_SYMBOL(drm_gem_dmabuf_vmap
);
692 * drm_gem_dmabuf_vunmap - dma_buf vunmap implementation for GEM
693 * @dma_buf: buffer to be unmapped
694 * @map: the virtual address of the buffer
696 * Releases a kernel virtual mapping. This can be used as the
697 * &dma_buf_ops.vunmap callback. Calls into &drm_gem_object_funcs.vunmap for device specific handling.
699 void drm_gem_dmabuf_vunmap(struct dma_buf
*dma_buf
, struct iosys_map
*map
)
701 struct drm_gem_object
*obj
= dma_buf
->priv
;
703 drm_gem_vunmap(obj
, map
);
705 EXPORT_SYMBOL(drm_gem_dmabuf_vunmap
);
708 * drm_gem_prime_mmap - PRIME mmap function for GEM drivers
710 * @vma: Virtual address range
712 * This function sets up a userspace mapping for PRIME exported buffers using
713 * the same codepath that is used for regular GEM buffer mapping on the DRM fd.
714 * The fake GEM offset is added to vma->vm_pgoff and &drm_driver->fops->mmap is
715 * called to set up the mapping.
717 int drm_gem_prime_mmap(struct drm_gem_object
*obj
, struct vm_area_struct
*vma
)
719 struct drm_file
*priv
;
723 /* Add the fake offset */
724 vma
->vm_pgoff
+= drm_vma_node_start(&obj
->vma_node
);
726 if (obj
->funcs
&& obj
->funcs
->mmap
) {
727 vma
->vm_ops
= obj
->funcs
->vm_ops
;
729 drm_gem_object_get(obj
);
730 ret
= obj
->funcs
->mmap(obj
, vma
);
732 drm_gem_object_put(obj
);
735 vma
->vm_private_data
= obj
;
739 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
740 fil
= kzalloc(sizeof(*fil
), GFP_KERNEL
);
746 /* Used by drm_gem_mmap() to lookup the GEM object */
747 priv
->minor
= obj
->dev
->primary
;
748 fil
->private_data
= priv
;
750 ret
= drm_vma_node_allow(&obj
->vma_node
, priv
);
754 ret
= obj
->dev
->driver
->fops
->mmap(fil
, vma
);
756 drm_vma_node_revoke(&obj
->vma_node
, priv
);
763 EXPORT_SYMBOL(drm_gem_prime_mmap
);
766 * drm_gem_dmabuf_mmap - dma_buf mmap implementation for GEM
767 * @dma_buf: buffer to be mapped
768 * @vma: virtual address range
770 * Provides memory mapping for the buffer. This can be used as the
771 * &dma_buf_ops.mmap callback. It just forwards to drm_gem_prime_mmap().
773 * Returns 0 on success or a negative error code on failure.
775 int drm_gem_dmabuf_mmap(struct dma_buf
*dma_buf
, struct vm_area_struct
*vma
)
777 struct drm_gem_object
*obj
= dma_buf
->priv
;
779 return drm_gem_prime_mmap(obj
, vma
);
781 EXPORT_SYMBOL(drm_gem_dmabuf_mmap
);
783 static const struct dma_buf_ops drm_gem_prime_dmabuf_ops
= {
784 .cache_sgt_mapping
= true,
785 .attach
= drm_gem_map_attach
,
786 .detach
= drm_gem_map_detach
,
787 .map_dma_buf
= drm_gem_map_dma_buf
,
788 .unmap_dma_buf
= drm_gem_unmap_dma_buf
,
789 .release
= drm_gem_dmabuf_release
,
790 .mmap
= drm_gem_dmabuf_mmap
,
791 .vmap
= drm_gem_dmabuf_vmap
,
792 .vunmap
= drm_gem_dmabuf_vunmap
,
796 * drm_prime_pages_to_sg - converts a page array into an sg list
798 * @pages: pointer to the array of page pointers to convert
799 * @nr_pages: length of the page vector
801 * This helper creates an sg table object from a set of pages
802 * the driver is responsible for mapping the pages into the
803 * importers address space for use with dma_buf itself.
805 * This is useful for implementing &drm_gem_object_funcs.get_sg_table.
807 struct sg_table
*drm_prime_pages_to_sg(struct drm_device
*dev
,
808 struct page
**pages
, unsigned int nr_pages
)
811 size_t max_segment
= 0;
814 sg
= kmalloc(sizeof(struct sg_table
), GFP_KERNEL
);
816 return ERR_PTR(-ENOMEM
);
819 max_segment
= dma_max_mapping_size(dev
->dev
);
820 if (max_segment
== 0)
821 max_segment
= UINT_MAX
;
822 err
= sg_alloc_table_from_pages_segment(sg
, pages
, nr_pages
, 0,
823 nr_pages
<< PAGE_SHIFT
,
824 max_segment
, GFP_KERNEL
);
831 EXPORT_SYMBOL(drm_prime_pages_to_sg
);
834 * drm_prime_get_contiguous_size - returns the contiguous size of the buffer
835 * @sgt: sg_table describing the buffer to check
837 * This helper calculates the contiguous size in the DMA address space
838 * of the buffer described by the provided sg_table.
840 * This is useful for implementing
841 * &drm_gem_object_funcs.gem_prime_import_sg_table.
843 unsigned long drm_prime_get_contiguous_size(struct sg_table
*sgt
)
845 dma_addr_t expected
= sg_dma_address(sgt
->sgl
);
846 struct scatterlist
*sg
;
847 unsigned long size
= 0;
850 for_each_sgtable_dma_sg(sgt
, sg
, i
) {
851 unsigned int len
= sg_dma_len(sg
);
855 if (sg_dma_address(sg
) != expected
)
862 EXPORT_SYMBOL(drm_prime_get_contiguous_size
);
865 * drm_gem_prime_export - helper library implementation of the export callback
866 * @obj: GEM object to export
867 * @flags: flags like DRM_CLOEXEC and DRM_RDWR
869 * This is the implementation of the &drm_gem_object_funcs.export functions for GEM drivers
870 * using the PRIME helpers. It is used as the default in
871 * drm_gem_prime_handle_to_fd().
873 struct dma_buf
*drm_gem_prime_export(struct drm_gem_object
*obj
,
876 struct drm_device
*dev
= obj
->dev
;
877 struct dma_buf_export_info exp_info
= {
878 .exp_name
= KBUILD_MODNAME
, /* white lie for debug */
879 .owner
= dev
->driver
->fops
->owner
,
880 .ops
= &drm_gem_prime_dmabuf_ops
,
887 return drm_gem_dmabuf_export(dev
, &exp_info
);
889 EXPORT_SYMBOL(drm_gem_prime_export
);
892 * drm_gem_prime_import_dev - core implementation of the import callback
893 * @dev: drm_device to import into
894 * @dma_buf: dma-buf object to import
895 * @attach_dev: struct device to dma_buf attach
897 * This is the core of drm_gem_prime_import(). It's designed to be called by
898 * drivers who want to use a different device structure than &drm_device.dev for
899 * attaching via dma_buf. This function calls
900 * &drm_driver.gem_prime_import_sg_table internally.
902 * Drivers must arrange to call drm_prime_gem_destroy() from their
903 * &drm_gem_object_funcs.free hook when using this function.
905 struct drm_gem_object
*drm_gem_prime_import_dev(struct drm_device
*dev
,
906 struct dma_buf
*dma_buf
,
907 struct device
*attach_dev
)
909 struct dma_buf_attachment
*attach
;
910 struct sg_table
*sgt
;
911 struct drm_gem_object
*obj
;
914 if (dma_buf
->ops
== &drm_gem_prime_dmabuf_ops
) {
916 if (obj
->dev
== dev
) {
918 * Importing dmabuf exported from our own gem increases
919 * refcount on gem itself instead of f_count of dmabuf.
921 drm_gem_object_get(obj
);
926 if (!dev
->driver
->gem_prime_import_sg_table
)
927 return ERR_PTR(-EINVAL
);
929 attach
= dma_buf_attach(dma_buf
, attach_dev
);
931 return ERR_CAST(attach
);
933 get_dma_buf(dma_buf
);
935 sgt
= dma_buf_map_attachment_unlocked(attach
, DMA_BIDIRECTIONAL
);
941 obj
= dev
->driver
->gem_prime_import_sg_table(dev
, attach
, sgt
);
947 obj
->import_attach
= attach
;
948 obj
->resv
= dma_buf
->resv
;
953 dma_buf_unmap_attachment_unlocked(attach
, sgt
, DMA_BIDIRECTIONAL
);
955 dma_buf_detach(dma_buf
, attach
);
956 dma_buf_put(dma_buf
);
960 EXPORT_SYMBOL(drm_gem_prime_import_dev
);
963 * drm_gem_prime_import - helper library implementation of the import callback
964 * @dev: drm_device to import into
965 * @dma_buf: dma-buf object to import
967 * This is the implementation of the gem_prime_import functions for GEM drivers
968 * using the PRIME helpers. Drivers can use this as their
969 * &drm_driver.gem_prime_import implementation. It is used as the default
970 * implementation in drm_gem_prime_fd_to_handle().
972 * Drivers must arrange to call drm_prime_gem_destroy() from their
973 * &drm_gem_object_funcs.free hook when using this function.
975 struct drm_gem_object
*drm_gem_prime_import(struct drm_device
*dev
,
976 struct dma_buf
*dma_buf
)
978 return drm_gem_prime_import_dev(dev
, dma_buf
, dev
->dev
);
980 EXPORT_SYMBOL(drm_gem_prime_import
);
983 * drm_prime_sg_to_page_array - convert an sg table into a page array
984 * @sgt: scatter-gather table to convert
985 * @pages: array of page pointers to store the pages in
986 * @max_entries: size of the passed-in array
988 * Exports an sg table into an array of pages.
990 * This function is deprecated and strongly discouraged to be used.
991 * The page array is only useful for page faults and those can corrupt fields
992 * in the struct page if they are not handled by the exporting driver.
994 int __deprecated
drm_prime_sg_to_page_array(struct sg_table
*sgt
,
998 struct sg_page_iter page_iter
;
999 struct page
**p
= pages
;
1001 for_each_sgtable_page(sgt
, &page_iter
, 0) {
1002 if (WARN_ON(p
- pages
>= max_entries
))
1004 *p
++ = sg_page_iter_page(&page_iter
);
1008 EXPORT_SYMBOL(drm_prime_sg_to_page_array
);
1011 * drm_prime_sg_to_dma_addr_array - convert an sg table into a dma addr array
1012 * @sgt: scatter-gather table to convert
1013 * @addrs: array to store the dma bus address of each page
1014 * @max_entries: size of both the passed-in arrays
1016 * Exports an sg table into an array of addresses.
1018 * Drivers should use this in their &drm_driver.gem_prime_import_sg_table
1021 int drm_prime_sg_to_dma_addr_array(struct sg_table
*sgt
, dma_addr_t
*addrs
,
1024 struct sg_dma_page_iter dma_iter
;
1025 dma_addr_t
*a
= addrs
;
1027 for_each_sgtable_dma_page(sgt
, &dma_iter
, 0) {
1028 if (WARN_ON(a
- addrs
>= max_entries
))
1030 *a
++ = sg_page_iter_dma_address(&dma_iter
);
1034 EXPORT_SYMBOL(drm_prime_sg_to_dma_addr_array
);
1037 * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object
1038 * @obj: GEM object which was created from a dma-buf
1039 * @sg: the sg-table which was pinned at import time
1041 * This is the cleanup functions which GEM drivers need to call when they use
1042 * drm_gem_prime_import() or drm_gem_prime_import_dev() to import dma-bufs.
1044 void drm_prime_gem_destroy(struct drm_gem_object
*obj
, struct sg_table
*sg
)
1046 struct dma_buf_attachment
*attach
;
1047 struct dma_buf
*dma_buf
;
1049 attach
= obj
->import_attach
;
1051 dma_buf_unmap_attachment_unlocked(attach
, sg
, DMA_BIDIRECTIONAL
);
1052 dma_buf
= attach
->dmabuf
;
1053 dma_buf_detach(attach
->dmabuf
, attach
);
1054 /* remove the reference */
1055 dma_buf_put(dma_buf
);
1057 EXPORT_SYMBOL(drm_prime_gem_destroy
);