1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3 // Copyright (c) 2018, Linaro Limited
5 #include <linux/completion.h>
6 #include <linux/device.h>
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
10 #include <linux/list.h>
11 #include <linux/miscdevice.h>
12 #include <linux/module.h>
13 #include <linux/of_address.h>
15 #include <linux/of_platform.h>
16 #include <linux/rpmsg.h>
17 #include <linux/scatterlist.h>
18 #include <linux/slab.h>
19 #include <uapi/misc/fastrpc.h>
21 #define ADSP_DOMAIN_ID (0)
22 #define MDSP_DOMAIN_ID (1)
23 #define SDSP_DOMAIN_ID (2)
24 #define CDSP_DOMAIN_ID (3)
25 #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
26 #define FASTRPC_MAX_SESSIONS 9 /*8 compute, 1 cpz*/
27 #define FASTRPC_ALIGN 128
28 #define FASTRPC_MAX_FDLIST 16
29 #define FASTRPC_MAX_CRCLIST 64
30 #define FASTRPC_PHYS(p) ((p) & 0xffffffff)
31 #define FASTRPC_CTX_MAX (256)
32 #define FASTRPC_INIT_HANDLE 1
33 #define FASTRPC_CTXID_MASK (0xFF0)
34 #define INIT_FILELEN_MAX (2 * 1024 * 1024)
35 #define INIT_MEMLEN_MAX (8 * 1024 * 1024)
36 #define FASTRPC_DEVICE_NAME "fastrpc"
38 /* Retrives number of input buffers from the scalars parameter */
39 #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
41 /* Retrives number of output buffers from the scalars parameter */
42 #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
44 /* Retrives number of input handles from the scalars parameter */
45 #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
47 /* Retrives number of output handles from the scalars parameter */
48 #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
50 #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
51 REMOTE_SCALARS_OUTBUFS(sc) + \
52 REMOTE_SCALARS_INHANDLES(sc)+ \
53 REMOTE_SCALARS_OUTHANDLES(sc))
54 #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
55 (((attr & 0x07) << 29) | \
56 ((method & 0x1f) << 24) | \
57 ((in & 0xff) << 16) | \
58 ((out & 0xff) << 8) | \
59 ((oin & 0x0f) << 4) | \
62 #define FASTRPC_SCALARS(method, in, out) \
63 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
65 #define FASTRPC_CREATE_PROCESS_NARGS 6
66 /* Remote Method id table */
67 #define FASTRPC_RMID_INIT_ATTACH 0
68 #define FASTRPC_RMID_INIT_RELEASE 1
69 #define FASTRPC_RMID_INIT_CREATE 6
70 #define FASTRPC_RMID_INIT_CREATE_ATTR 7
71 #define FASTRPC_RMID_INIT_CREATE_STATIC 8
73 #define miscdev_to_cctx(d) container_of(d, struct fastrpc_channel_ctx, miscdev)
75 static const char *domains
[FASTRPC_DEV_MAX
] = { "adsp", "mdsp",
77 struct fastrpc_phy_page
{
78 u64 addr
; /* physical address */
79 u64 size
; /* size of contiguous region */
82 struct fastrpc_invoke_buf
{
83 u32 num
; /* number of contiguous regions */
84 u32 pgidx
; /* index to start of contiguous region */
87 struct fastrpc_remote_arg
{
93 int pid
; /* process group id */
94 int tid
; /* thread id */
95 u64 ctx
; /* invoke caller context */
96 u32 handle
; /* handle to invoke */
97 u32 sc
; /* scalars structure describing the data */
98 u64 addr
; /* physical address */
99 u64 size
; /* size of contiguous region */
102 struct fastrpc_invoke_rsp
{
103 u64 ctx
; /* invoke caller context */
104 int retval
; /* invoke return value */
108 struct fastrpc_user
*fl
;
109 struct dma_buf
*dmabuf
;
114 /* Lock for dma buf attachments */
116 struct list_head attachments
;
119 struct fastrpc_dma_buf_attachment
{
122 struct list_head node
;
126 struct list_head node
;
127 struct fastrpc_user
*fl
;
130 struct sg_table
*table
;
131 struct dma_buf_attachment
*attach
;
136 struct kref refcount
;
139 struct fastrpc_invoke_ctx
{
149 struct kref refcount
;
150 struct list_head node
; /* list of ctxs */
151 struct completion work
;
152 struct fastrpc_msg msg
;
153 struct fastrpc_user
*fl
;
154 struct fastrpc_remote_arg
*rpra
;
155 struct fastrpc_map
**maps
;
156 struct fastrpc_buf
*buf
;
157 struct fastrpc_invoke_args
*args
;
158 struct fastrpc_channel_ctx
*cctx
;
161 struct fastrpc_session_ctx
{
168 struct fastrpc_channel_ctx
{
171 struct rpmsg_device
*rpdev
;
172 struct fastrpc_session_ctx session
[FASTRPC_MAX_SESSIONS
];
175 struct list_head users
;
176 struct miscdevice miscdev
;
179 struct fastrpc_user
{
180 struct list_head user
;
181 struct list_head maps
;
182 struct list_head pending
;
184 struct fastrpc_channel_ctx
*cctx
;
185 struct fastrpc_session_ctx
*sctx
;
186 struct fastrpc_buf
*init_mem
;
192 /* lock for allocations */
196 static void fastrpc_free_map(struct kref
*ref
)
198 struct fastrpc_map
*map
;
200 map
= container_of(ref
, struct fastrpc_map
, refcount
);
203 dma_buf_unmap_attachment(map
->attach
, map
->table
,
205 dma_buf_detach(map
->buf
, map
->attach
);
206 dma_buf_put(map
->buf
);
212 static void fastrpc_map_put(struct fastrpc_map
*map
)
215 kref_put(&map
->refcount
, fastrpc_free_map
);
218 static void fastrpc_map_get(struct fastrpc_map
*map
)
221 kref_get(&map
->refcount
);
224 static int fastrpc_map_find(struct fastrpc_user
*fl
, int fd
,
225 struct fastrpc_map
**ppmap
)
227 struct fastrpc_map
*map
= NULL
;
229 mutex_lock(&fl
->mutex
);
230 list_for_each_entry(map
, &fl
->maps
, node
) {
232 fastrpc_map_get(map
);
234 mutex_unlock(&fl
->mutex
);
238 mutex_unlock(&fl
->mutex
);
243 static void fastrpc_buf_free(struct fastrpc_buf
*buf
)
245 dma_free_coherent(buf
->dev
, buf
->size
, buf
->virt
,
246 FASTRPC_PHYS(buf
->phys
));
250 static int fastrpc_buf_alloc(struct fastrpc_user
*fl
, struct device
*dev
,
251 u64 size
, struct fastrpc_buf
**obuf
)
253 struct fastrpc_buf
*buf
;
255 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
259 INIT_LIST_HEAD(&buf
->attachments
);
260 mutex_init(&buf
->lock
);
268 buf
->virt
= dma_alloc_coherent(dev
, buf
->size
, (dma_addr_t
*)&buf
->phys
,
273 if (fl
->sctx
&& fl
->sctx
->sid
)
274 buf
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
281 static void fastrpc_context_free(struct kref
*ref
)
283 struct fastrpc_invoke_ctx
*ctx
;
284 struct fastrpc_channel_ctx
*cctx
;
287 ctx
= container_of(ref
, struct fastrpc_invoke_ctx
, refcount
);
290 for (i
= 0; i
< ctx
->nscalars
; i
++)
291 fastrpc_map_put(ctx
->maps
[i
]);
294 fastrpc_buf_free(ctx
->buf
);
296 spin_lock(&cctx
->lock
);
297 idr_remove(&cctx
->ctx_idr
, ctx
->ctxid
>> 4);
298 spin_unlock(&cctx
->lock
);
304 static void fastrpc_context_get(struct fastrpc_invoke_ctx
*ctx
)
306 kref_get(&ctx
->refcount
);
309 static void fastrpc_context_put(struct fastrpc_invoke_ctx
*ctx
)
311 kref_put(&ctx
->refcount
, fastrpc_context_free
);
314 static struct fastrpc_invoke_ctx
*fastrpc_context_alloc(
315 struct fastrpc_user
*user
, u32 kernel
, u32 sc
,
316 struct fastrpc_invoke_args
*args
)
318 struct fastrpc_channel_ctx
*cctx
= user
->cctx
;
319 struct fastrpc_invoke_ctx
*ctx
= NULL
;
322 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
324 return ERR_PTR(-ENOMEM
);
326 INIT_LIST_HEAD(&ctx
->node
);
328 ctx
->nscalars
= REMOTE_SCALARS_LENGTH(sc
);
329 ctx
->nbufs
= REMOTE_SCALARS_INBUFS(sc
) +
330 REMOTE_SCALARS_OUTBUFS(sc
);
333 ctx
->maps
= kcalloc(ctx
->nscalars
,
334 sizeof(*ctx
->maps
), GFP_KERNEL
);
337 return ERR_PTR(-ENOMEM
);
344 ctx
->pid
= current
->pid
;
345 ctx
->tgid
= user
->tgid
;
347 init_completion(&ctx
->work
);
349 spin_lock(&user
->lock
);
350 list_add_tail(&ctx
->node
, &user
->pending
);
351 spin_unlock(&user
->lock
);
353 spin_lock(&cctx
->lock
);
354 ret
= idr_alloc_cyclic(&cctx
->ctx_idr
, ctx
, 1,
355 FASTRPC_CTX_MAX
, GFP_ATOMIC
);
357 spin_unlock(&cctx
->lock
);
360 ctx
->ctxid
= ret
<< 4;
361 spin_unlock(&cctx
->lock
);
363 kref_init(&ctx
->refcount
);
367 spin_lock(&user
->lock
);
368 list_del(&ctx
->node
);
369 spin_unlock(&user
->lock
);
376 static struct sg_table
*
377 fastrpc_map_dma_buf(struct dma_buf_attachment
*attachment
,
378 enum dma_data_direction dir
)
380 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
381 struct sg_table
*table
;
385 if (!dma_map_sg(attachment
->dev
, table
->sgl
, table
->nents
, dir
))
386 return ERR_PTR(-ENOMEM
);
391 static void fastrpc_unmap_dma_buf(struct dma_buf_attachment
*attach
,
392 struct sg_table
*table
,
393 enum dma_data_direction dir
)
395 dma_unmap_sg(attach
->dev
, table
->sgl
, table
->nents
, dir
);
398 static void fastrpc_release(struct dma_buf
*dmabuf
)
400 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
402 fastrpc_buf_free(buffer
);
405 static int fastrpc_dma_buf_attach(struct dma_buf
*dmabuf
,
406 struct dma_buf_attachment
*attachment
)
408 struct fastrpc_dma_buf_attachment
*a
;
409 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
412 a
= kzalloc(sizeof(*a
), GFP_KERNEL
);
416 ret
= dma_get_sgtable(buffer
->dev
, &a
->sgt
, buffer
->virt
,
417 FASTRPC_PHYS(buffer
->phys
), buffer
->size
);
419 dev_err(buffer
->dev
, "failed to get scatterlist from DMA API\n");
423 a
->dev
= attachment
->dev
;
424 INIT_LIST_HEAD(&a
->node
);
425 attachment
->priv
= a
;
427 mutex_lock(&buffer
->lock
);
428 list_add(&a
->node
, &buffer
->attachments
);
429 mutex_unlock(&buffer
->lock
);
434 static void fastrpc_dma_buf_detatch(struct dma_buf
*dmabuf
,
435 struct dma_buf_attachment
*attachment
)
437 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
438 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
440 mutex_lock(&buffer
->lock
);
442 mutex_unlock(&buffer
->lock
);
446 static void *fastrpc_kmap(struct dma_buf
*dmabuf
, unsigned long pgnum
)
448 struct fastrpc_buf
*buf
= dmabuf
->priv
;
450 return buf
->virt
? buf
->virt
+ pgnum
* PAGE_SIZE
: NULL
;
453 static void *fastrpc_vmap(struct dma_buf
*dmabuf
)
455 struct fastrpc_buf
*buf
= dmabuf
->priv
;
460 static int fastrpc_mmap(struct dma_buf
*dmabuf
,
461 struct vm_area_struct
*vma
)
463 struct fastrpc_buf
*buf
= dmabuf
->priv
;
464 size_t size
= vma
->vm_end
- vma
->vm_start
;
466 return dma_mmap_coherent(buf
->dev
, vma
, buf
->virt
,
467 FASTRPC_PHYS(buf
->phys
), size
);
470 static const struct dma_buf_ops fastrpc_dma_buf_ops
= {
471 .attach
= fastrpc_dma_buf_attach
,
472 .detach
= fastrpc_dma_buf_detatch
,
473 .map_dma_buf
= fastrpc_map_dma_buf
,
474 .unmap_dma_buf
= fastrpc_unmap_dma_buf
,
475 .mmap
= fastrpc_mmap
,
477 .vmap
= fastrpc_vmap
,
478 .release
= fastrpc_release
,
481 static int fastrpc_map_create(struct fastrpc_user
*fl
, int fd
,
482 u64 len
, struct fastrpc_map
**ppmap
)
484 struct fastrpc_session_ctx
*sess
= fl
->sctx
;
485 struct fastrpc_map
*map
= NULL
;
488 if (!fastrpc_map_find(fl
, fd
, ppmap
))
491 map
= kzalloc(sizeof(*map
), GFP_KERNEL
);
495 INIT_LIST_HEAD(&map
->node
);
498 map
->buf
= dma_buf_get(fd
);
499 if (IS_ERR(map
->buf
)) {
500 err
= PTR_ERR(map
->buf
);
504 map
->attach
= dma_buf_attach(map
->buf
, sess
->dev
);
505 if (IS_ERR(map
->attach
)) {
506 dev_err(sess
->dev
, "Failed to attach dmabuf\n");
507 err
= PTR_ERR(map
->attach
);
511 map
->table
= dma_buf_map_attachment(map
->attach
, DMA_BIDIRECTIONAL
);
512 if (IS_ERR(map
->table
)) {
513 err
= PTR_ERR(map
->table
);
517 map
->phys
= sg_dma_address(map
->table
->sgl
);
518 map
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
520 map
->va
= sg_virt(map
->table
->sgl
);
522 kref_init(&map
->refcount
);
524 spin_lock(&fl
->lock
);
525 list_add_tail(&map
->node
, &fl
->maps
);
526 spin_unlock(&fl
->lock
);
532 dma_buf_detach(map
->buf
, map
->attach
);
534 dma_buf_put(map
->buf
);
542 * Fastrpc payload buffer with metadata looks like:
544 * >>>>>> START of METADATA <<<<<<<<<
545 * +---------------------------------+
547 * | type:(struct fastrpc_remote_arg)|
549 * +---------------------------------+
550 * | Invoke Buffer list |
551 * | type:(struct fastrpc_invoke_buf)|
553 * +---------------------------------+
555 * | type:(struct fastrpc_phy_page) |
557 * +---------------------------------+
559 * |(can be specific to SoC/Firmware)|
560 * +---------------------------------+
561 * >>>>>>>> END of METADATA <<<<<<<<<
562 * +---------------------------------+
565 * +---------------------------------+
568 static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx
*ctx
)
572 size
= (sizeof(struct fastrpc_remote_arg
) +
573 sizeof(struct fastrpc_invoke_buf
) +
574 sizeof(struct fastrpc_phy_page
)) * ctx
->nscalars
+
575 sizeof(u64
) * FASTRPC_MAX_FDLIST
+
576 sizeof(u32
) * FASTRPC_MAX_CRCLIST
;
581 static u64
fastrpc_get_payload_size(struct fastrpc_invoke_ctx
*ctx
, int metalen
)
586 size
= ALIGN(metalen
, FASTRPC_ALIGN
);
587 for (i
= 0; i
< ctx
->nscalars
; i
++) {
588 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1) {
589 size
= ALIGN(size
, FASTRPC_ALIGN
);
590 size
+= ctx
->args
[i
].length
;
597 static int fastrpc_create_maps(struct fastrpc_invoke_ctx
*ctx
)
599 struct device
*dev
= ctx
->fl
->sctx
->dev
;
602 for (i
= 0; i
< ctx
->nscalars
; ++i
) {
603 /* Make sure reserved field is set to 0 */
604 if (ctx
->args
[i
].reserved
)
607 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1 ||
608 ctx
->args
[i
].length
== 0)
611 err
= fastrpc_map_create(ctx
->fl
, ctx
->args
[i
].fd
,
612 ctx
->args
[i
].length
, &ctx
->maps
[i
]);
614 dev_err(dev
, "Error Creating map %d\n", err
);
622 static int fastrpc_get_args(u32 kernel
, struct fastrpc_invoke_ctx
*ctx
)
624 struct device
*dev
= ctx
->fl
->sctx
->dev
;
625 struct fastrpc_remote_arg
*rpra
;
626 struct fastrpc_invoke_buf
*list
;
627 struct fastrpc_phy_page
*pages
;
628 int inbufs
, i
, err
= 0;
634 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
635 metalen
= fastrpc_get_meta_size(ctx
);
636 pkt_size
= fastrpc_get_payload_size(ctx
, metalen
);
638 err
= fastrpc_create_maps(ctx
);
642 ctx
->msg_sz
= pkt_size
;
644 err
= fastrpc_buf_alloc(ctx
->fl
, dev
, pkt_size
, &ctx
->buf
);
648 rpra
= ctx
->buf
->virt
;
649 list
= ctx
->buf
->virt
+ ctx
->nscalars
* sizeof(*rpra
);
650 pages
= ctx
->buf
->virt
+ ctx
->nscalars
* (sizeof(*list
) +
652 args
= (uintptr_t)ctx
->buf
->virt
+ metalen
;
653 rlen
= pkt_size
- metalen
;
656 for (i
= 0; i
< ctx
->nbufs
; ++i
) {
657 u64 len
= ctx
->args
[i
].length
;
661 list
[i
].num
= len
? 1 : 0;
667 pages
[i
].size
= roundup(len
, PAGE_SIZE
);
670 rpra
[i
].pv
= (u64
) ctx
->args
[i
].ptr
;
671 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
673 rlen
-= ALIGN(args
, FASTRPC_ALIGN
) - args
;
674 args
= ALIGN(args
, FASTRPC_ALIGN
);
679 pages
[i
].addr
= ctx
->buf
->phys
+ (pkt_size
- rlen
);
680 pages
[i
].addr
= pages
[i
].addr
& PAGE_MASK
;
685 if (i
< inbufs
&& !ctx
->maps
[i
]) {
686 void *dst
= (void *)(uintptr_t)rpra
[i
].pv
;
687 void *src
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
690 if (copy_from_user(dst
, (void __user
*)src
,
696 memcpy(dst
, src
, len
);
701 for (i
= ctx
->nbufs
; i
< ctx
->nscalars
; ++i
) {
702 rpra
[i
].pv
= (u64
) ctx
->args
[i
].ptr
;
703 rpra
[i
].len
= ctx
->args
[i
].length
;
704 list
[i
].num
= ctx
->args
[i
].length
? 1 : 0;
706 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
707 pages
[i
].size
= ctx
->maps
[i
]->size
;
712 dev_err(dev
, "Error: get invoke args failed:%d\n", err
);
717 static int fastrpc_put_args(struct fastrpc_invoke_ctx
*ctx
,
720 struct fastrpc_remote_arg
*rpra
= ctx
->rpra
;
723 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
725 for (i
= inbufs
; i
< ctx
->nbufs
; ++i
) {
726 void *src
= (void *)(uintptr_t)rpra
[i
].pv
;
727 void *dst
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
728 u64 len
= rpra
[i
].len
;
731 if (copy_to_user((void __user
*)dst
, src
, len
))
734 memcpy(dst
, src
, len
);
741 static int fastrpc_invoke_send(struct fastrpc_session_ctx
*sctx
,
742 struct fastrpc_invoke_ctx
*ctx
,
743 u32 kernel
, uint32_t handle
)
745 struct fastrpc_channel_ctx
*cctx
;
746 struct fastrpc_user
*fl
= ctx
->fl
;
747 struct fastrpc_msg
*msg
= &ctx
->msg
;
751 msg
->tid
= current
->pid
;
756 msg
->ctx
= ctx
->ctxid
| fl
->pd
;
757 msg
->handle
= handle
;
759 msg
->addr
= ctx
->buf
? ctx
->buf
->phys
: 0;
760 msg
->size
= roundup(ctx
->msg_sz
, PAGE_SIZE
);
761 fastrpc_context_get(ctx
);
763 return rpmsg_send(cctx
->rpdev
->ept
, (void *)msg
, sizeof(*msg
));
766 static int fastrpc_internal_invoke(struct fastrpc_user
*fl
, u32 kernel
,
768 struct fastrpc_invoke_args
*args
)
770 struct fastrpc_invoke_ctx
*ctx
= NULL
;
776 ctx
= fastrpc_context_alloc(fl
, kernel
, sc
, args
);
781 err
= fastrpc_get_args(kernel
, ctx
);
785 /* Send invoke buffer to remote dsp */
786 err
= fastrpc_invoke_send(fl
->sctx
, ctx
, kernel
, handle
);
790 /* Wait for remote dsp to respond or time out */
791 err
= wait_for_completion_interruptible(&ctx
->work
);
795 /* Check the response from remote dsp */
801 /* populate all the output buffers with results */
802 err
= fastrpc_put_args(ctx
, kernel
);
808 /* We are done with this compute context, remove it from pending list */
809 spin_lock(&fl
->lock
);
810 list_del(&ctx
->node
);
811 spin_unlock(&fl
->lock
);
812 fastrpc_context_put(ctx
);
815 dev_dbg(fl
->sctx
->dev
, "Error: Invoke Failed %d\n", err
);
820 static int fastrpc_init_create_process(struct fastrpc_user
*fl
,
823 struct fastrpc_init_create init
;
824 struct fastrpc_invoke_args
*args
;
825 struct fastrpc_phy_page pages
[1];
826 struct fastrpc_map
*map
= NULL
;
827 struct fastrpc_buf
*imem
= NULL
;
840 args
= kcalloc(FASTRPC_CREATE_PROCESS_NARGS
, sizeof(*args
), GFP_KERNEL
);
844 if (copy_from_user(&init
, argp
, sizeof(init
))) {
849 if (init
.filelen
> INIT_FILELEN_MAX
) {
854 inbuf
.pgid
= fl
->tgid
;
855 inbuf
.namelen
= strlen(current
->comm
) + 1;
856 inbuf
.filelen
= init
.filelen
;
858 inbuf
.attrs
= init
.attrs
;
859 inbuf
.siglen
= init
.siglen
;
862 if (init
.filelen
&& init
.filefd
) {
863 err
= fastrpc_map_create(fl
, init
.filefd
, init
.filelen
, &map
);
868 memlen
= ALIGN(max(INIT_FILELEN_MAX
, (int)init
.filelen
* 4),
870 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, memlen
,
873 fastrpc_map_put(map
);
878 args
[0].ptr
= (u64
)(uintptr_t)&inbuf
;
879 args
[0].length
= sizeof(inbuf
);
882 args
[1].ptr
= (u64
)(uintptr_t)current
->comm
;
883 args
[1].length
= inbuf
.namelen
;
886 args
[2].ptr
= (u64
) init
.file
;
887 args
[2].length
= inbuf
.filelen
;
888 args
[2].fd
= init
.filefd
;
890 pages
[0].addr
= imem
->phys
;
891 pages
[0].size
= imem
->size
;
893 args
[3].ptr
= (u64
)(uintptr_t) pages
;
894 args
[3].length
= 1 * sizeof(*pages
);
897 args
[4].ptr
= (u64
)(uintptr_t)&inbuf
.attrs
;
898 args
[4].length
= sizeof(inbuf
.attrs
);
901 args
[5].ptr
= (u64
)(uintptr_t) &inbuf
.siglen
;
902 args
[5].length
= sizeof(inbuf
.siglen
);
905 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE
, 4, 0);
907 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR
, 6, 0);
909 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
913 fastrpc_map_put(map
);
914 fastrpc_buf_free(imem
);
923 static struct fastrpc_session_ctx
*fastrpc_session_alloc(
924 struct fastrpc_channel_ctx
*cctx
)
926 struct fastrpc_session_ctx
*session
= NULL
;
929 spin_lock(&cctx
->lock
);
930 for (i
= 0; i
< cctx
->sesscount
; i
++) {
931 if (!cctx
->session
[i
].used
&& cctx
->session
[i
].valid
) {
932 cctx
->session
[i
].used
= true;
933 session
= &cctx
->session
[i
];
937 spin_unlock(&cctx
->lock
);
942 static void fastrpc_session_free(struct fastrpc_channel_ctx
*cctx
,
943 struct fastrpc_session_ctx
*session
)
945 spin_lock(&cctx
->lock
);
946 session
->used
= false;
947 spin_unlock(&cctx
->lock
);
950 static int fastrpc_release_current_dsp_process(struct fastrpc_user
*fl
)
952 struct fastrpc_invoke_args args
[1];
957 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
958 args
[0].length
= sizeof(tgid
);
960 args
[0].reserved
= 0;
961 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE
, 1, 0);
963 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
967 static int fastrpc_device_release(struct inode
*inode
, struct file
*file
)
969 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
970 struct fastrpc_channel_ctx
*cctx
= fl
->cctx
;
971 struct fastrpc_invoke_ctx
*ctx
, *n
;
972 struct fastrpc_map
*map
, *m
;
974 fastrpc_release_current_dsp_process(fl
);
976 spin_lock(&cctx
->lock
);
978 spin_unlock(&cctx
->lock
);
981 fastrpc_buf_free(fl
->init_mem
);
983 list_for_each_entry_safe(ctx
, n
, &fl
->pending
, node
) {
984 list_del(&ctx
->node
);
985 fastrpc_context_put(ctx
);
988 list_for_each_entry_safe(map
, m
, &fl
->maps
, node
) {
989 list_del(&map
->node
);
990 fastrpc_map_put(map
);
993 fastrpc_session_free(cctx
, fl
->sctx
);
995 mutex_destroy(&fl
->mutex
);
997 file
->private_data
= NULL
;
1002 static int fastrpc_device_open(struct inode
*inode
, struct file
*filp
)
1004 struct fastrpc_channel_ctx
*cctx
= miscdev_to_cctx(filp
->private_data
);
1005 struct fastrpc_user
*fl
= NULL
;
1007 fl
= kzalloc(sizeof(*fl
), GFP_KERNEL
);
1011 filp
->private_data
= fl
;
1012 spin_lock_init(&fl
->lock
);
1013 mutex_init(&fl
->mutex
);
1014 INIT_LIST_HEAD(&fl
->pending
);
1015 INIT_LIST_HEAD(&fl
->maps
);
1016 INIT_LIST_HEAD(&fl
->user
);
1017 fl
->tgid
= current
->tgid
;
1020 fl
->sctx
= fastrpc_session_alloc(cctx
);
1022 dev_err(&cctx
->rpdev
->dev
, "No session available\n");
1023 mutex_destroy(&fl
->mutex
);
1029 spin_lock(&cctx
->lock
);
1030 list_add_tail(&fl
->user
, &cctx
->users
);
1031 spin_unlock(&cctx
->lock
);
1036 static int fastrpc_dmabuf_free(struct fastrpc_user
*fl
, char __user
*argp
)
1038 struct dma_buf
*buf
;
1041 if (copy_from_user(&info
, argp
, sizeof(info
)))
1044 buf
= dma_buf_get(info
);
1045 if (IS_ERR_OR_NULL(buf
))
1048 * one for the last get and other for the ALLOC_DMA_BUFF ioctl
1056 static int fastrpc_dmabuf_alloc(struct fastrpc_user
*fl
, char __user
*argp
)
1058 struct fastrpc_alloc_dma_buf bp
;
1059 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
1060 struct fastrpc_buf
*buf
= NULL
;
1063 if (copy_from_user(&bp
, argp
, sizeof(bp
)))
1066 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, bp
.size
, &buf
);
1069 exp_info
.ops
= &fastrpc_dma_buf_ops
;
1070 exp_info
.size
= bp
.size
;
1071 exp_info
.flags
= O_RDWR
;
1072 exp_info
.priv
= buf
;
1073 buf
->dmabuf
= dma_buf_export(&exp_info
);
1074 if (IS_ERR(buf
->dmabuf
)) {
1075 err
= PTR_ERR(buf
->dmabuf
);
1076 fastrpc_buf_free(buf
);
1080 bp
.fd
= dma_buf_fd(buf
->dmabuf
, O_ACCMODE
);
1082 dma_buf_put(buf
->dmabuf
);
1086 if (copy_to_user(argp
, &bp
, sizeof(bp
))) {
1087 dma_buf_put(buf
->dmabuf
);
1091 get_dma_buf(buf
->dmabuf
);
1096 static int fastrpc_init_attach(struct fastrpc_user
*fl
)
1098 struct fastrpc_invoke_args args
[1];
1099 int tgid
= fl
->tgid
;
1102 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
1103 args
[0].length
= sizeof(tgid
);
1105 args
[0].reserved
= 0;
1106 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH
, 1, 0);
1109 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1113 static int fastrpc_invoke(struct fastrpc_user
*fl
, char __user
*argp
)
1115 struct fastrpc_invoke_args
*args
= NULL
;
1116 struct fastrpc_invoke inv
;
1120 if (copy_from_user(&inv
, argp
, sizeof(inv
)))
1123 /* nscalars is truncated here to max supported value */
1124 nscalars
= REMOTE_SCALARS_LENGTH(inv
.sc
);
1126 args
= kcalloc(nscalars
, sizeof(*args
), GFP_KERNEL
);
1130 if (copy_from_user(args
, (void __user
*)(uintptr_t)inv
.args
,
1131 nscalars
* sizeof(*args
))) {
1137 err
= fastrpc_internal_invoke(fl
, false, inv
.handle
, inv
.sc
, args
);
1143 static long fastrpc_device_ioctl(struct file
*file
, unsigned int cmd
,
1146 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
1147 char __user
*argp
= (char __user
*)arg
;
1151 case FASTRPC_IOCTL_INVOKE
:
1152 err
= fastrpc_invoke(fl
, argp
);
1154 case FASTRPC_IOCTL_INIT_ATTACH
:
1155 err
= fastrpc_init_attach(fl
);
1157 case FASTRPC_IOCTL_INIT_CREATE
:
1158 err
= fastrpc_init_create_process(fl
, argp
);
1160 case FASTRPC_IOCTL_FREE_DMA_BUFF
:
1161 err
= fastrpc_dmabuf_free(fl
, argp
);
1163 case FASTRPC_IOCTL_ALLOC_DMA_BUFF
:
1164 err
= fastrpc_dmabuf_alloc(fl
, argp
);
1174 static const struct file_operations fastrpc_fops
= {
1175 .open
= fastrpc_device_open
,
1176 .release
= fastrpc_device_release
,
1177 .unlocked_ioctl
= fastrpc_device_ioctl
,
1178 .compat_ioctl
= fastrpc_device_ioctl
,
1181 static int fastrpc_cb_probe(struct platform_device
*pdev
)
1183 struct fastrpc_channel_ctx
*cctx
;
1184 struct fastrpc_session_ctx
*sess
;
1185 struct device
*dev
= &pdev
->dev
;
1186 int i
, sessions
= 0;
1188 cctx
= dev_get_drvdata(dev
->parent
);
1192 of_property_read_u32(dev
->of_node
, "qcom,nsessions", &sessions
);
1194 spin_lock(&cctx
->lock
);
1195 sess
= &cctx
->session
[cctx
->sesscount
];
1199 dev_set_drvdata(dev
, sess
);
1201 if (of_property_read_u32(dev
->of_node
, "reg", &sess
->sid
))
1202 dev_info(dev
, "FastRPC Session ID not specified in DT\n");
1205 struct fastrpc_session_ctx
*dup_sess
;
1207 for (i
= 1; i
< sessions
; i
++) {
1208 if (cctx
->sesscount
++ >= FASTRPC_MAX_SESSIONS
)
1210 dup_sess
= &cctx
->session
[cctx
->sesscount
];
1211 memcpy(dup_sess
, sess
, sizeof(*dup_sess
));
1215 spin_unlock(&cctx
->lock
);
1216 dma_set_mask(dev
, DMA_BIT_MASK(32));
1221 static int fastrpc_cb_remove(struct platform_device
*pdev
)
1223 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(pdev
->dev
.parent
);
1224 struct fastrpc_session_ctx
*sess
= dev_get_drvdata(&pdev
->dev
);
1227 spin_lock(&cctx
->lock
);
1228 for (i
= 1; i
< FASTRPC_MAX_SESSIONS
; i
++) {
1229 if (cctx
->session
[i
].sid
== sess
->sid
) {
1230 cctx
->session
[i
].valid
= false;
1234 spin_unlock(&cctx
->lock
);
1239 static const struct of_device_id fastrpc_match_table
[] = {
1240 { .compatible
= "qcom,fastrpc-compute-cb", },
1244 static struct platform_driver fastrpc_cb_driver
= {
1245 .probe
= fastrpc_cb_probe
,
1246 .remove
= fastrpc_cb_remove
,
1248 .name
= "qcom,fastrpc-cb",
1249 .of_match_table
= fastrpc_match_table
,
1250 .suppress_bind_attrs
= true,
1254 static int fastrpc_rpmsg_probe(struct rpmsg_device
*rpdev
)
1256 struct device
*rdev
= &rpdev
->dev
;
1257 struct fastrpc_channel_ctx
*data
;
1258 int i
, err
, domain_id
= -1;
1261 data
= devm_kzalloc(rdev
, sizeof(*data
), GFP_KERNEL
);
1265 err
= of_property_read_string(rdev
->of_node
, "label", &domain
);
1267 dev_info(rdev
, "FastRPC Domain not specified in DT\n");
1271 for (i
= 0; i
<= CDSP_DOMAIN_ID
; i
++) {
1272 if (!strcmp(domains
[i
], domain
)) {
1278 if (domain_id
< 0) {
1279 dev_info(rdev
, "FastRPC Invalid Domain ID %d\n", domain_id
);
1283 data
->miscdev
.minor
= MISC_DYNAMIC_MINOR
;
1284 data
->miscdev
.name
= kasprintf(GFP_KERNEL
, "fastrpc-%s",
1285 domains
[domain_id
]);
1286 data
->miscdev
.fops
= &fastrpc_fops
;
1287 err
= misc_register(&data
->miscdev
);
1291 dev_set_drvdata(&rpdev
->dev
, data
);
1292 dma_set_mask_and_coherent(rdev
, DMA_BIT_MASK(32));
1293 INIT_LIST_HEAD(&data
->users
);
1294 spin_lock_init(&data
->lock
);
1295 idr_init(&data
->ctx_idr
);
1296 data
->domain_id
= domain_id
;
1297 data
->rpdev
= rpdev
;
1299 return of_platform_populate(rdev
->of_node
, NULL
, NULL
, rdev
);
1302 static void fastrpc_notify_users(struct fastrpc_user
*user
)
1304 struct fastrpc_invoke_ctx
*ctx
;
1306 spin_lock(&user
->lock
);
1307 list_for_each_entry(ctx
, &user
->pending
, node
)
1308 complete(&ctx
->work
);
1309 spin_unlock(&user
->lock
);
1312 static void fastrpc_rpmsg_remove(struct rpmsg_device
*rpdev
)
1314 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
1315 struct fastrpc_user
*user
;
1317 spin_lock(&cctx
->lock
);
1318 list_for_each_entry(user
, &cctx
->users
, user
)
1319 fastrpc_notify_users(user
);
1320 spin_unlock(&cctx
->lock
);
1322 misc_deregister(&cctx
->miscdev
);
1323 of_platform_depopulate(&rpdev
->dev
);
1327 static int fastrpc_rpmsg_callback(struct rpmsg_device
*rpdev
, void *data
,
1328 int len
, void *priv
, u32 addr
)
1330 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
1331 struct fastrpc_invoke_rsp
*rsp
= data
;
1332 struct fastrpc_invoke_ctx
*ctx
;
1333 unsigned long flags
;
1334 unsigned long ctxid
;
1336 if (len
< sizeof(*rsp
))
1339 ctxid
= ((rsp
->ctx
& FASTRPC_CTXID_MASK
) >> 4);
1341 spin_lock_irqsave(&cctx
->lock
, flags
);
1342 ctx
= idr_find(&cctx
->ctx_idr
, ctxid
);
1343 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1346 dev_err(&rpdev
->dev
, "No context ID matches response\n");
1350 ctx
->retval
= rsp
->retval
;
1351 complete(&ctx
->work
);
1352 fastrpc_context_put(ctx
);
1357 static const struct of_device_id fastrpc_rpmsg_of_match
[] = {
1358 { .compatible
= "qcom,fastrpc" },
1361 MODULE_DEVICE_TABLE(of
, fastrpc_rpmsg_of_match
);
1363 static struct rpmsg_driver fastrpc_driver
= {
1364 .probe
= fastrpc_rpmsg_probe
,
1365 .remove
= fastrpc_rpmsg_remove
,
1366 .callback
= fastrpc_rpmsg_callback
,
1368 .name
= "qcom,fastrpc",
1369 .of_match_table
= fastrpc_rpmsg_of_match
,
1373 static int fastrpc_init(void)
1377 ret
= platform_driver_register(&fastrpc_cb_driver
);
1379 pr_err("fastrpc: failed to register cb driver\n");
1383 ret
= register_rpmsg_driver(&fastrpc_driver
);
1385 pr_err("fastrpc: failed to register rpmsg driver\n");
1386 platform_driver_unregister(&fastrpc_cb_driver
);
1392 module_init(fastrpc_init
);
1394 static void fastrpc_exit(void)
1396 platform_driver_unregister(&fastrpc_cb_driver
);
1397 unregister_rpmsg_driver(&fastrpc_driver
);
1399 module_exit(fastrpc_exit
);
1401 MODULE_LICENSE("GPL v2");