1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
3 // Copyright (c) 2018, Linaro Limited
5 #include <linux/completion.h>
6 #include <linux/device.h>
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/dma-resv.h>
10 #include <linux/idr.h>
11 #include <linux/list.h>
12 #include <linux/miscdevice.h>
13 #include <linux/module.h>
14 #include <linux/of_address.h>
16 #include <linux/platform_device.h>
17 #include <linux/sort.h>
18 #include <linux/of_platform.h>
19 #include <linux/rpmsg.h>
20 #include <linux/scatterlist.h>
21 #include <linux/slab.h>
22 #include <linux/firmware/qcom/qcom_scm.h>
23 #include <uapi/misc/fastrpc.h>
24 #include <linux/of_reserved_mem.h>
26 #define ADSP_DOMAIN_ID (0)
27 #define MDSP_DOMAIN_ID (1)
28 #define SDSP_DOMAIN_ID (2)
29 #define CDSP_DOMAIN_ID (3)
30 #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/
31 #define FASTRPC_MAX_SESSIONS 14
32 #define FASTRPC_MAX_VMIDS 16
33 #define FASTRPC_ALIGN 128
34 #define FASTRPC_MAX_FDLIST 16
35 #define FASTRPC_MAX_CRCLIST 64
36 #define FASTRPC_PHYS(p) ((p) & 0xffffffff)
37 #define FASTRPC_CTX_MAX (256)
38 #define FASTRPC_INIT_HANDLE 1
39 #define FASTRPC_DSP_UTILITIES_HANDLE 2
40 #define FASTRPC_CTXID_MASK (0xFF0)
41 #define INIT_FILELEN_MAX (2 * 1024 * 1024)
42 #define INIT_FILE_NAMELEN_MAX (128)
43 #define FASTRPC_DEVICE_NAME "fastrpc"
45 /* Add memory to static PD pool, protection thru XPU */
46 #define ADSP_MMAP_HEAP_ADDR 4
47 /* MAP static DMA buffer on DSP User PD */
48 #define ADSP_MMAP_DMA_BUFFER 6
49 /* Add memory to static PD pool protection thru hypervisor */
50 #define ADSP_MMAP_REMOTE_HEAP_ADDR 8
51 /* Add memory to userPD pool, for user heap */
52 #define ADSP_MMAP_ADD_PAGES 0x1000
53 /* Add memory to userPD pool, for LLC heap */
54 #define ADSP_MMAP_ADD_PAGES_LLC 0x3000,
56 #define DSP_UNSUPPORTED_API (0x80000414)
57 /* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */
58 #define FASTRPC_MAX_DSP_ATTRIBUTES (256)
59 #define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES)
61 /* Retrives number of input buffers from the scalars parameter */
62 #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff)
64 /* Retrives number of output buffers from the scalars parameter */
65 #define REMOTE_SCALARS_OUTBUFS(sc) (((sc) >> 8) & 0x0ff)
67 /* Retrives number of input handles from the scalars parameter */
68 #define REMOTE_SCALARS_INHANDLES(sc) (((sc) >> 4) & 0x0f)
70 /* Retrives number of output handles from the scalars parameter */
71 #define REMOTE_SCALARS_OUTHANDLES(sc) ((sc) & 0x0f)
73 #define REMOTE_SCALARS_LENGTH(sc) (REMOTE_SCALARS_INBUFS(sc) + \
74 REMOTE_SCALARS_OUTBUFS(sc) + \
75 REMOTE_SCALARS_INHANDLES(sc)+ \
76 REMOTE_SCALARS_OUTHANDLES(sc))
77 #define FASTRPC_BUILD_SCALARS(attr, method, in, out, oin, oout) \
78 (((attr & 0x07) << 29) | \
79 ((method & 0x1f) << 24) | \
80 ((in & 0xff) << 16) | \
81 ((out & 0xff) << 8) | \
82 ((oin & 0x0f) << 4) | \
85 #define FASTRPC_SCALARS(method, in, out) \
86 FASTRPC_BUILD_SCALARS(0, method, in, out, 0, 0)
88 #define FASTRPC_CREATE_PROCESS_NARGS 6
89 #define FASTRPC_CREATE_STATIC_PROCESS_NARGS 3
90 /* Remote Method id table */
91 #define FASTRPC_RMID_INIT_ATTACH 0
92 #define FASTRPC_RMID_INIT_RELEASE 1
93 #define FASTRPC_RMID_INIT_MMAP 4
94 #define FASTRPC_RMID_INIT_MUNMAP 5
95 #define FASTRPC_RMID_INIT_CREATE 6
96 #define FASTRPC_RMID_INIT_CREATE_ATTR 7
97 #define FASTRPC_RMID_INIT_CREATE_STATIC 8
98 #define FASTRPC_RMID_INIT_MEM_MAP 10
99 #define FASTRPC_RMID_INIT_MEM_UNMAP 11
101 /* Protection Domain(PD) ids */
104 #define SENSORS_PD (2)
106 #define miscdev_to_fdevice(d) container_of(d, struct fastrpc_device, miscdev)
108 static const char *domains
[FASTRPC_DEV_MAX
] = { "adsp", "mdsp",
110 struct fastrpc_phy_page
{
111 u64 addr
; /* physical address */
112 u64 size
; /* size of contiguous region */
115 struct fastrpc_invoke_buf
{
116 u32 num
; /* number of contiguous regions */
117 u32 pgidx
; /* index to start of contiguous region */
120 struct fastrpc_remote_dmahandle
{
121 s32 fd
; /* dma handle fd */
122 u32 offset
; /* dma handle offset */
123 u32 len
; /* dma handle length */
126 struct fastrpc_remote_buf
{
127 u64 pv
; /* buffer pointer */
128 u64 len
; /* length of buffer */
131 union fastrpc_remote_arg
{
132 struct fastrpc_remote_buf buf
;
133 struct fastrpc_remote_dmahandle dma
;
136 struct fastrpc_mmap_rsp_msg
{
140 struct fastrpc_mmap_req_msg
{
147 struct fastrpc_mem_map_req_msg
{
157 struct fastrpc_munmap_req_msg
{
163 struct fastrpc_mem_unmap_req_msg
{
171 int pid
; /* process group id */
172 int tid
; /* thread id */
173 u64 ctx
; /* invoke caller context */
174 u32 handle
; /* handle to invoke */
175 u32 sc
; /* scalars structure describing the data */
176 u64 addr
; /* physical address */
177 u64 size
; /* size of contiguous region */
180 struct fastrpc_invoke_rsp
{
181 u64 ctx
; /* invoke caller context */
182 int retval
; /* invoke return value */
185 struct fastrpc_buf_overlap
{
195 struct fastrpc_user
*fl
;
196 struct dma_buf
*dmabuf
;
201 /* Lock for dma buf attachments */
203 struct list_head attachments
;
205 struct list_head node
; /* list of user requested mmaps */
209 struct fastrpc_dma_buf_attachment
{
212 struct list_head node
;
216 struct list_head node
;
217 struct fastrpc_user
*fl
;
220 struct sg_table
*table
;
221 struct dma_buf_attachment
*attach
;
228 struct kref refcount
;
231 struct fastrpc_invoke_ctx
{
241 struct kref refcount
;
242 struct list_head node
; /* list of ctxs */
243 struct completion work
;
244 struct work_struct put_work
;
245 struct fastrpc_msg msg
;
246 struct fastrpc_user
*fl
;
247 union fastrpc_remote_arg
*rpra
;
248 struct fastrpc_map
**maps
;
249 struct fastrpc_buf
*buf
;
250 struct fastrpc_invoke_args
*args
;
251 struct fastrpc_buf_overlap
*olaps
;
252 struct fastrpc_channel_ctx
*cctx
;
255 struct fastrpc_session_ctx
{
262 struct fastrpc_channel_ctx
{
266 struct qcom_scm_vmperm vmperms
[FASTRPC_MAX_VMIDS
];
267 struct rpmsg_device
*rpdev
;
268 struct fastrpc_session_ctx session
[FASTRPC_MAX_SESSIONS
];
271 struct list_head users
;
272 struct kref refcount
;
273 /* Flag if dsp attributes are cached */
274 bool valid_attributes
;
275 u32 dsp_attributes
[FASTRPC_MAX_DSP_ATTRIBUTES
];
276 struct fastrpc_device
*secure_fdevice
;
277 struct fastrpc_device
*fdevice
;
278 struct fastrpc_buf
*remote_heap
;
279 struct list_head invoke_interrupted_mmaps
;
281 bool unsigned_support
;
285 struct fastrpc_device
{
286 struct fastrpc_channel_ctx
*cctx
;
287 struct miscdevice miscdev
;
291 struct fastrpc_user
{
292 struct list_head user
;
293 struct list_head maps
;
294 struct list_head pending
;
295 struct list_head mmaps
;
297 struct fastrpc_channel_ctx
*cctx
;
298 struct fastrpc_session_ctx
*sctx
;
299 struct fastrpc_buf
*init_mem
;
306 /* lock for allocations */
310 static void fastrpc_free_map(struct kref
*ref
)
312 struct fastrpc_map
*map
;
314 map
= container_of(ref
, struct fastrpc_map
, refcount
);
317 if (map
->attr
& FASTRPC_ATTR_SECUREMAP
) {
318 struct qcom_scm_vmperm perm
;
319 int vmid
= map
->fl
->cctx
->vmperms
[0].vmid
;
320 u64 src_perms
= BIT(QCOM_SCM_VMID_HLOS
) | BIT(vmid
);
323 perm
.vmid
= QCOM_SCM_VMID_HLOS
;
324 perm
.perm
= QCOM_SCM_PERM_RWX
;
325 err
= qcom_scm_assign_mem(map
->phys
, map
->size
,
326 &src_perms
, &perm
, 1);
328 dev_err(map
->fl
->sctx
->dev
, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
329 map
->phys
, map
->size
, err
);
333 dma_buf_unmap_attachment_unlocked(map
->attach
, map
->table
,
335 dma_buf_detach(map
->buf
, map
->attach
);
336 dma_buf_put(map
->buf
);
340 spin_lock(&map
->fl
->lock
);
341 list_del(&map
->node
);
342 spin_unlock(&map
->fl
->lock
);
349 static void fastrpc_map_put(struct fastrpc_map
*map
)
352 kref_put(&map
->refcount
, fastrpc_free_map
);
355 static int fastrpc_map_get(struct fastrpc_map
*map
)
360 return kref_get_unless_zero(&map
->refcount
) ? 0 : -ENOENT
;
364 static int fastrpc_map_lookup(struct fastrpc_user
*fl
, int fd
,
365 struct fastrpc_map
**ppmap
, bool take_ref
)
367 struct fastrpc_session_ctx
*sess
= fl
->sctx
;
368 struct fastrpc_map
*map
= NULL
;
371 spin_lock(&fl
->lock
);
372 list_for_each_entry(map
, &fl
->maps
, node
) {
377 ret
= fastrpc_map_get(map
);
379 dev_dbg(sess
->dev
, "%s: Failed to get map fd=%d ret=%d\n",
389 spin_unlock(&fl
->lock
);
394 static void fastrpc_buf_free(struct fastrpc_buf
*buf
)
396 dma_free_coherent(buf
->dev
, buf
->size
, buf
->virt
,
397 FASTRPC_PHYS(buf
->phys
));
401 static int __fastrpc_buf_alloc(struct fastrpc_user
*fl
, struct device
*dev
,
402 u64 size
, struct fastrpc_buf
**obuf
)
404 struct fastrpc_buf
*buf
;
406 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
410 INIT_LIST_HEAD(&buf
->attachments
);
411 INIT_LIST_HEAD(&buf
->node
);
412 mutex_init(&buf
->lock
);
421 buf
->virt
= dma_alloc_coherent(dev
, buf
->size
, (dma_addr_t
*)&buf
->phys
,
424 mutex_destroy(&buf
->lock
);
434 static int fastrpc_buf_alloc(struct fastrpc_user
*fl
, struct device
*dev
,
435 u64 size
, struct fastrpc_buf
**obuf
)
438 struct fastrpc_buf
*buf
;
440 ret
= __fastrpc_buf_alloc(fl
, dev
, size
, obuf
);
446 if (fl
->sctx
&& fl
->sctx
->sid
)
447 buf
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
452 static int fastrpc_remote_heap_alloc(struct fastrpc_user
*fl
, struct device
*dev
,
453 u64 size
, struct fastrpc_buf
**obuf
)
455 struct device
*rdev
= &fl
->cctx
->rpdev
->dev
;
457 return __fastrpc_buf_alloc(fl
, rdev
, size
, obuf
);
460 static void fastrpc_channel_ctx_free(struct kref
*ref
)
462 struct fastrpc_channel_ctx
*cctx
;
464 cctx
= container_of(ref
, struct fastrpc_channel_ctx
, refcount
);
469 static void fastrpc_channel_ctx_get(struct fastrpc_channel_ctx
*cctx
)
471 kref_get(&cctx
->refcount
);
474 static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx
*cctx
)
476 kref_put(&cctx
->refcount
, fastrpc_channel_ctx_free
);
479 static void fastrpc_context_free(struct kref
*ref
)
481 struct fastrpc_invoke_ctx
*ctx
;
482 struct fastrpc_channel_ctx
*cctx
;
486 ctx
= container_of(ref
, struct fastrpc_invoke_ctx
, refcount
);
489 for (i
= 0; i
< ctx
->nbufs
; i
++)
490 fastrpc_map_put(ctx
->maps
[i
]);
493 fastrpc_buf_free(ctx
->buf
);
495 spin_lock_irqsave(&cctx
->lock
, flags
);
496 idr_remove(&cctx
->ctx_idr
, ctx
->ctxid
>> 4);
497 spin_unlock_irqrestore(&cctx
->lock
, flags
);
503 fastrpc_channel_ctx_put(cctx
);
506 static void fastrpc_context_get(struct fastrpc_invoke_ctx
*ctx
)
508 kref_get(&ctx
->refcount
);
511 static void fastrpc_context_put(struct fastrpc_invoke_ctx
*ctx
)
513 kref_put(&ctx
->refcount
, fastrpc_context_free
);
516 static void fastrpc_context_put_wq(struct work_struct
*work
)
518 struct fastrpc_invoke_ctx
*ctx
=
519 container_of(work
, struct fastrpc_invoke_ctx
, put_work
);
521 fastrpc_context_put(ctx
);
524 #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
525 static int olaps_cmp(const void *a
, const void *b
)
527 struct fastrpc_buf_overlap
*pa
= (struct fastrpc_buf_overlap
*)a
;
528 struct fastrpc_buf_overlap
*pb
= (struct fastrpc_buf_overlap
*)b
;
529 /* sort with lowest starting buffer first */
530 int st
= CMP(pa
->start
, pb
->start
);
531 /* sort with highest ending buffer first */
532 int ed
= CMP(pb
->end
, pa
->end
);
534 return st
== 0 ? ed
: st
;
537 static void fastrpc_get_buff_overlaps(struct fastrpc_invoke_ctx
*ctx
)
542 for (i
= 0; i
< ctx
->nbufs
; ++i
) {
543 ctx
->olaps
[i
].start
= ctx
->args
[i
].ptr
;
544 ctx
->olaps
[i
].end
= ctx
->olaps
[i
].start
+ ctx
->args
[i
].length
;
545 ctx
->olaps
[i
].raix
= i
;
548 sort(ctx
->olaps
, ctx
->nbufs
, sizeof(*ctx
->olaps
), olaps_cmp
, NULL
);
550 for (i
= 0; i
< ctx
->nbufs
; ++i
) {
551 /* Falling inside previous range */
552 if (ctx
->olaps
[i
].start
< max_end
) {
553 ctx
->olaps
[i
].mstart
= max_end
;
554 ctx
->olaps
[i
].mend
= ctx
->olaps
[i
].end
;
555 ctx
->olaps
[i
].offset
= max_end
- ctx
->olaps
[i
].start
;
557 if (ctx
->olaps
[i
].end
> max_end
) {
558 max_end
= ctx
->olaps
[i
].end
;
560 ctx
->olaps
[i
].mend
= 0;
561 ctx
->olaps
[i
].mstart
= 0;
565 ctx
->olaps
[i
].mend
= ctx
->olaps
[i
].end
;
566 ctx
->olaps
[i
].mstart
= ctx
->olaps
[i
].start
;
567 ctx
->olaps
[i
].offset
= 0;
568 max_end
= ctx
->olaps
[i
].end
;
573 static struct fastrpc_invoke_ctx
*fastrpc_context_alloc(
574 struct fastrpc_user
*user
, u32 kernel
, u32 sc
,
575 struct fastrpc_invoke_args
*args
)
577 struct fastrpc_channel_ctx
*cctx
= user
->cctx
;
578 struct fastrpc_invoke_ctx
*ctx
= NULL
;
582 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
584 return ERR_PTR(-ENOMEM
);
586 INIT_LIST_HEAD(&ctx
->node
);
588 ctx
->nscalars
= REMOTE_SCALARS_LENGTH(sc
);
589 ctx
->nbufs
= REMOTE_SCALARS_INBUFS(sc
) +
590 REMOTE_SCALARS_OUTBUFS(sc
);
593 ctx
->maps
= kcalloc(ctx
->nscalars
,
594 sizeof(*ctx
->maps
), GFP_KERNEL
);
597 return ERR_PTR(-ENOMEM
);
599 ctx
->olaps
= kcalloc(ctx
->nscalars
,
600 sizeof(*ctx
->olaps
), GFP_KERNEL
);
604 return ERR_PTR(-ENOMEM
);
607 fastrpc_get_buff_overlaps(ctx
);
610 /* Released in fastrpc_context_put() */
611 fastrpc_channel_ctx_get(cctx
);
615 ctx
->pid
= current
->pid
;
616 ctx
->tgid
= user
->tgid
;
618 init_completion(&ctx
->work
);
619 INIT_WORK(&ctx
->put_work
, fastrpc_context_put_wq
);
621 spin_lock(&user
->lock
);
622 list_add_tail(&ctx
->node
, &user
->pending
);
623 spin_unlock(&user
->lock
);
625 spin_lock_irqsave(&cctx
->lock
, flags
);
626 ret
= idr_alloc_cyclic(&cctx
->ctx_idr
, ctx
, 1,
627 FASTRPC_CTX_MAX
, GFP_ATOMIC
);
629 spin_unlock_irqrestore(&cctx
->lock
, flags
);
632 ctx
->ctxid
= ret
<< 4;
633 spin_unlock_irqrestore(&cctx
->lock
, flags
);
635 kref_init(&ctx
->refcount
);
639 spin_lock(&user
->lock
);
640 list_del(&ctx
->node
);
641 spin_unlock(&user
->lock
);
642 fastrpc_channel_ctx_put(cctx
);
650 static struct sg_table
*
651 fastrpc_map_dma_buf(struct dma_buf_attachment
*attachment
,
652 enum dma_data_direction dir
)
654 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
655 struct sg_table
*table
;
660 ret
= dma_map_sgtable(attachment
->dev
, table
, dir
, 0);
662 table
= ERR_PTR(ret
);
666 static void fastrpc_unmap_dma_buf(struct dma_buf_attachment
*attach
,
667 struct sg_table
*table
,
668 enum dma_data_direction dir
)
670 dma_unmap_sgtable(attach
->dev
, table
, dir
, 0);
673 static void fastrpc_release(struct dma_buf
*dmabuf
)
675 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
677 fastrpc_buf_free(buffer
);
680 static int fastrpc_dma_buf_attach(struct dma_buf
*dmabuf
,
681 struct dma_buf_attachment
*attachment
)
683 struct fastrpc_dma_buf_attachment
*a
;
684 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
687 a
= kzalloc(sizeof(*a
), GFP_KERNEL
);
691 ret
= dma_get_sgtable(buffer
->dev
, &a
->sgt
, buffer
->virt
,
692 FASTRPC_PHYS(buffer
->phys
), buffer
->size
);
694 dev_err(buffer
->dev
, "failed to get scatterlist from DMA API\n");
699 a
->dev
= attachment
->dev
;
700 INIT_LIST_HEAD(&a
->node
);
701 attachment
->priv
= a
;
703 mutex_lock(&buffer
->lock
);
704 list_add(&a
->node
, &buffer
->attachments
);
705 mutex_unlock(&buffer
->lock
);
710 static void fastrpc_dma_buf_detatch(struct dma_buf
*dmabuf
,
711 struct dma_buf_attachment
*attachment
)
713 struct fastrpc_dma_buf_attachment
*a
= attachment
->priv
;
714 struct fastrpc_buf
*buffer
= dmabuf
->priv
;
716 mutex_lock(&buffer
->lock
);
718 mutex_unlock(&buffer
->lock
);
719 sg_free_table(&a
->sgt
);
723 static int fastrpc_vmap(struct dma_buf
*dmabuf
, struct iosys_map
*map
)
725 struct fastrpc_buf
*buf
= dmabuf
->priv
;
727 iosys_map_set_vaddr(map
, buf
->virt
);
732 static int fastrpc_mmap(struct dma_buf
*dmabuf
,
733 struct vm_area_struct
*vma
)
735 struct fastrpc_buf
*buf
= dmabuf
->priv
;
736 size_t size
= vma
->vm_end
- vma
->vm_start
;
738 dma_resv_assert_held(dmabuf
->resv
);
740 return dma_mmap_coherent(buf
->dev
, vma
, buf
->virt
,
741 FASTRPC_PHYS(buf
->phys
), size
);
744 static const struct dma_buf_ops fastrpc_dma_buf_ops
= {
745 .attach
= fastrpc_dma_buf_attach
,
746 .detach
= fastrpc_dma_buf_detatch
,
747 .map_dma_buf
= fastrpc_map_dma_buf
,
748 .unmap_dma_buf
= fastrpc_unmap_dma_buf
,
749 .mmap
= fastrpc_mmap
,
750 .vmap
= fastrpc_vmap
,
751 .release
= fastrpc_release
,
754 static int fastrpc_map_create(struct fastrpc_user
*fl
, int fd
,
755 u64 len
, u32 attr
, struct fastrpc_map
**ppmap
)
757 struct fastrpc_session_ctx
*sess
= fl
->sctx
;
758 struct fastrpc_map
*map
= NULL
;
759 struct sg_table
*table
;
762 if (!fastrpc_map_lookup(fl
, fd
, ppmap
, true))
765 map
= kzalloc(sizeof(*map
), GFP_KERNEL
);
769 INIT_LIST_HEAD(&map
->node
);
770 kref_init(&map
->refcount
);
774 map
->buf
= dma_buf_get(fd
);
775 if (IS_ERR(map
->buf
)) {
776 err
= PTR_ERR(map
->buf
);
780 map
->attach
= dma_buf_attach(map
->buf
, sess
->dev
);
781 if (IS_ERR(map
->attach
)) {
782 dev_err(sess
->dev
, "Failed to attach dmabuf\n");
783 err
= PTR_ERR(map
->attach
);
787 table
= dma_buf_map_attachment_unlocked(map
->attach
, DMA_BIDIRECTIONAL
);
789 err
= PTR_ERR(table
);
794 if (attr
& FASTRPC_ATTR_SECUREMAP
) {
795 map
->phys
= sg_phys(map
->table
->sgl
);
797 map
->phys
= sg_dma_address(map
->table
->sgl
);
798 map
->phys
+= ((u64
)fl
->sctx
->sid
<< 32);
801 map
->va
= sg_virt(map
->table
->sgl
);
804 if (attr
& FASTRPC_ATTR_SECUREMAP
) {
806 * If subsystem VMIDs are defined in DTSI, then do
807 * hyp_assign from HLOS to those VM(s)
809 u64 src_perms
= BIT(QCOM_SCM_VMID_HLOS
);
810 struct qcom_scm_vmperm dst_perms
[2] = {0};
812 dst_perms
[0].vmid
= QCOM_SCM_VMID_HLOS
;
813 dst_perms
[0].perm
= QCOM_SCM_PERM_RW
;
814 dst_perms
[1].vmid
= fl
->cctx
->vmperms
[0].vmid
;
815 dst_perms
[1].perm
= QCOM_SCM_PERM_RWX
;
817 err
= qcom_scm_assign_mem(map
->phys
, (u64
)map
->size
, &src_perms
, dst_perms
, 2);
819 dev_err(sess
->dev
, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
820 map
->phys
, map
->size
, err
);
824 spin_lock(&fl
->lock
);
825 list_add_tail(&map
->node
, &fl
->maps
);
826 spin_unlock(&fl
->lock
);
832 dma_buf_detach(map
->buf
, map
->attach
);
834 dma_buf_put(map
->buf
);
836 fastrpc_map_put(map
);
842 * Fastrpc payload buffer with metadata looks like:
844 * >>>>>> START of METADATA <<<<<<<<<
845 * +---------------------------------+
847 * | type:(union fastrpc_remote_arg)|
849 * +---------------------------------+
850 * | Invoke Buffer list |
851 * | type:(struct fastrpc_invoke_buf)|
853 * +---------------------------------+
855 * | type:(struct fastrpc_phy_page) |
857 * +---------------------------------+
859 * |(can be specific to SoC/Firmware)|
860 * +---------------------------------+
861 * >>>>>>>> END of METADATA <<<<<<<<<
862 * +---------------------------------+
865 * +---------------------------------+
868 static int fastrpc_get_meta_size(struct fastrpc_invoke_ctx
*ctx
)
872 size
= (sizeof(struct fastrpc_remote_buf
) +
873 sizeof(struct fastrpc_invoke_buf
) +
874 sizeof(struct fastrpc_phy_page
)) * ctx
->nscalars
+
875 sizeof(u64
) * FASTRPC_MAX_FDLIST
+
876 sizeof(u32
) * FASTRPC_MAX_CRCLIST
;
881 static u64
fastrpc_get_payload_size(struct fastrpc_invoke_ctx
*ctx
, int metalen
)
886 size
= ALIGN(metalen
, FASTRPC_ALIGN
);
887 for (oix
= 0; oix
< ctx
->nbufs
; oix
++) {
888 int i
= ctx
->olaps
[oix
].raix
;
890 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1) {
892 if (ctx
->olaps
[oix
].offset
== 0)
893 size
= ALIGN(size
, FASTRPC_ALIGN
);
895 size
+= (ctx
->olaps
[oix
].mend
- ctx
->olaps
[oix
].mstart
);
902 static int fastrpc_create_maps(struct fastrpc_invoke_ctx
*ctx
)
904 struct device
*dev
= ctx
->fl
->sctx
->dev
;
907 for (i
= 0; i
< ctx
->nscalars
; ++i
) {
909 if (ctx
->args
[i
].fd
== 0 || ctx
->args
[i
].fd
== -1 ||
910 ctx
->args
[i
].length
== 0)
913 err
= fastrpc_map_create(ctx
->fl
, ctx
->args
[i
].fd
,
914 ctx
->args
[i
].length
, ctx
->args
[i
].attr
, &ctx
->maps
[i
]);
916 dev_err(dev
, "Error Creating map %d\n", err
);
924 static struct fastrpc_invoke_buf
*fastrpc_invoke_buf_start(union fastrpc_remote_arg
*pra
, int len
)
926 return (struct fastrpc_invoke_buf
*)(&pra
[len
]);
929 static struct fastrpc_phy_page
*fastrpc_phy_page_start(struct fastrpc_invoke_buf
*buf
, int len
)
931 return (struct fastrpc_phy_page
*)(&buf
[len
]);
934 static int fastrpc_get_args(u32 kernel
, struct fastrpc_invoke_ctx
*ctx
)
936 struct device
*dev
= ctx
->fl
->sctx
->dev
;
937 union fastrpc_remote_arg
*rpra
;
938 struct fastrpc_invoke_buf
*list
;
939 struct fastrpc_phy_page
*pages
;
940 int inbufs
, i
, oix
, err
= 0;
941 u64 len
, rlen
, pkt_size
;
942 u64 pg_start
, pg_end
;
946 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
947 metalen
= fastrpc_get_meta_size(ctx
);
948 pkt_size
= fastrpc_get_payload_size(ctx
, metalen
);
950 err
= fastrpc_create_maps(ctx
);
954 ctx
->msg_sz
= pkt_size
;
956 if (ctx
->fl
->sctx
->sid
)
957 err
= fastrpc_buf_alloc(ctx
->fl
, dev
, pkt_size
, &ctx
->buf
);
959 err
= fastrpc_remote_heap_alloc(ctx
->fl
, dev
, pkt_size
, &ctx
->buf
);
963 memset(ctx
->buf
->virt
, 0, pkt_size
);
964 rpra
= ctx
->buf
->virt
;
965 list
= fastrpc_invoke_buf_start(rpra
, ctx
->nscalars
);
966 pages
= fastrpc_phy_page_start(list
, ctx
->nscalars
);
967 args
= (uintptr_t)ctx
->buf
->virt
+ metalen
;
968 rlen
= pkt_size
- metalen
;
971 for (oix
= 0; oix
< ctx
->nbufs
; ++oix
) {
974 i
= ctx
->olaps
[oix
].raix
;
975 len
= ctx
->args
[i
].length
;
978 rpra
[i
].buf
.len
= len
;
979 list
[i
].num
= len
? 1 : 0;
986 struct vm_area_struct
*vma
= NULL
;
988 rpra
[i
].buf
.pv
= (u64
) ctx
->args
[i
].ptr
;
989 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
991 mmap_read_lock(current
->mm
);
992 vma
= find_vma(current
->mm
, ctx
->args
[i
].ptr
);
994 pages
[i
].addr
+= ctx
->args
[i
].ptr
-
996 mmap_read_unlock(current
->mm
);
998 pg_start
= (ctx
->args
[i
].ptr
& PAGE_MASK
) >> PAGE_SHIFT
;
999 pg_end
= ((ctx
->args
[i
].ptr
+ len
- 1) & PAGE_MASK
) >>
1001 pages
[i
].size
= (pg_end
- pg_start
+ 1) * PAGE_SIZE
;
1005 if (ctx
->olaps
[oix
].offset
== 0) {
1006 rlen
-= ALIGN(args
, FASTRPC_ALIGN
) - args
;
1007 args
= ALIGN(args
, FASTRPC_ALIGN
);
1010 mlen
= ctx
->olaps
[oix
].mend
- ctx
->olaps
[oix
].mstart
;
1015 rpra
[i
].buf
.pv
= args
- ctx
->olaps
[oix
].offset
;
1016 pages
[i
].addr
= ctx
->buf
->phys
-
1017 ctx
->olaps
[oix
].offset
+
1019 pages
[i
].addr
= pages
[i
].addr
& PAGE_MASK
;
1021 pg_start
= (args
& PAGE_MASK
) >> PAGE_SHIFT
;
1022 pg_end
= ((args
+ len
- 1) & PAGE_MASK
) >> PAGE_SHIFT
;
1023 pages
[i
].size
= (pg_end
- pg_start
+ 1) * PAGE_SIZE
;
1028 if (i
< inbufs
&& !ctx
->maps
[i
]) {
1029 void *dst
= (void *)(uintptr_t)rpra
[i
].buf
.pv
;
1030 void *src
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
1033 if (copy_from_user(dst
, (void __user
*)src
,
1039 memcpy(dst
, src
, len
);
1044 for (i
= ctx
->nbufs
; i
< ctx
->nscalars
; ++i
) {
1045 list
[i
].num
= ctx
->args
[i
].length
? 1 : 0;
1048 pages
[i
].addr
= ctx
->maps
[i
]->phys
;
1049 pages
[i
].size
= ctx
->maps
[i
]->size
;
1051 rpra
[i
].dma
.fd
= ctx
->args
[i
].fd
;
1052 rpra
[i
].dma
.len
= ctx
->args
[i
].length
;
1053 rpra
[i
].dma
.offset
= (u64
) ctx
->args
[i
].ptr
;
1058 dev_err(dev
, "Error: get invoke args failed:%d\n", err
);
1063 static int fastrpc_put_args(struct fastrpc_invoke_ctx
*ctx
,
1066 union fastrpc_remote_arg
*rpra
= ctx
->rpra
;
1067 struct fastrpc_user
*fl
= ctx
->fl
;
1068 struct fastrpc_map
*mmap
= NULL
;
1069 struct fastrpc_invoke_buf
*list
;
1070 struct fastrpc_phy_page
*pages
;
1072 int i
, inbufs
, outbufs
, handles
;
1074 inbufs
= REMOTE_SCALARS_INBUFS(ctx
->sc
);
1075 outbufs
= REMOTE_SCALARS_OUTBUFS(ctx
->sc
);
1076 handles
= REMOTE_SCALARS_INHANDLES(ctx
->sc
) + REMOTE_SCALARS_OUTHANDLES(ctx
->sc
);
1077 list
= fastrpc_invoke_buf_start(rpra
, ctx
->nscalars
);
1078 pages
= fastrpc_phy_page_start(list
, ctx
->nscalars
);
1079 fdlist
= (uint64_t *)(pages
+ inbufs
+ outbufs
+ handles
);
1081 for (i
= inbufs
; i
< ctx
->nbufs
; ++i
) {
1082 if (!ctx
->maps
[i
]) {
1083 void *src
= (void *)(uintptr_t)rpra
[i
].buf
.pv
;
1084 void *dst
= (void *)(uintptr_t)ctx
->args
[i
].ptr
;
1085 u64 len
= rpra
[i
].buf
.len
;
1088 if (copy_to_user((void __user
*)dst
, src
, len
))
1091 memcpy(dst
, src
, len
);
1096 /* Clean up fdlist which is updated by DSP */
1097 for (i
= 0; i
< FASTRPC_MAX_FDLIST
; i
++) {
1100 if (!fastrpc_map_lookup(fl
, (int)fdlist
[i
], &mmap
, false))
1101 fastrpc_map_put(mmap
);
1107 static int fastrpc_invoke_send(struct fastrpc_session_ctx
*sctx
,
1108 struct fastrpc_invoke_ctx
*ctx
,
1109 u32 kernel
, uint32_t handle
)
1111 struct fastrpc_channel_ctx
*cctx
;
1112 struct fastrpc_user
*fl
= ctx
->fl
;
1113 struct fastrpc_msg
*msg
= &ctx
->msg
;
1117 msg
->pid
= fl
->tgid
;
1118 msg
->tid
= current
->pid
;
1123 msg
->ctx
= ctx
->ctxid
| fl
->pd
;
1124 msg
->handle
= handle
;
1126 msg
->addr
= ctx
->buf
? ctx
->buf
->phys
: 0;
1127 msg
->size
= roundup(ctx
->msg_sz
, PAGE_SIZE
);
1128 fastrpc_context_get(ctx
);
1130 ret
= rpmsg_send(cctx
->rpdev
->ept
, (void *)msg
, sizeof(*msg
));
1133 fastrpc_context_put(ctx
);
1139 static int fastrpc_internal_invoke(struct fastrpc_user
*fl
, u32 kernel
,
1141 struct fastrpc_invoke_args
*args
)
1143 struct fastrpc_invoke_ctx
*ctx
= NULL
;
1144 struct fastrpc_buf
*buf
, *b
;
1151 if (!fl
->cctx
->rpdev
)
1154 if (handle
== FASTRPC_INIT_HANDLE
&& !kernel
) {
1155 dev_warn_ratelimited(fl
->sctx
->dev
, "user app trying to send a kernel RPC message (%d)\n", handle
);
1159 ctx
= fastrpc_context_alloc(fl
, kernel
, sc
, args
);
1161 return PTR_ERR(ctx
);
1163 err
= fastrpc_get_args(kernel
, ctx
);
1167 /* make sure that all CPU memory writes are seen by DSP */
1169 /* Send invoke buffer to remote dsp */
1170 err
= fastrpc_invoke_send(fl
->sctx
, ctx
, kernel
, handle
);
1175 if (!wait_for_completion_timeout(&ctx
->work
, 10 * HZ
))
1178 err
= wait_for_completion_interruptible(&ctx
->work
);
1184 /* make sure that all memory writes by DSP are seen by CPU */
1186 /* populate all the output buffers with results */
1187 err
= fastrpc_put_args(ctx
, kernel
);
1191 /* Check the response from remote dsp */
1197 if (err
!= -ERESTARTSYS
&& err
!= -ETIMEDOUT
) {
1198 /* We are done with this compute context */
1199 spin_lock(&fl
->lock
);
1200 list_del(&ctx
->node
);
1201 spin_unlock(&fl
->lock
);
1202 fastrpc_context_put(ctx
);
1205 if (err
== -ERESTARTSYS
) {
1206 list_for_each_entry_safe(buf
, b
, &fl
->mmaps
, node
) {
1207 list_del(&buf
->node
);
1208 list_add_tail(&buf
->node
, &fl
->cctx
->invoke_interrupted_mmaps
);
1213 dev_dbg(fl
->sctx
->dev
, "Error: Invoke Failed %d\n", err
);
1218 static bool is_session_rejected(struct fastrpc_user
*fl
, bool unsigned_pd_request
)
1220 /* Check if the device node is non-secure and channel is secure*/
1221 if (!fl
->is_secure_dev
&& fl
->cctx
->secure
) {
1223 * Allow untrusted applications to offload only to Unsigned PD when
1224 * channel is configured as secure and block untrusted apps on channel
1225 * that does not support unsigned PD offload
1227 if (!fl
->cctx
->unsigned_support
|| !unsigned_pd_request
) {
1228 dev_err(&fl
->cctx
->rpdev
->dev
, "Error: Untrusted application trying to offload to signed PD\n");
1236 static int fastrpc_init_create_static_process(struct fastrpc_user
*fl
,
1239 struct fastrpc_init_create_static init
;
1240 struct fastrpc_invoke_args
*args
;
1241 struct fastrpc_phy_page pages
[1];
1244 bool scm_done
= false;
1252 args
= kcalloc(FASTRPC_CREATE_STATIC_PROCESS_NARGS
, sizeof(*args
), GFP_KERNEL
);
1256 if (copy_from_user(&init
, argp
, sizeof(init
))) {
1261 if (init
.namelen
> INIT_FILE_NAMELEN_MAX
) {
1266 name
= memdup_user(u64_to_user_ptr(init
.name
), init
.namelen
);
1268 err
= PTR_ERR(name
);
1272 if (!fl
->cctx
->remote_heap
) {
1273 err
= fastrpc_remote_heap_alloc(fl
, fl
->sctx
->dev
, init
.memlen
,
1274 &fl
->cctx
->remote_heap
);
1278 /* Map if we have any heap VMIDs associated with this ADSP Static Process. */
1279 if (fl
->cctx
->vmcount
) {
1280 u64 src_perms
= BIT(QCOM_SCM_VMID_HLOS
);
1282 err
= qcom_scm_assign_mem(fl
->cctx
->remote_heap
->phys
,
1283 (u64
)fl
->cctx
->remote_heap
->size
,
1285 fl
->cctx
->vmperms
, fl
->cctx
->vmcount
);
1287 dev_err(fl
->sctx
->dev
, "Failed to assign memory with phys 0x%llx size 0x%llx err %d\n",
1288 fl
->cctx
->remote_heap
->phys
, fl
->cctx
->remote_heap
->size
, err
);
1295 inbuf
.pgid
= fl
->tgid
;
1296 inbuf
.namelen
= init
.namelen
;
1300 args
[0].ptr
= (u64
)(uintptr_t)&inbuf
;
1301 args
[0].length
= sizeof(inbuf
);
1304 args
[1].ptr
= (u64
)(uintptr_t)name
;
1305 args
[1].length
= inbuf
.namelen
;
1308 pages
[0].addr
= fl
->cctx
->remote_heap
->phys
;
1309 pages
[0].size
= fl
->cctx
->remote_heap
->size
;
1311 args
[2].ptr
= (u64
)(uintptr_t) pages
;
1312 args
[2].length
= sizeof(*pages
);
1315 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_STATIC
, 3, 0);
1317 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1327 if (fl
->cctx
->vmcount
&& scm_done
) {
1329 struct qcom_scm_vmperm dst_perms
;
1332 for (i
= 0; i
< fl
->cctx
->vmcount
; i
++)
1333 src_perms
|= BIT(fl
->cctx
->vmperms
[i
].vmid
);
1335 dst_perms
.vmid
= QCOM_SCM_VMID_HLOS
;
1336 dst_perms
.perm
= QCOM_SCM_PERM_RWX
;
1337 err
= qcom_scm_assign_mem(fl
->cctx
->remote_heap
->phys
,
1338 (u64
)fl
->cctx
->remote_heap
->size
,
1339 &src_perms
, &dst_perms
, 1);
1341 dev_err(fl
->sctx
->dev
, "Failed to assign memory phys 0x%llx size 0x%llx err %d\n",
1342 fl
->cctx
->remote_heap
->phys
, fl
->cctx
->remote_heap
->size
, err
);
1345 fastrpc_buf_free(fl
->cctx
->remote_heap
);
1354 static int fastrpc_init_create_process(struct fastrpc_user
*fl
,
1357 struct fastrpc_init_create init
;
1358 struct fastrpc_invoke_args
*args
;
1359 struct fastrpc_phy_page pages
[1];
1360 struct fastrpc_map
*map
= NULL
;
1361 struct fastrpc_buf
*imem
= NULL
;
1373 bool unsigned_module
= false;
1375 args
= kcalloc(FASTRPC_CREATE_PROCESS_NARGS
, sizeof(*args
), GFP_KERNEL
);
1379 if (copy_from_user(&init
, argp
, sizeof(init
))) {
1384 if (init
.attrs
& FASTRPC_MODE_UNSIGNED_MODULE
)
1385 unsigned_module
= true;
1387 if (is_session_rejected(fl
, unsigned_module
)) {
1388 err
= -ECONNREFUSED
;
1392 if (init
.filelen
> INIT_FILELEN_MAX
) {
1397 inbuf
.pgid
= fl
->tgid
;
1398 inbuf
.namelen
= strlen(current
->comm
) + 1;
1399 inbuf
.filelen
= init
.filelen
;
1401 inbuf
.attrs
= init
.attrs
;
1402 inbuf
.siglen
= init
.siglen
;
1405 if (init
.filelen
&& init
.filefd
) {
1406 err
= fastrpc_map_create(fl
, init
.filefd
, init
.filelen
, 0, &map
);
1411 memlen
= ALIGN(max(INIT_FILELEN_MAX
, (int)init
.filelen
* 4),
1413 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, memlen
,
1418 fl
->init_mem
= imem
;
1419 args
[0].ptr
= (u64
)(uintptr_t)&inbuf
;
1420 args
[0].length
= sizeof(inbuf
);
1423 args
[1].ptr
= (u64
)(uintptr_t)current
->comm
;
1424 args
[1].length
= inbuf
.namelen
;
1427 args
[2].ptr
= (u64
) init
.file
;
1428 args
[2].length
= inbuf
.filelen
;
1429 args
[2].fd
= init
.filefd
;
1431 pages
[0].addr
= imem
->phys
;
1432 pages
[0].size
= imem
->size
;
1434 args
[3].ptr
= (u64
)(uintptr_t) pages
;
1435 args
[3].length
= 1 * sizeof(*pages
);
1438 args
[4].ptr
= (u64
)(uintptr_t)&inbuf
.attrs
;
1439 args
[4].length
= sizeof(inbuf
.attrs
);
1442 args
[5].ptr
= (u64
)(uintptr_t) &inbuf
.siglen
;
1443 args
[5].length
= sizeof(inbuf
.siglen
);
1446 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE
, 4, 0);
1448 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_CREATE_ATTR
, 4, 0);
1450 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1460 fl
->init_mem
= NULL
;
1461 fastrpc_buf_free(imem
);
1463 fastrpc_map_put(map
);
1470 static struct fastrpc_session_ctx
*fastrpc_session_alloc(
1471 struct fastrpc_channel_ctx
*cctx
)
1473 struct fastrpc_session_ctx
*session
= NULL
;
1474 unsigned long flags
;
1477 spin_lock_irqsave(&cctx
->lock
, flags
);
1478 for (i
= 0; i
< cctx
->sesscount
; i
++) {
1479 if (!cctx
->session
[i
].used
&& cctx
->session
[i
].valid
) {
1480 cctx
->session
[i
].used
= true;
1481 session
= &cctx
->session
[i
];
1485 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1490 static void fastrpc_session_free(struct fastrpc_channel_ctx
*cctx
,
1491 struct fastrpc_session_ctx
*session
)
1493 unsigned long flags
;
1495 spin_lock_irqsave(&cctx
->lock
, flags
);
1496 session
->used
= false;
1497 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1500 static int fastrpc_release_current_dsp_process(struct fastrpc_user
*fl
)
1502 struct fastrpc_invoke_args args
[1];
1507 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
1508 args
[0].length
= sizeof(tgid
);
1510 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE
, 1, 0);
1512 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1516 static int fastrpc_device_release(struct inode
*inode
, struct file
*file
)
1518 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
1519 struct fastrpc_channel_ctx
*cctx
= fl
->cctx
;
1520 struct fastrpc_invoke_ctx
*ctx
, *n
;
1521 struct fastrpc_map
*map
, *m
;
1522 struct fastrpc_buf
*buf
, *b
;
1523 unsigned long flags
;
1525 fastrpc_release_current_dsp_process(fl
);
1527 spin_lock_irqsave(&cctx
->lock
, flags
);
1528 list_del(&fl
->user
);
1529 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1532 fastrpc_buf_free(fl
->init_mem
);
1534 list_for_each_entry_safe(ctx
, n
, &fl
->pending
, node
) {
1535 list_del(&ctx
->node
);
1536 fastrpc_context_put(ctx
);
1539 list_for_each_entry_safe(map
, m
, &fl
->maps
, node
)
1540 fastrpc_map_put(map
);
1542 list_for_each_entry_safe(buf
, b
, &fl
->mmaps
, node
) {
1543 list_del(&buf
->node
);
1544 fastrpc_buf_free(buf
);
1547 fastrpc_session_free(cctx
, fl
->sctx
);
1548 fastrpc_channel_ctx_put(cctx
);
1550 mutex_destroy(&fl
->mutex
);
1552 file
->private_data
= NULL
;
1557 static int fastrpc_device_open(struct inode
*inode
, struct file
*filp
)
1559 struct fastrpc_channel_ctx
*cctx
;
1560 struct fastrpc_device
*fdevice
;
1561 struct fastrpc_user
*fl
= NULL
;
1562 unsigned long flags
;
1564 fdevice
= miscdev_to_fdevice(filp
->private_data
);
1565 cctx
= fdevice
->cctx
;
1567 fl
= kzalloc(sizeof(*fl
), GFP_KERNEL
);
1571 /* Released in fastrpc_device_release() */
1572 fastrpc_channel_ctx_get(cctx
);
1574 filp
->private_data
= fl
;
1575 spin_lock_init(&fl
->lock
);
1576 mutex_init(&fl
->mutex
);
1577 INIT_LIST_HEAD(&fl
->pending
);
1578 INIT_LIST_HEAD(&fl
->maps
);
1579 INIT_LIST_HEAD(&fl
->mmaps
);
1580 INIT_LIST_HEAD(&fl
->user
);
1581 fl
->tgid
= current
->tgid
;
1583 fl
->is_secure_dev
= fdevice
->secure
;
1585 fl
->sctx
= fastrpc_session_alloc(cctx
);
1587 dev_err(&cctx
->rpdev
->dev
, "No session available\n");
1588 mutex_destroy(&fl
->mutex
);
1594 spin_lock_irqsave(&cctx
->lock
, flags
);
1595 list_add_tail(&fl
->user
, &cctx
->users
);
1596 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1601 static int fastrpc_dmabuf_alloc(struct fastrpc_user
*fl
, char __user
*argp
)
1603 struct fastrpc_alloc_dma_buf bp
;
1604 DEFINE_DMA_BUF_EXPORT_INFO(exp_info
);
1605 struct fastrpc_buf
*buf
= NULL
;
1608 if (copy_from_user(&bp
, argp
, sizeof(bp
)))
1611 err
= fastrpc_buf_alloc(fl
, fl
->sctx
->dev
, bp
.size
, &buf
);
1614 exp_info
.ops
= &fastrpc_dma_buf_ops
;
1615 exp_info
.size
= bp
.size
;
1616 exp_info
.flags
= O_RDWR
;
1617 exp_info
.priv
= buf
;
1618 buf
->dmabuf
= dma_buf_export(&exp_info
);
1619 if (IS_ERR(buf
->dmabuf
)) {
1620 err
= PTR_ERR(buf
->dmabuf
);
1621 fastrpc_buf_free(buf
);
1625 bp
.fd
= dma_buf_fd(buf
->dmabuf
, O_ACCMODE
);
1627 dma_buf_put(buf
->dmabuf
);
1631 if (copy_to_user(argp
, &bp
, sizeof(bp
))) {
1633 * The usercopy failed, but we can't do much about it, as
1634 * dma_buf_fd() already called fd_install() and made the
1635 * file descriptor accessible for the current process. It
1636 * might already be closed and dmabuf no longer valid when
1637 * we reach this point. Therefore "leak" the fd and rely on
1638 * the process exit path to do any required cleanup.
1646 static int fastrpc_init_attach(struct fastrpc_user
*fl
, int pd
)
1648 struct fastrpc_invoke_args args
[1];
1649 int tgid
= fl
->tgid
;
1652 args
[0].ptr
= (u64
)(uintptr_t) &tgid
;
1653 args
[0].length
= sizeof(tgid
);
1655 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH
, 1, 0);
1658 return fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
,
1662 static int fastrpc_invoke(struct fastrpc_user
*fl
, char __user
*argp
)
1664 struct fastrpc_invoke_args
*args
= NULL
;
1665 struct fastrpc_invoke inv
;
1669 if (copy_from_user(&inv
, argp
, sizeof(inv
)))
1672 /* nscalars is truncated here to max supported value */
1673 nscalars
= REMOTE_SCALARS_LENGTH(inv
.sc
);
1675 args
= kcalloc(nscalars
, sizeof(*args
), GFP_KERNEL
);
1679 if (copy_from_user(args
, (void __user
*)(uintptr_t)inv
.args
,
1680 nscalars
* sizeof(*args
))) {
1686 err
= fastrpc_internal_invoke(fl
, false, inv
.handle
, inv
.sc
, args
);
1692 static int fastrpc_get_info_from_dsp(struct fastrpc_user
*fl
, uint32_t *dsp_attr_buf
,
1693 uint32_t dsp_attr_buf_len
)
1695 struct fastrpc_invoke_args args
[2] = { 0 };
1698 * Capability filled in userspace. This carries the information
1699 * about the remoteproc support which is fetched from the remoteproc
1700 * sysfs node by userspace.
1702 dsp_attr_buf
[0] = 0;
1703 dsp_attr_buf_len
-= 1;
1705 args
[0].ptr
= (u64
)(uintptr_t)&dsp_attr_buf_len
;
1706 args
[0].length
= sizeof(dsp_attr_buf_len
);
1708 args
[1].ptr
= (u64
)(uintptr_t)&dsp_attr_buf
[1];
1709 args
[1].length
= dsp_attr_buf_len
* sizeof(u32
);
1712 return fastrpc_internal_invoke(fl
, true, FASTRPC_DSP_UTILITIES_HANDLE
,
1713 FASTRPC_SCALARS(0, 1, 1), args
);
1716 static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability
*cap
,
1717 struct fastrpc_user
*fl
)
1719 struct fastrpc_channel_ctx
*cctx
= fl
->cctx
;
1720 uint32_t attribute_id
= cap
->attribute_id
;
1721 uint32_t *dsp_attributes
;
1722 unsigned long flags
;
1723 uint32_t domain
= cap
->domain
;
1726 spin_lock_irqsave(&cctx
->lock
, flags
);
1727 /* check if we already have queried dsp for attributes */
1728 if (cctx
->valid_attributes
) {
1729 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1732 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1734 dsp_attributes
= kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN
, GFP_KERNEL
);
1735 if (!dsp_attributes
)
1738 err
= fastrpc_get_info_from_dsp(fl
, dsp_attributes
, FASTRPC_MAX_DSP_ATTRIBUTES
);
1739 if (err
== DSP_UNSUPPORTED_API
) {
1740 dev_info(&cctx
->rpdev
->dev
,
1741 "Warning: DSP capabilities not supported on domain: %d\n", domain
);
1742 kfree(dsp_attributes
);
1745 dev_err(&cctx
->rpdev
->dev
, "Error: dsp information is incorrect err: %d\n", err
);
1746 kfree(dsp_attributes
);
1750 spin_lock_irqsave(&cctx
->lock
, flags
);
1751 memcpy(cctx
->dsp_attributes
, dsp_attributes
, FASTRPC_MAX_DSP_ATTRIBUTES_LEN
);
1752 cctx
->valid_attributes
= true;
1753 spin_unlock_irqrestore(&cctx
->lock
, flags
);
1754 kfree(dsp_attributes
);
1756 cap
->capability
= cctx
->dsp_attributes
[attribute_id
];
1760 static int fastrpc_get_dsp_info(struct fastrpc_user
*fl
, char __user
*argp
)
1762 struct fastrpc_ioctl_capability cap
= {0};
1765 if (copy_from_user(&cap
, argp
, sizeof(cap
)))
1769 if (cap
.domain
>= FASTRPC_DEV_MAX
) {
1770 dev_err(&fl
->cctx
->rpdev
->dev
, "Error: Invalid domain id:%d, err:%d\n",
1775 /* Fastrpc Capablities does not support modem domain */
1776 if (cap
.domain
== MDSP_DOMAIN_ID
) {
1777 dev_err(&fl
->cctx
->rpdev
->dev
, "Error: modem not supported %d\n", err
);
1781 if (cap
.attribute_id
>= FASTRPC_MAX_DSP_ATTRIBUTES
) {
1782 dev_err(&fl
->cctx
->rpdev
->dev
, "Error: invalid attribute: %d, err: %d\n",
1783 cap
.attribute_id
, err
);
1787 err
= fastrpc_get_info_from_kernel(&cap
, fl
);
1791 if (copy_to_user(argp
, &cap
, sizeof(cap
)))
1797 static int fastrpc_req_munmap_impl(struct fastrpc_user
*fl
, struct fastrpc_buf
*buf
)
1799 struct fastrpc_invoke_args args
[1] = { [0] = { 0 } };
1800 struct fastrpc_munmap_req_msg req_msg
;
1801 struct device
*dev
= fl
->sctx
->dev
;
1805 req_msg
.pgid
= fl
->tgid
;
1806 req_msg
.size
= buf
->size
;
1807 req_msg
.vaddr
= buf
->raddr
;
1809 args
[0].ptr
= (u64
) (uintptr_t) &req_msg
;
1810 args
[0].length
= sizeof(req_msg
);
1812 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_MUNMAP
, 1, 0);
1813 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
, sc
,
1816 dev_dbg(dev
, "unmmap\tpt 0x%09lx OK\n", buf
->raddr
);
1817 spin_lock(&fl
->lock
);
1818 list_del(&buf
->node
);
1819 spin_unlock(&fl
->lock
);
1820 fastrpc_buf_free(buf
);
1822 dev_err(dev
, "unmmap\tpt 0x%09lx ERROR\n", buf
->raddr
);
1828 static int fastrpc_req_munmap(struct fastrpc_user
*fl
, char __user
*argp
)
1830 struct fastrpc_buf
*buf
= NULL
, *iter
, *b
;
1831 struct fastrpc_req_munmap req
;
1832 struct device
*dev
= fl
->sctx
->dev
;
1834 if (copy_from_user(&req
, argp
, sizeof(req
)))
1837 spin_lock(&fl
->lock
);
1838 list_for_each_entry_safe(iter
, b
, &fl
->mmaps
, node
) {
1839 if ((iter
->raddr
== req
.vaddrout
) && (iter
->size
== req
.size
)) {
1844 spin_unlock(&fl
->lock
);
1847 dev_err(dev
, "mmap\t\tpt 0x%09llx [len 0x%08llx] not in list\n",
1848 req
.vaddrout
, req
.size
);
1852 return fastrpc_req_munmap_impl(fl
, buf
);
1855 static int fastrpc_req_mmap(struct fastrpc_user
*fl
, char __user
*argp
)
1857 struct fastrpc_invoke_args args
[3] = { [0 ... 2] = { 0 } };
1858 struct fastrpc_buf
*buf
= NULL
;
1859 struct fastrpc_mmap_req_msg req_msg
;
1860 struct fastrpc_mmap_rsp_msg rsp_msg
;
1861 struct fastrpc_phy_page pages
;
1862 struct fastrpc_req_mmap req
;
1863 struct device
*dev
= fl
->sctx
->dev
;
1867 if (copy_from_user(&req
, argp
, sizeof(req
)))
1870 if (req
.flags
!= ADSP_MMAP_ADD_PAGES
&& req
.flags
!= ADSP_MMAP_REMOTE_HEAP_ADDR
) {
1871 dev_err(dev
, "flag not supported 0x%x\n", req
.flags
);
1877 dev_err(dev
, "adding user allocated pages is not supported\n");
1881 if (req
.flags
== ADSP_MMAP_REMOTE_HEAP_ADDR
)
1882 err
= fastrpc_remote_heap_alloc(fl
, dev
, req
.size
, &buf
);
1884 err
= fastrpc_buf_alloc(fl
, dev
, req
.size
, &buf
);
1887 dev_err(dev
, "failed to allocate buffer\n");
1891 req_msg
.pgid
= fl
->tgid
;
1892 req_msg
.flags
= req
.flags
;
1893 req_msg
.vaddr
= req
.vaddrin
;
1894 req_msg
.num
= sizeof(pages
);
1896 args
[0].ptr
= (u64
) (uintptr_t) &req_msg
;
1897 args
[0].length
= sizeof(req_msg
);
1899 pages
.addr
= buf
->phys
;
1900 pages
.size
= buf
->size
;
1902 args
[1].ptr
= (u64
) (uintptr_t) &pages
;
1903 args
[1].length
= sizeof(pages
);
1905 args
[2].ptr
= (u64
) (uintptr_t) &rsp_msg
;
1906 args
[2].length
= sizeof(rsp_msg
);
1908 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_MMAP
, 2, 1);
1909 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
, sc
,
1912 dev_err(dev
, "mmap error (len 0x%08llx)\n", buf
->size
);
1913 fastrpc_buf_free(buf
);
1917 /* update the buffer to be able to deallocate the memory on the DSP */
1918 buf
->raddr
= (uintptr_t) rsp_msg
.vaddr
;
1920 /* let the client know the address to use */
1921 req
.vaddrout
= rsp_msg
.vaddr
;
1923 /* Add memory to static PD pool, protection thru hypervisor */
1924 if (req
.flags
== ADSP_MMAP_REMOTE_HEAP_ADDR
&& fl
->cctx
->vmcount
) {
1925 u64 src_perms
= BIT(QCOM_SCM_VMID_HLOS
);
1927 err
= qcom_scm_assign_mem(buf
->phys
, (u64
)buf
->size
,
1928 &src_perms
, fl
->cctx
->vmperms
, fl
->cctx
->vmcount
);
1930 dev_err(fl
->sctx
->dev
, "Failed to assign memory phys 0x%llx size 0x%llx err %d",
1931 buf
->phys
, buf
->size
, err
);
1936 spin_lock(&fl
->lock
);
1937 list_add_tail(&buf
->node
, &fl
->mmaps
);
1938 spin_unlock(&fl
->lock
);
1940 if (copy_to_user((void __user
*)argp
, &req
, sizeof(req
))) {
1945 dev_dbg(dev
, "mmap\t\tpt 0x%09lx OK [len 0x%08llx]\n",
1946 buf
->raddr
, buf
->size
);
1951 fastrpc_req_munmap_impl(fl
, buf
);
1956 static int fastrpc_req_mem_unmap_impl(struct fastrpc_user
*fl
, struct fastrpc_mem_unmap
*req
)
1958 struct fastrpc_invoke_args args
[1] = { [0] = { 0 } };
1959 struct fastrpc_map
*map
= NULL
, *iter
, *m
;
1960 struct fastrpc_mem_unmap_req_msg req_msg
= { 0 };
1963 struct device
*dev
= fl
->sctx
->dev
;
1965 spin_lock(&fl
->lock
);
1966 list_for_each_entry_safe(iter
, m
, &fl
->maps
, node
) {
1967 if ((req
->fd
< 0 || iter
->fd
== req
->fd
) && (iter
->raddr
== req
->vaddr
)) {
1973 spin_unlock(&fl
->lock
);
1976 dev_err(dev
, "map not in list\n");
1980 req_msg
.pgid
= fl
->tgid
;
1981 req_msg
.len
= map
->len
;
1982 req_msg
.vaddrin
= map
->raddr
;
1983 req_msg
.fd
= map
->fd
;
1985 args
[0].ptr
= (u64
) (uintptr_t) &req_msg
;
1986 args
[0].length
= sizeof(req_msg
);
1988 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP
, 1, 0);
1989 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
, sc
,
1992 dev_err(dev
, "unmmap\tpt fd = %d, 0x%09llx error\n", map
->fd
, map
->raddr
);
1995 fastrpc_map_put(map
);
2000 static int fastrpc_req_mem_unmap(struct fastrpc_user
*fl
, char __user
*argp
)
2002 struct fastrpc_mem_unmap req
;
2004 if (copy_from_user(&req
, argp
, sizeof(req
)))
2007 return fastrpc_req_mem_unmap_impl(fl
, &req
);
2010 static int fastrpc_req_mem_map(struct fastrpc_user
*fl
, char __user
*argp
)
2012 struct fastrpc_invoke_args args
[4] = { [0 ... 3] = { 0 } };
2013 struct fastrpc_mem_map_req_msg req_msg
= { 0 };
2014 struct fastrpc_mmap_rsp_msg rsp_msg
= { 0 };
2015 struct fastrpc_mem_unmap req_unmap
= { 0 };
2016 struct fastrpc_phy_page pages
= { 0 };
2017 struct fastrpc_mem_map req
;
2018 struct device
*dev
= fl
->sctx
->dev
;
2019 struct fastrpc_map
*map
= NULL
;
2023 if (copy_from_user(&req
, argp
, sizeof(req
)))
2026 /* create SMMU mapping */
2027 err
= fastrpc_map_create(fl
, req
.fd
, req
.length
, 0, &map
);
2029 dev_err(dev
, "failed to map buffer, fd = %d\n", req
.fd
);
2033 req_msg
.pgid
= fl
->tgid
;
2034 req_msg
.fd
= req
.fd
;
2035 req_msg
.offset
= req
.offset
;
2036 req_msg
.vaddrin
= req
.vaddrin
;
2037 map
->va
= (void *) (uintptr_t) req
.vaddrin
;
2038 req_msg
.flags
= req
.flags
;
2039 req_msg
.num
= sizeof(pages
);
2040 req_msg
.data_len
= 0;
2042 args
[0].ptr
= (u64
) (uintptr_t) &req_msg
;
2043 args
[0].length
= sizeof(req_msg
);
2045 pages
.addr
= map
->phys
;
2046 pages
.size
= map
->size
;
2048 args
[1].ptr
= (u64
) (uintptr_t) &pages
;
2049 args
[1].length
= sizeof(pages
);
2051 args
[2].ptr
= (u64
) (uintptr_t) &pages
;
2054 args
[3].ptr
= (u64
) (uintptr_t) &rsp_msg
;
2055 args
[3].length
= sizeof(rsp_msg
);
2057 sc
= FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP
, 3, 1);
2058 err
= fastrpc_internal_invoke(fl
, true, FASTRPC_INIT_HANDLE
, sc
, &args
[0]);
2060 dev_err(dev
, "mem mmap error, fd %d, vaddr %llx, size %lld\n",
2061 req
.fd
, req
.vaddrin
, map
->size
);
2065 /* update the buffer to be able to deallocate the memory on the DSP */
2066 map
->raddr
= rsp_msg
.vaddr
;
2068 /* let the client know the address to use */
2069 req
.vaddrout
= rsp_msg
.vaddr
;
2071 if (copy_to_user((void __user
*)argp
, &req
, sizeof(req
))) {
2072 /* unmap the memory and release the buffer */
2073 req_unmap
.vaddr
= (uintptr_t) rsp_msg
.vaddr
;
2074 req_unmap
.length
= map
->size
;
2075 fastrpc_req_mem_unmap_impl(fl
, &req_unmap
);
2082 fastrpc_map_put(map
);
2087 static long fastrpc_device_ioctl(struct file
*file
, unsigned int cmd
,
2090 struct fastrpc_user
*fl
= (struct fastrpc_user
*)file
->private_data
;
2091 char __user
*argp
= (char __user
*)arg
;
2095 case FASTRPC_IOCTL_INVOKE
:
2096 err
= fastrpc_invoke(fl
, argp
);
2098 case FASTRPC_IOCTL_INIT_ATTACH
:
2099 err
= fastrpc_init_attach(fl
, ROOT_PD
);
2101 case FASTRPC_IOCTL_INIT_ATTACH_SNS
:
2102 err
= fastrpc_init_attach(fl
, SENSORS_PD
);
2104 case FASTRPC_IOCTL_INIT_CREATE_STATIC
:
2105 err
= fastrpc_init_create_static_process(fl
, argp
);
2107 case FASTRPC_IOCTL_INIT_CREATE
:
2108 err
= fastrpc_init_create_process(fl
, argp
);
2110 case FASTRPC_IOCTL_ALLOC_DMA_BUFF
:
2111 err
= fastrpc_dmabuf_alloc(fl
, argp
);
2113 case FASTRPC_IOCTL_MMAP
:
2114 err
= fastrpc_req_mmap(fl
, argp
);
2116 case FASTRPC_IOCTL_MUNMAP
:
2117 err
= fastrpc_req_munmap(fl
, argp
);
2119 case FASTRPC_IOCTL_MEM_MAP
:
2120 err
= fastrpc_req_mem_map(fl
, argp
);
2122 case FASTRPC_IOCTL_MEM_UNMAP
:
2123 err
= fastrpc_req_mem_unmap(fl
, argp
);
2125 case FASTRPC_IOCTL_GET_DSP_INFO
:
2126 err
= fastrpc_get_dsp_info(fl
, argp
);
2136 static const struct file_operations fastrpc_fops
= {
2137 .open
= fastrpc_device_open
,
2138 .release
= fastrpc_device_release
,
2139 .unlocked_ioctl
= fastrpc_device_ioctl
,
2140 .compat_ioctl
= fastrpc_device_ioctl
,
2143 static int fastrpc_cb_probe(struct platform_device
*pdev
)
2145 struct fastrpc_channel_ctx
*cctx
;
2146 struct fastrpc_session_ctx
*sess
;
2147 struct device
*dev
= &pdev
->dev
;
2148 int i
, sessions
= 0;
2149 unsigned long flags
;
2152 cctx
= dev_get_drvdata(dev
->parent
);
2156 of_property_read_u32(dev
->of_node
, "qcom,nsessions", &sessions
);
2158 spin_lock_irqsave(&cctx
->lock
, flags
);
2159 if (cctx
->sesscount
>= FASTRPC_MAX_SESSIONS
) {
2160 dev_err(&pdev
->dev
, "too many sessions\n");
2161 spin_unlock_irqrestore(&cctx
->lock
, flags
);
2164 sess
= &cctx
->session
[cctx
->sesscount
++];
2168 dev_set_drvdata(dev
, sess
);
2170 if (of_property_read_u32(dev
->of_node
, "reg", &sess
->sid
))
2171 dev_info(dev
, "FastRPC Session ID not specified in DT\n");
2174 struct fastrpc_session_ctx
*dup_sess
;
2176 for (i
= 1; i
< sessions
; i
++) {
2177 if (cctx
->sesscount
>= FASTRPC_MAX_SESSIONS
)
2179 dup_sess
= &cctx
->session
[cctx
->sesscount
++];
2180 memcpy(dup_sess
, sess
, sizeof(*dup_sess
));
2183 spin_unlock_irqrestore(&cctx
->lock
, flags
);
2184 rc
= dma_set_mask(dev
, DMA_BIT_MASK(32));
2186 dev_err(dev
, "32-bit DMA enable failed\n");
2193 static void fastrpc_cb_remove(struct platform_device
*pdev
)
2195 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(pdev
->dev
.parent
);
2196 struct fastrpc_session_ctx
*sess
= dev_get_drvdata(&pdev
->dev
);
2197 unsigned long flags
;
2200 spin_lock_irqsave(&cctx
->lock
, flags
);
2201 for (i
= 0; i
< FASTRPC_MAX_SESSIONS
; i
++) {
2202 if (cctx
->session
[i
].sid
== sess
->sid
) {
2203 cctx
->session
[i
].valid
= false;
2207 spin_unlock_irqrestore(&cctx
->lock
, flags
);
2210 static const struct of_device_id fastrpc_match_table
[] = {
2211 { .compatible
= "qcom,fastrpc-compute-cb", },
2215 static struct platform_driver fastrpc_cb_driver
= {
2216 .probe
= fastrpc_cb_probe
,
2217 .remove_new
= fastrpc_cb_remove
,
2219 .name
= "qcom,fastrpc-cb",
2220 .of_match_table
= fastrpc_match_table
,
2221 .suppress_bind_attrs
= true,
2225 static int fastrpc_device_register(struct device
*dev
, struct fastrpc_channel_ctx
*cctx
,
2226 bool is_secured
, const char *domain
)
2228 struct fastrpc_device
*fdev
;
2231 fdev
= devm_kzalloc(dev
, sizeof(*fdev
), GFP_KERNEL
);
2235 fdev
->secure
= is_secured
;
2237 fdev
->miscdev
.minor
= MISC_DYNAMIC_MINOR
;
2238 fdev
->miscdev
.fops
= &fastrpc_fops
;
2239 fdev
->miscdev
.name
= devm_kasprintf(dev
, GFP_KERNEL
, "fastrpc-%s%s",
2240 domain
, is_secured
? "-secure" : "");
2241 if (!fdev
->miscdev
.name
)
2244 err
= misc_register(&fdev
->miscdev
);
2247 cctx
->secure_fdevice
= fdev
;
2249 cctx
->fdevice
= fdev
;
2255 static int fastrpc_rpmsg_probe(struct rpmsg_device
*rpdev
)
2257 struct device
*rdev
= &rpdev
->dev
;
2258 struct fastrpc_channel_ctx
*data
;
2259 int i
, err
, domain_id
= -1, vmcount
;
2262 struct device_node
*rmem_node
;
2263 struct reserved_mem
*rmem
;
2264 unsigned int vmids
[FASTRPC_MAX_VMIDS
];
2266 err
= of_property_read_string(rdev
->of_node
, "label", &domain
);
2268 dev_info(rdev
, "FastRPC Domain not specified in DT\n");
2272 for (i
= 0; i
<= CDSP_DOMAIN_ID
; i
++) {
2273 if (!strcmp(domains
[i
], domain
)) {
2279 if (domain_id
< 0) {
2280 dev_info(rdev
, "FastRPC Invalid Domain ID %d\n", domain_id
);
2284 if (of_reserved_mem_device_init_by_idx(rdev
, rdev
->of_node
, 0))
2285 dev_info(rdev
, "no reserved DMA memory for FASTRPC\n");
2287 vmcount
= of_property_read_variable_u32_array(rdev
->of_node
,
2288 "qcom,vmids", &vmids
[0], 0, FASTRPC_MAX_VMIDS
);
2291 else if (!qcom_scm_is_available())
2292 return -EPROBE_DEFER
;
2294 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
2299 data
->vmcount
= vmcount
;
2300 for (i
= 0; i
< data
->vmcount
; i
++) {
2301 data
->vmperms
[i
].vmid
= vmids
[i
];
2302 data
->vmperms
[i
].perm
= QCOM_SCM_PERM_RWX
;
2306 rmem_node
= of_parse_phandle(rdev
->of_node
, "memory-region", 0);
2307 if (domain_id
== SDSP_DOMAIN_ID
&& rmem_node
) {
2310 rmem
= of_reserved_mem_lookup(rmem_node
);
2316 src_perms
= BIT(QCOM_SCM_VMID_HLOS
);
2318 qcom_scm_assign_mem(rmem
->base
, rmem
->size
, &src_perms
,
2319 data
->vmperms
, data
->vmcount
);
2323 secure_dsp
= !(of_property_read_bool(rdev
->of_node
, "qcom,non-secure-domain"));
2324 data
->secure
= secure_dsp
;
2326 switch (domain_id
) {
2327 case ADSP_DOMAIN_ID
:
2328 case MDSP_DOMAIN_ID
:
2329 case SDSP_DOMAIN_ID
:
2330 /* Unsigned PD offloading is only supported on CDSP*/
2331 data
->unsigned_support
= false;
2332 err
= fastrpc_device_register(rdev
, data
, secure_dsp
, domains
[domain_id
]);
2336 case CDSP_DOMAIN_ID
:
2337 data
->unsigned_support
= true;
2338 /* Create both device nodes so that we can allow both Signed and Unsigned PD */
2339 err
= fastrpc_device_register(rdev
, data
, true, domains
[domain_id
]);
2343 err
= fastrpc_device_register(rdev
, data
, false, domains
[domain_id
]);
2352 kref_init(&data
->refcount
);
2354 dev_set_drvdata(&rpdev
->dev
, data
);
2355 rdev
->dma_mask
= &data
->dma_mask
;
2356 dma_set_mask_and_coherent(rdev
, DMA_BIT_MASK(32));
2357 INIT_LIST_HEAD(&data
->users
);
2358 INIT_LIST_HEAD(&data
->invoke_interrupted_mmaps
);
2359 spin_lock_init(&data
->lock
);
2360 idr_init(&data
->ctx_idr
);
2361 data
->domain_id
= domain_id
;
2362 data
->rpdev
= rpdev
;
2364 err
= of_platform_populate(rdev
->of_node
, NULL
, NULL
, rdev
);
2366 goto populate_error
;
2372 misc_deregister(&data
->fdevice
->miscdev
);
2373 if (data
->secure_fdevice
)
2374 misc_deregister(&data
->secure_fdevice
->miscdev
);
2381 static void fastrpc_notify_users(struct fastrpc_user
*user
)
2383 struct fastrpc_invoke_ctx
*ctx
;
2385 spin_lock(&user
->lock
);
2386 list_for_each_entry(ctx
, &user
->pending
, node
) {
2387 ctx
->retval
= -EPIPE
;
2388 complete(&ctx
->work
);
2390 spin_unlock(&user
->lock
);
2393 static void fastrpc_rpmsg_remove(struct rpmsg_device
*rpdev
)
2395 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
2396 struct fastrpc_buf
*buf
, *b
;
2397 struct fastrpc_user
*user
;
2398 unsigned long flags
;
2400 /* No invocations past this point */
2401 spin_lock_irqsave(&cctx
->lock
, flags
);
2403 list_for_each_entry(user
, &cctx
->users
, user
)
2404 fastrpc_notify_users(user
);
2405 spin_unlock_irqrestore(&cctx
->lock
, flags
);
2408 misc_deregister(&cctx
->fdevice
->miscdev
);
2410 if (cctx
->secure_fdevice
)
2411 misc_deregister(&cctx
->secure_fdevice
->miscdev
);
2413 list_for_each_entry_safe(buf
, b
, &cctx
->invoke_interrupted_mmaps
, node
)
2414 list_del(&buf
->node
);
2416 if (cctx
->remote_heap
)
2417 fastrpc_buf_free(cctx
->remote_heap
);
2419 of_platform_depopulate(&rpdev
->dev
);
2421 fastrpc_channel_ctx_put(cctx
);
2424 static int fastrpc_rpmsg_callback(struct rpmsg_device
*rpdev
, void *data
,
2425 int len
, void *priv
, u32 addr
)
2427 struct fastrpc_channel_ctx
*cctx
= dev_get_drvdata(&rpdev
->dev
);
2428 struct fastrpc_invoke_rsp
*rsp
= data
;
2429 struct fastrpc_invoke_ctx
*ctx
;
2430 unsigned long flags
;
2431 unsigned long ctxid
;
2433 if (len
< sizeof(*rsp
))
2436 ctxid
= ((rsp
->ctx
& FASTRPC_CTXID_MASK
) >> 4);
2438 spin_lock_irqsave(&cctx
->lock
, flags
);
2439 ctx
= idr_find(&cctx
->ctx_idr
, ctxid
);
2440 spin_unlock_irqrestore(&cctx
->lock
, flags
);
2443 dev_err(&rpdev
->dev
, "No context ID matches response\n");
2447 ctx
->retval
= rsp
->retval
;
2448 complete(&ctx
->work
);
2451 * The DMA buffer associated with the context cannot be freed in
2452 * interrupt context so schedule it through a worker thread to
2453 * avoid a kernel BUG.
2455 schedule_work(&ctx
->put_work
);
2460 static const struct of_device_id fastrpc_rpmsg_of_match
[] = {
2461 { .compatible
= "qcom,fastrpc" },
2464 MODULE_DEVICE_TABLE(of
, fastrpc_rpmsg_of_match
);
2466 static struct rpmsg_driver fastrpc_driver
= {
2467 .probe
= fastrpc_rpmsg_probe
,
2468 .remove
= fastrpc_rpmsg_remove
,
2469 .callback
= fastrpc_rpmsg_callback
,
2471 .name
= "qcom,fastrpc",
2472 .of_match_table
= fastrpc_rpmsg_of_match
,
2476 static int fastrpc_init(void)
2480 ret
= platform_driver_register(&fastrpc_cb_driver
);
2482 pr_err("fastrpc: failed to register cb driver\n");
2486 ret
= register_rpmsg_driver(&fastrpc_driver
);
2488 pr_err("fastrpc: failed to register rpmsg driver\n");
2489 platform_driver_unregister(&fastrpc_cb_driver
);
2495 module_init(fastrpc_init
);
2497 static void fastrpc_exit(void)
2499 platform_driver_unregister(&fastrpc_cb_driver
);
2500 unregister_rpmsg_driver(&fastrpc_driver
);
2502 module_exit(fastrpc_exit
);
2504 MODULE_DESCRIPTION("Qualcomm FastRPC");
2505 MODULE_LICENSE("GPL v2");
2506 MODULE_IMPORT_NS(DMA_BUF
);