1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * af_alg: User-space algorithm interface
5 * This file provides the user-space API for algorithms.
7 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
10 #include <linux/atomic.h>
11 #include <crypto/if_alg.h>
12 #include <linux/crypto.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/key.h>
16 #include <linux/key-type.h>
17 #include <linux/list.h>
18 #include <linux/module.h>
19 #include <linux/net.h>
20 #include <linux/rwsem.h>
21 #include <linux/sched.h>
22 #include <linux/sched/signal.h>
23 #include <linux/security.h>
24 #include <linux/string.h>
25 #include <keys/user-type.h>
26 #include <keys/trusted-type.h>
27 #include <keys/encrypted-type.h>
29 struct alg_type_list
{
30 const struct af_alg_type
*type
;
31 struct list_head list
;
34 static struct proto alg_proto
= {
37 .obj_size
= sizeof(struct alg_sock
),
40 static LIST_HEAD(alg_types
);
41 static DECLARE_RWSEM(alg_types_sem
);
43 static const struct af_alg_type
*alg_get_type(const char *name
)
45 const struct af_alg_type
*type
= ERR_PTR(-ENOENT
);
46 struct alg_type_list
*node
;
48 down_read(&alg_types_sem
);
49 list_for_each_entry(node
, &alg_types
, list
) {
50 if (strcmp(node
->type
->name
, name
))
53 if (try_module_get(node
->type
->owner
))
57 up_read(&alg_types_sem
);
62 int af_alg_register_type(const struct af_alg_type
*type
)
64 struct alg_type_list
*node
;
67 down_write(&alg_types_sem
);
68 list_for_each_entry(node
, &alg_types
, list
) {
69 if (!strcmp(node
->type
->name
, type
->name
))
73 node
= kmalloc(sizeof(*node
), GFP_KERNEL
);
78 type
->ops
->owner
= THIS_MODULE
;
80 type
->ops_nokey
->owner
= THIS_MODULE
;
82 list_add(&node
->list
, &alg_types
);
86 up_write(&alg_types_sem
);
90 EXPORT_SYMBOL_GPL(af_alg_register_type
);
92 int af_alg_unregister_type(const struct af_alg_type
*type
)
94 struct alg_type_list
*node
;
97 down_write(&alg_types_sem
);
98 list_for_each_entry(node
, &alg_types
, list
) {
99 if (strcmp(node
->type
->name
, type
->name
))
102 list_del(&node
->list
);
107 up_write(&alg_types_sem
);
111 EXPORT_SYMBOL_GPL(af_alg_unregister_type
);
113 static void alg_do_release(const struct af_alg_type
*type
, void *private)
118 type
->release(private);
119 module_put(type
->owner
);
122 int af_alg_release(struct socket
*sock
)
130 EXPORT_SYMBOL_GPL(af_alg_release
);
132 void af_alg_release_parent(struct sock
*sk
)
134 struct alg_sock
*ask
= alg_sk(sk
);
135 unsigned int nokey
= atomic_read(&ask
->nokey_refcnt
);
141 atomic_dec(&ask
->nokey_refcnt
);
143 if (atomic_dec_and_test(&ask
->refcnt
))
146 EXPORT_SYMBOL_GPL(af_alg_release_parent
);
148 static int alg_bind(struct socket
*sock
, struct sockaddr
*uaddr
, int addr_len
)
150 const u32 allowed
= CRYPTO_ALG_KERN_DRIVER_ONLY
;
151 struct sock
*sk
= sock
->sk
;
152 struct alg_sock
*ask
= alg_sk(sk
);
153 struct sockaddr_alg_new
*sa
= (void *)uaddr
;
154 const struct af_alg_type
*type
;
158 if (sock
->state
== SS_CONNECTED
)
161 BUILD_BUG_ON(offsetof(struct sockaddr_alg_new
, salg_name
) !=
162 offsetof(struct sockaddr_alg
, salg_name
));
163 BUILD_BUG_ON(offsetof(struct sockaddr_alg
, salg_name
) != sizeof(*sa
));
165 if (addr_len
< sizeof(*sa
) + 1)
168 /* If caller uses non-allowed flag, return error. */
169 if ((sa
->salg_feat
& ~allowed
) || (sa
->salg_mask
& ~allowed
))
172 sa
->salg_type
[sizeof(sa
->salg_type
) - 1] = 0;
173 sa
->salg_name
[addr_len
- sizeof(*sa
) - 1] = 0;
175 type
= alg_get_type(sa
->salg_type
);
176 if (PTR_ERR(type
) == -ENOENT
) {
177 request_module("algif-%s", sa
->salg_type
);
178 type
= alg_get_type(sa
->salg_type
);
182 return PTR_ERR(type
);
184 private = type
->bind(sa
->salg_name
, sa
->salg_feat
, sa
->salg_mask
);
185 if (IS_ERR(private)) {
186 module_put(type
->owner
);
187 return PTR_ERR(private);
192 if (atomic_read(&ask
->refcnt
))
195 swap(ask
->type
, type
);
196 swap(ask
->private, private);
203 alg_do_release(type
, private);
208 static int alg_setkey(struct sock
*sk
, sockptr_t ukey
, unsigned int keylen
)
210 struct alg_sock
*ask
= alg_sk(sk
);
211 const struct af_alg_type
*type
= ask
->type
;
215 key
= sock_kmalloc(sk
, keylen
, GFP_KERNEL
);
220 if (copy_from_sockptr(key
, ukey
, keylen
))
223 err
= type
->setkey(ask
->private, key
, keylen
);
226 sock_kzfree_s(sk
, key
, keylen
);
233 static const u8
*key_data_ptr_user(const struct key
*key
,
234 unsigned int *datalen
)
236 const struct user_key_payload
*ukp
;
238 ukp
= user_key_payload_locked(key
);
239 if (IS_ERR_OR_NULL(ukp
))
240 return ERR_PTR(-EKEYREVOKED
);
242 *datalen
= key
->datalen
;
247 static const u8
*key_data_ptr_encrypted(const struct key
*key
,
248 unsigned int *datalen
)
250 const struct encrypted_key_payload
*ekp
;
252 ekp
= dereference_key_locked(key
);
253 if (IS_ERR_OR_NULL(ekp
))
254 return ERR_PTR(-EKEYREVOKED
);
256 *datalen
= ekp
->decrypted_datalen
;
258 return ekp
->decrypted_data
;
261 static const u8
*key_data_ptr_trusted(const struct key
*key
,
262 unsigned int *datalen
)
264 const struct trusted_key_payload
*tkp
;
266 tkp
= dereference_key_locked(key
);
267 if (IS_ERR_OR_NULL(tkp
))
268 return ERR_PTR(-EKEYREVOKED
);
270 *datalen
= tkp
->key_len
;
275 static struct key
*lookup_key(key_serial_t serial
)
279 key_ref
= lookup_user_key(serial
, 0, KEY_NEED_SEARCH
);
281 return ERR_CAST(key_ref
);
283 return key_ref_to_ptr(key_ref
);
286 static int alg_setkey_by_key_serial(struct alg_sock
*ask
, sockptr_t optval
,
289 const struct af_alg_type
*type
= ask
->type
;
291 unsigned int key_datalen
;
297 if (optlen
!= sizeof(serial
))
300 if (copy_from_sockptr(&serial
, optval
, optlen
))
303 key
= lookup_key(serial
);
307 down_read(&key
->sem
);
309 ret
= ERR_PTR(-ENOPROTOOPT
);
310 if (!strcmp(key
->type
->name
, "user") ||
311 !strcmp(key
->type
->name
, "logon")) {
312 ret
= key_data_ptr_user(key
, &key_datalen
);
313 } else if (IS_REACHABLE(CONFIG_ENCRYPTED_KEYS
) &&
314 !strcmp(key
->type
->name
, "encrypted")) {
315 ret
= key_data_ptr_encrypted(key
, &key_datalen
);
316 } else if (IS_REACHABLE(CONFIG_TRUSTED_KEYS
) &&
317 !strcmp(key
->type
->name
, "trusted")) {
318 ret
= key_data_ptr_trusted(key
, &key_datalen
);
327 key_data
= sock_kmalloc(&ask
->sk
, key_datalen
, GFP_KERNEL
);
334 memcpy(key_data
, ret
, key_datalen
);
339 err
= type
->setkey(ask
->private, key_data
, key_datalen
);
341 sock_kzfree_s(&ask
->sk
, key_data
, key_datalen
);
348 static inline int alg_setkey_by_key_serial(struct alg_sock
*ask
,
357 static int alg_setsockopt(struct socket
*sock
, int level
, int optname
,
358 sockptr_t optval
, unsigned int optlen
)
360 struct sock
*sk
= sock
->sk
;
361 struct alg_sock
*ask
= alg_sk(sk
);
362 const struct af_alg_type
*type
;
366 if (atomic_read(&ask
->refcnt
) != atomic_read(&ask
->nokey_refcnt
))
372 if (level
!= SOL_ALG
|| !type
)
377 case ALG_SET_KEY_BY_KEY_SERIAL
:
378 if (sock
->state
== SS_CONNECTED
)
383 if (optname
== ALG_SET_KEY_BY_KEY_SERIAL
)
384 err
= alg_setkey_by_key_serial(ask
, optval
, optlen
);
386 err
= alg_setkey(sk
, optval
, optlen
);
388 case ALG_SET_AEAD_AUTHSIZE
:
389 if (sock
->state
== SS_CONNECTED
)
391 if (!type
->setauthsize
)
393 err
= type
->setauthsize(ask
->private, optlen
);
395 case ALG_SET_DRBG_ENTROPY
:
396 if (sock
->state
== SS_CONNECTED
)
398 if (!type
->setentropy
)
401 err
= type
->setentropy(ask
->private, optval
, optlen
);
410 int af_alg_accept(struct sock
*sk
, struct socket
*newsock
, bool kern
)
412 struct alg_sock
*ask
= alg_sk(sk
);
413 const struct af_alg_type
*type
;
425 sk2
= sk_alloc(sock_net(sk
), PF_ALG
, GFP_KERNEL
, &alg_proto
, kern
);
430 sock_init_data(newsock
, sk2
);
431 security_sock_graft(sk2
, newsock
);
432 security_sk_clone(sk
, sk2
);
435 * newsock->ops assigned here to allow type->accept call to override
436 * them when required.
438 newsock
->ops
= type
->ops
;
439 err
= type
->accept(ask
->private, sk2
);
441 nokey
= err
== -ENOKEY
;
442 if (nokey
&& type
->accept_nokey
)
443 err
= type
->accept_nokey(ask
->private, sk2
);
448 if (atomic_inc_return_relaxed(&ask
->refcnt
) == 1)
451 atomic_inc(&ask
->nokey_refcnt
);
452 atomic_set(&alg_sk(sk2
)->nokey_refcnt
, 1);
454 alg_sk(sk2
)->parent
= sk
;
455 alg_sk(sk2
)->type
= type
;
457 newsock
->state
= SS_CONNECTED
;
460 newsock
->ops
= type
->ops_nokey
;
469 EXPORT_SYMBOL_GPL(af_alg_accept
);
471 static int alg_accept(struct socket
*sock
, struct socket
*newsock
, int flags
,
474 return af_alg_accept(sock
->sk
, newsock
, kern
);
477 static const struct proto_ops alg_proto_ops
= {
479 .owner
= THIS_MODULE
,
481 .connect
= sock_no_connect
,
482 .socketpair
= sock_no_socketpair
,
483 .getname
= sock_no_getname
,
484 .ioctl
= sock_no_ioctl
,
485 .listen
= sock_no_listen
,
486 .shutdown
= sock_no_shutdown
,
487 .mmap
= sock_no_mmap
,
488 .sendmsg
= sock_no_sendmsg
,
489 .recvmsg
= sock_no_recvmsg
,
492 .release
= af_alg_release
,
493 .setsockopt
= alg_setsockopt
,
494 .accept
= alg_accept
,
497 static void alg_sock_destruct(struct sock
*sk
)
499 struct alg_sock
*ask
= alg_sk(sk
);
501 alg_do_release(ask
->type
, ask
->private);
504 static int alg_create(struct net
*net
, struct socket
*sock
, int protocol
,
510 if (sock
->type
!= SOCK_SEQPACKET
)
511 return -ESOCKTNOSUPPORT
;
513 return -EPROTONOSUPPORT
;
516 sk
= sk_alloc(net
, PF_ALG
, GFP_KERNEL
, &alg_proto
, kern
);
520 sock
->ops
= &alg_proto_ops
;
521 sock_init_data(sock
, sk
);
523 sk
->sk_destruct
= alg_sock_destruct
;
530 static const struct net_proto_family alg_family
= {
532 .create
= alg_create
,
533 .owner
= THIS_MODULE
,
536 static void af_alg_link_sg(struct af_alg_sgl
*sgl_prev
,
537 struct af_alg_sgl
*sgl_new
)
539 sg_unmark_end(sgl_prev
->sgt
.sgl
+ sgl_prev
->sgt
.nents
- 1);
540 sg_chain(sgl_prev
->sgt
.sgl
, sgl_prev
->sgt
.nents
+ 1, sgl_new
->sgt
.sgl
);
543 void af_alg_free_sg(struct af_alg_sgl
*sgl
)
549 for (i
= 0; i
< sgl
->sgt
.nents
; i
++)
550 unpin_user_page(sg_page(&sgl
->sgt
.sgl
[i
]));
551 if (sgl
->sgt
.sgl
!= sgl
->sgl
)
552 kvfree(sgl
->sgt
.sgl
);
556 EXPORT_SYMBOL_GPL(af_alg_free_sg
);
558 static int af_alg_cmsg_send(struct msghdr
*msg
, struct af_alg_control
*con
)
560 struct cmsghdr
*cmsg
;
562 for_each_cmsghdr(cmsg
, msg
) {
563 if (!CMSG_OK(msg
, cmsg
))
565 if (cmsg
->cmsg_level
!= SOL_ALG
)
568 switch (cmsg
->cmsg_type
) {
570 if (cmsg
->cmsg_len
< CMSG_LEN(sizeof(*con
->iv
)))
572 con
->iv
= (void *)CMSG_DATA(cmsg
);
573 if (cmsg
->cmsg_len
< CMSG_LEN(con
->iv
->ivlen
+
579 if (cmsg
->cmsg_len
< CMSG_LEN(sizeof(u32
)))
581 con
->op
= *(u32
*)CMSG_DATA(cmsg
);
584 case ALG_SET_AEAD_ASSOCLEN
:
585 if (cmsg
->cmsg_len
< CMSG_LEN(sizeof(u32
)))
587 con
->aead_assoclen
= *(u32
*)CMSG_DATA(cmsg
);
599 * af_alg_alloc_tsgl - allocate the TX SGL
601 * @sk: socket of connection to user space
602 * Return: 0 upon success, < 0 upon error
604 static int af_alg_alloc_tsgl(struct sock
*sk
)
606 struct alg_sock
*ask
= alg_sk(sk
);
607 struct af_alg_ctx
*ctx
= ask
->private;
608 struct af_alg_tsgl
*sgl
;
609 struct scatterlist
*sg
= NULL
;
611 sgl
= list_entry(ctx
->tsgl_list
.prev
, struct af_alg_tsgl
, list
);
612 if (!list_empty(&ctx
->tsgl_list
))
615 if (!sg
|| sgl
->cur
>= MAX_SGL_ENTS
) {
616 sgl
= sock_kmalloc(sk
,
617 struct_size(sgl
, sg
, (MAX_SGL_ENTS
+ 1)),
622 sg_init_table(sgl
->sg
, MAX_SGL_ENTS
+ 1);
626 sg_chain(sg
, MAX_SGL_ENTS
+ 1, sgl
->sg
);
628 list_add_tail(&sgl
->list
, &ctx
->tsgl_list
);
635 * af_alg_count_tsgl - Count number of TX SG entries
637 * The counting starts from the beginning of the SGL to @bytes. If
638 * an @offset is provided, the counting of the SG entries starts at the @offset.
640 * @sk: socket of connection to user space
641 * @bytes: Count the number of SG entries holding given number of bytes.
642 * @offset: Start the counting of SG entries from the given offset.
643 * Return: Number of TX SG entries found given the constraints
645 unsigned int af_alg_count_tsgl(struct sock
*sk
, size_t bytes
, size_t offset
)
647 const struct alg_sock
*ask
= alg_sk(sk
);
648 const struct af_alg_ctx
*ctx
= ask
->private;
649 const struct af_alg_tsgl
*sgl
;
651 unsigned int sgl_count
= 0;
656 list_for_each_entry(sgl
, &ctx
->tsgl_list
, list
) {
657 const struct scatterlist
*sg
= sgl
->sg
;
659 for (i
= 0; i
< sgl
->cur
; i
++) {
663 if (offset
>= sg
[i
].length
) {
664 offset
-= sg
[i
].length
;
665 bytes
-= sg
[i
].length
;
669 bytes_count
= sg
[i
].length
- offset
;
674 /* If we have seen requested number of bytes, stop */
675 if (bytes_count
>= bytes
)
678 bytes
-= bytes_count
;
684 EXPORT_SYMBOL_GPL(af_alg_count_tsgl
);
687 * af_alg_pull_tsgl - Release the specified buffers from TX SGL
689 * If @dst is non-null, reassign the pages to @dst. The caller must release
690 * the pages. If @dst_offset is given only reassign the pages to @dst starting
691 * at the @dst_offset (byte). The caller must ensure that @dst is large
692 * enough (e.g. by using af_alg_count_tsgl with the same offset).
694 * @sk: socket of connection to user space
695 * @used: Number of bytes to pull from TX SGL
696 * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The
697 * caller must release the buffers in dst.
698 * @dst_offset: Reassign the TX SGL from given offset. All buffers before
699 * reaching the offset is released.
701 void af_alg_pull_tsgl(struct sock
*sk
, size_t used
, struct scatterlist
*dst
,
704 struct alg_sock
*ask
= alg_sk(sk
);
705 struct af_alg_ctx
*ctx
= ask
->private;
706 struct af_alg_tsgl
*sgl
;
707 struct scatterlist
*sg
;
708 unsigned int i
, j
= 0;
710 while (!list_empty(&ctx
->tsgl_list
)) {
711 sgl
= list_first_entry(&ctx
->tsgl_list
, struct af_alg_tsgl
,
715 for (i
= 0; i
< sgl
->cur
; i
++) {
716 size_t plen
= min_t(size_t, used
, sg
[i
].length
);
717 struct page
*page
= sg_page(sg
+ i
);
723 * Assumption: caller created af_alg_count_tsgl(len)
727 if (dst_offset
>= plen
) {
728 /* discard page before offset */
731 /* reassign page to dst after offset */
733 sg_set_page(dst
+ j
, page
,
735 sg
[i
].offset
+ dst_offset
);
741 sg
[i
].length
-= plen
;
742 sg
[i
].offset
+= plen
;
751 sg_assign_page(sg
+ i
, NULL
);
754 list_del(&sgl
->list
);
755 sock_kfree_s(sk
, sgl
, struct_size(sgl
, sg
, MAX_SGL_ENTS
+ 1));
760 ctx
->init
= ctx
->more
;
762 EXPORT_SYMBOL_GPL(af_alg_pull_tsgl
);
765 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request
767 * @areq: Request holding the TX and RX SGL
769 static void af_alg_free_areq_sgls(struct af_alg_async_req
*areq
)
771 struct sock
*sk
= areq
->sk
;
772 struct alg_sock
*ask
= alg_sk(sk
);
773 struct af_alg_ctx
*ctx
= ask
->private;
774 struct af_alg_rsgl
*rsgl
, *tmp
;
775 struct scatterlist
*tsgl
;
776 struct scatterlist
*sg
;
779 list_for_each_entry_safe(rsgl
, tmp
, &areq
->rsgl_list
, list
) {
780 atomic_sub(rsgl
->sg_num_bytes
, &ctx
->rcvused
);
781 af_alg_free_sg(&rsgl
->sgl
);
782 list_del(&rsgl
->list
);
783 if (rsgl
!= &areq
->first_rsgl
)
784 sock_kfree_s(sk
, rsgl
, sizeof(*rsgl
));
789 for_each_sg(tsgl
, sg
, areq
->tsgl_entries
, i
) {
792 put_page(sg_page(sg
));
795 sock_kfree_s(sk
, tsgl
, areq
->tsgl_entries
* sizeof(*tsgl
));
800 * af_alg_wait_for_wmem - wait for availability of writable memory
802 * @sk: socket of connection to user space
803 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep
804 * Return: 0 when writable memory is available, < 0 upon error
806 static int af_alg_wait_for_wmem(struct sock
*sk
, unsigned int flags
)
808 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
809 int err
= -ERESTARTSYS
;
812 if (flags
& MSG_DONTWAIT
)
815 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
817 add_wait_queue(sk_sleep(sk
), &wait
);
819 if (signal_pending(current
))
821 timeout
= MAX_SCHEDULE_TIMEOUT
;
822 if (sk_wait_event(sk
, &timeout
, af_alg_writable(sk
), &wait
)) {
827 remove_wait_queue(sk_sleep(sk
), &wait
);
833 * af_alg_wmem_wakeup - wakeup caller when writable memory is available
835 * @sk: socket of connection to user space
837 void af_alg_wmem_wakeup(struct sock
*sk
)
839 struct socket_wq
*wq
;
841 if (!af_alg_writable(sk
))
845 wq
= rcu_dereference(sk
->sk_wq
);
846 if (skwq_has_sleeper(wq
))
847 wake_up_interruptible_sync_poll(&wq
->wait
, EPOLLIN
|
850 sk_wake_async(sk
, SOCK_WAKE_WAITD
, POLL_IN
);
853 EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup
);
856 * af_alg_wait_for_data - wait for availability of TX data
858 * @sk: socket of connection to user space
859 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep
860 * @min: Set to minimum request size if partial requests are allowed.
861 * Return: 0 when writable memory is available, < 0 upon error
863 int af_alg_wait_for_data(struct sock
*sk
, unsigned flags
, unsigned min
)
865 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
866 struct alg_sock
*ask
= alg_sk(sk
);
867 struct af_alg_ctx
*ctx
= ask
->private;
869 int err
= -ERESTARTSYS
;
871 if (flags
& MSG_DONTWAIT
)
874 sk_set_bit(SOCKWQ_ASYNC_WAITDATA
, sk
);
876 add_wait_queue(sk_sleep(sk
), &wait
);
878 if (signal_pending(current
))
880 timeout
= MAX_SCHEDULE_TIMEOUT
;
881 if (sk_wait_event(sk
, &timeout
,
882 ctx
->init
&& (!ctx
->more
||
883 (min
&& ctx
->used
>= min
)),
889 remove_wait_queue(sk_sleep(sk
), &wait
);
891 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA
, sk
);
895 EXPORT_SYMBOL_GPL(af_alg_wait_for_data
);
898 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel
900 * @sk: socket of connection to user space
902 static void af_alg_data_wakeup(struct sock
*sk
)
904 struct alg_sock
*ask
= alg_sk(sk
);
905 struct af_alg_ctx
*ctx
= ask
->private;
906 struct socket_wq
*wq
;
912 wq
= rcu_dereference(sk
->sk_wq
);
913 if (skwq_has_sleeper(wq
))
914 wake_up_interruptible_sync_poll(&wq
->wait
, EPOLLOUT
|
917 sk_wake_async(sk
, SOCK_WAKE_SPACE
, POLL_OUT
);
922 * af_alg_sendmsg - implementation of sendmsg system call handler
924 * The sendmsg system call handler obtains the user data and stores it
925 * in ctx->tsgl_list. This implies allocation of the required numbers of
926 * struct af_alg_tsgl.
928 * In addition, the ctx is filled with the information sent via CMSG.
930 * @sock: socket of connection to user space
931 * @msg: message from user space
932 * @size: size of message from user space
933 * @ivsize: the size of the IV for the cipher operation to verify that the
934 * user-space-provided IV has the right size
935 * Return: the number of copied data upon success, < 0 upon error
937 int af_alg_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t size
,
940 struct sock
*sk
= sock
->sk
;
941 struct alg_sock
*ask
= alg_sk(sk
);
942 struct af_alg_ctx
*ctx
= ask
->private;
943 struct af_alg_tsgl
*sgl
;
944 struct af_alg_control con
= {};
950 if (msg
->msg_controllen
) {
951 err
= af_alg_cmsg_send(msg
, &con
);
967 if (con
.iv
&& con
.iv
->ivlen
!= ivsize
)
972 if (ctx
->init
&& !ctx
->more
) {
979 "%s sent an empty control message without MSG_MORE.\n",
987 memcpy(ctx
->iv
, con
.iv
->iv
, ivsize
);
989 ctx
->aead_assoclen
= con
.aead_assoclen
;
993 struct scatterlist
*sg
;
997 /* use the existing memory in an allocated page */
998 if (ctx
->merge
&& !(msg
->msg_flags
& MSG_SPLICE_PAGES
)) {
999 sgl
= list_entry(ctx
->tsgl_list
.prev
,
1000 struct af_alg_tsgl
, list
);
1001 sg
= sgl
->sg
+ sgl
->cur
- 1;
1002 len
= min_t(size_t, len
,
1003 PAGE_SIZE
- sg
->offset
- sg
->length
);
1005 err
= memcpy_from_msg(page_address(sg_page(sg
)) +
1006 sg
->offset
+ sg
->length
,
1012 ctx
->merge
= (sg
->offset
+ sg
->length
) &
1021 if (!af_alg_writable(sk
)) {
1022 err
= af_alg_wait_for_wmem(sk
, msg
->msg_flags
);
1027 /* allocate a new page */
1028 len
= min_t(unsigned long, len
, af_alg_sndbuf(sk
));
1030 err
= af_alg_alloc_tsgl(sk
);
1034 sgl
= list_entry(ctx
->tsgl_list
.prev
, struct af_alg_tsgl
,
1038 sg_unmark_end(sg
+ sgl
->cur
- 1);
1040 if (msg
->msg_flags
& MSG_SPLICE_PAGES
) {
1041 struct sg_table sgtable
= {
1044 .orig_nents
= sgl
->cur
,
1047 plen
= extract_iter_to_sg(&msg
->msg_iter
, len
, &sgtable
,
1048 MAX_SGL_ENTS
- sgl
->cur
, 0);
1054 for (; sgl
->cur
< sgtable
.nents
; sgl
->cur
++)
1055 get_page(sg_page(&sg
[sgl
->cur
]));
1064 unsigned int i
= sgl
->cur
;
1066 plen
= min_t(size_t, len
, PAGE_SIZE
);
1068 pg
= alloc_page(GFP_KERNEL
);
1074 sg_assign_page(sg
+ i
, pg
);
1076 err
= memcpy_from_msg(
1077 page_address(sg_page(sg
+ i
)),
1080 __free_page(sg_page(sg
+ i
));
1081 sg_assign_page(sg
+ i
, NULL
);
1085 sg
[i
].length
= plen
;
1091 } while (len
&& sgl
->cur
< MAX_SGL_ENTS
);
1093 ctx
->merge
= plen
& (PAGE_SIZE
- 1);
1097 sg_mark_end(sg
+ sgl
->cur
- 1);
1102 ctx
->more
= msg
->msg_flags
& MSG_MORE
;
1105 af_alg_data_wakeup(sk
);
1108 return copied
?: err
;
1110 EXPORT_SYMBOL_GPL(af_alg_sendmsg
);
1113 * af_alg_free_resources - release resources required for crypto request
1114 * @areq: Request holding the TX and RX SGL
1116 void af_alg_free_resources(struct af_alg_async_req
*areq
)
1118 struct sock
*sk
= areq
->sk
;
1120 af_alg_free_areq_sgls(areq
);
1121 sock_kfree_s(sk
, areq
, areq
->areqlen
);
1123 EXPORT_SYMBOL_GPL(af_alg_free_resources
);
1126 * af_alg_async_cb - AIO callback handler
1127 * @data: async request completion data
1128 * @err: if non-zero, error result to be returned via ki_complete();
1129 * otherwise return the AIO output length via ki_complete().
1131 * This handler cleans up the struct af_alg_async_req upon completion of the
1134 * The number of bytes to be generated with the AIO operation must be set
1135 * in areq->outlen before the AIO callback handler is invoked.
1137 void af_alg_async_cb(void *data
, int err
)
1139 struct af_alg_async_req
*areq
= data
;
1140 struct sock
*sk
= areq
->sk
;
1141 struct kiocb
*iocb
= areq
->iocb
;
1142 unsigned int resultlen
;
1144 /* Buffer size written by crypto operation. */
1145 resultlen
= areq
->outlen
;
1147 af_alg_free_resources(areq
);
1150 iocb
->ki_complete(iocb
, err
? err
: (int)resultlen
);
1152 EXPORT_SYMBOL_GPL(af_alg_async_cb
);
1155 * af_alg_poll - poll system call handler
1156 * @file: file pointer
1157 * @sock: socket to poll
1160 __poll_t
af_alg_poll(struct file
*file
, struct socket
*sock
,
1163 struct sock
*sk
= sock
->sk
;
1164 struct alg_sock
*ask
= alg_sk(sk
);
1165 struct af_alg_ctx
*ctx
= ask
->private;
1168 sock_poll_wait(file
, sock
, wait
);
1171 if (!ctx
->more
|| ctx
->used
)
1172 mask
|= EPOLLIN
| EPOLLRDNORM
;
1174 if (af_alg_writable(sk
))
1175 mask
|= EPOLLOUT
| EPOLLWRNORM
| EPOLLWRBAND
;
1179 EXPORT_SYMBOL_GPL(af_alg_poll
);
1182 * af_alg_alloc_areq - allocate struct af_alg_async_req
1184 * @sk: socket of connection to user space
1185 * @areqlen: size of struct af_alg_async_req + crypto_*_reqsize
1186 * Return: allocated data structure or ERR_PTR upon error
1188 struct af_alg_async_req
*af_alg_alloc_areq(struct sock
*sk
,
1189 unsigned int areqlen
)
1191 struct af_alg_async_req
*areq
= sock_kmalloc(sk
, areqlen
, GFP_KERNEL
);
1193 if (unlikely(!areq
))
1194 return ERR_PTR(-ENOMEM
);
1196 areq
->areqlen
= areqlen
;
1198 areq
->first_rsgl
.sgl
.sgt
.sgl
= areq
->first_rsgl
.sgl
.sgl
;
1199 areq
->last_rsgl
= NULL
;
1200 INIT_LIST_HEAD(&areq
->rsgl_list
);
1202 areq
->tsgl_entries
= 0;
1206 EXPORT_SYMBOL_GPL(af_alg_alloc_areq
);
1209 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto
1212 * @sk: socket of connection to user space
1213 * @msg: user space message
1214 * @flags: flags used to invoke recvmsg with
1215 * @areq: instance of the cryptographic request that will hold the RX SGL
1216 * @maxsize: maximum number of bytes to be pulled from user space
1217 * @outlen: number of bytes in the RX SGL
1218 * Return: 0 on success, < 0 upon error
1220 int af_alg_get_rsgl(struct sock
*sk
, struct msghdr
*msg
, int flags
,
1221 struct af_alg_async_req
*areq
, size_t maxsize
,
1224 struct alg_sock
*ask
= alg_sk(sk
);
1225 struct af_alg_ctx
*ctx
= ask
->private;
1228 while (maxsize
> len
&& msg_data_left(msg
)) {
1229 struct af_alg_rsgl
*rsgl
;
1233 /* limit the amount of readable buffers */
1234 if (!af_alg_readable(sk
))
1237 seglen
= min_t(size_t, (maxsize
- len
),
1238 msg_data_left(msg
));
1240 if (list_empty(&areq
->rsgl_list
)) {
1241 rsgl
= &areq
->first_rsgl
;
1243 rsgl
= sock_kmalloc(sk
, sizeof(*rsgl
), GFP_KERNEL
);
1244 if (unlikely(!rsgl
))
1248 rsgl
->sgl
.need_unpin
=
1249 iov_iter_extract_will_pin(&msg
->msg_iter
);
1250 rsgl
->sgl
.sgt
.sgl
= rsgl
->sgl
.sgl
;
1251 rsgl
->sgl
.sgt
.nents
= 0;
1252 rsgl
->sgl
.sgt
.orig_nents
= 0;
1253 list_add_tail(&rsgl
->list
, &areq
->rsgl_list
);
1255 sg_init_table(rsgl
->sgl
.sgt
.sgl
, ALG_MAX_PAGES
);
1256 err
= extract_iter_to_sg(&msg
->msg_iter
, seglen
, &rsgl
->sgl
.sgt
,
1259 rsgl
->sg_num_bytes
= 0;
1263 sg_mark_end(rsgl
->sgl
.sgt
.sgl
+ rsgl
->sgl
.sgt
.nents
- 1);
1265 /* chain the new scatterlist with previous one */
1266 if (areq
->last_rsgl
)
1267 af_alg_link_sg(&areq
->last_rsgl
->sgl
, &rsgl
->sgl
);
1269 areq
->last_rsgl
= rsgl
;
1271 atomic_add(err
, &ctx
->rcvused
);
1272 rsgl
->sg_num_bytes
= err
;
1278 EXPORT_SYMBOL_GPL(af_alg_get_rsgl
);
1280 static int __init
af_alg_init(void)
1282 int err
= proto_register(&alg_proto
, 0);
1287 err
= sock_register(&alg_family
);
1289 goto out_unregister_proto
;
1294 out_unregister_proto
:
1295 proto_unregister(&alg_proto
);
1299 static void __exit
af_alg_exit(void)
1301 sock_unregister(PF_ALG
);
1302 proto_unregister(&alg_proto
);
1305 module_init(af_alg_init
);
1306 module_exit(af_alg_exit
);
1307 MODULE_LICENSE("GPL");
1308 MODULE_ALIAS_NETPROTO(AF_ALG
);