2 * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the OpenSSL license (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
10 /* Required for vmsplice */
18 #include <openssl/engine.h>
19 #include <openssl/async.h>
20 #include <openssl/err.h>
22 #include <sys/socket.h>
23 #include <linux/version.h>
27 #if LINUX_VERSION_CODE <= KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2) || \
30 # warning "AFALG ENGINE requires Kernel Headers >= 4.1.0"
31 # warning "Skipping Compilation of AFALG engine"
33 void engine_load_afalg_int(void);
34 void engine_load_afalg_int(void)
39 # include <linux/if_alg.h>
41 # include <sys/utsname.h>
43 # include <linux/aio_abi.h>
44 # include <sys/syscall.h>
48 # include "e_afalg_err.c"
55 # ifndef SPLICE_F_GIFT
56 # define SPLICE_F_GIFT (0x08)
60 # define ALG_AES_IV_LEN 16
61 # define ALG_IV_LEN(len) (sizeof(struct af_alg_iv) + (len))
62 # define ALG_OP_TYPE unsigned int
63 # define ALG_OP_LEN (sizeof(ALG_OP_TYPE))
65 #define ALG_MAX_SALG_NAME 64
66 #define ALG_MAX_SALG_TYPE 14
68 # ifdef OPENSSL_NO_DYNAMIC_ENGINE
69 void engine_load_afalg_int(void);
72 /* Local Linkage Functions */
73 static int afalg_init_aio(afalg_aio
*aio
);
74 static int afalg_fin_cipher_aio(afalg_aio
*ptr
, int sfd
,
75 unsigned char *buf
, size_t len
);
76 static int afalg_create_sk(afalg_ctx
*actx
, const char *ciphertype
,
77 const char *ciphername
);
78 static int afalg_destroy(ENGINE
*e
);
79 static int afalg_init(ENGINE
*e
);
80 static int afalg_finish(ENGINE
*e
);
81 const EVP_CIPHER
*afalg_aes_128_cbc(void);
82 static int afalg_ciphers(ENGINE
*e
, const EVP_CIPHER
**cipher
,
83 const int **nids
, int nid
);
84 static int afalg_cipher_init(EVP_CIPHER_CTX
*ctx
, const unsigned char *key
,
85 const unsigned char *iv
, int enc
);
86 static int afalg_do_cipher(EVP_CIPHER_CTX
*ctx
, unsigned char *out
,
87 const unsigned char *in
, size_t inl
);
88 static int afalg_cipher_cleanup(EVP_CIPHER_CTX
*ctx
);
89 static int afalg_chk_platform(void);
91 /* Engine Id and Name */
92 static const char *engine_afalg_id
= "afalg";
93 static const char *engine_afalg_name
= "AFALG engine support";
95 static int afalg_cipher_nids
[] = {
99 static EVP_CIPHER
*_hidden_aes_128_cbc
= NULL
;
101 static ossl_inline
int io_setup(unsigned n
, aio_context_t
*ctx
)
103 return syscall(__NR_io_setup
, n
, ctx
);
106 static ossl_inline
int eventfd(int n
)
108 return syscall(__NR_eventfd
, n
);
111 static ossl_inline
int io_destroy(aio_context_t ctx
)
113 return syscall(__NR_io_destroy
, ctx
);
116 static ossl_inline
int io_read(aio_context_t ctx
, long n
, struct iocb
**iocb
)
118 return syscall(__NR_io_submit
, ctx
, n
, iocb
);
121 static ossl_inline
int io_getevents(aio_context_t ctx
, long min
, long max
,
122 struct io_event
*events
,
123 struct timespec
*timeout
)
125 return syscall(__NR_io_getevents
, ctx
, min
, max
, events
, timeout
);
128 static void afalg_waitfd_cleanup(ASYNC_WAIT_CTX
*ctx
, const void *key
,
129 OSSL_ASYNC_FD waitfd
, void *custom
)
134 static int afalg_setup_async_event_notification(afalg_aio
*aio
)
137 ASYNC_WAIT_CTX
*waitctx
;
141 if ((job
= ASYNC_get_current_job()) != NULL
) {
143 waitctx
= ASYNC_get_wait_ctx(job
);
144 if (waitctx
== NULL
) {
145 ALG_WARN("%s: ASYNC_get_wait_ctx error", __func__
);
148 /* Get waitfd from ASYNC_WAIT_CTX if it is alreday set */
149 ret
= ASYNC_WAIT_CTX_get_fd(waitctx
, engine_afalg_id
,
153 * waitfd is not set in ASYNC_WAIT_CTX, create a new one
154 * and set it. efd will be signaled when AIO operation completes
156 aio
->efd
= eventfd(0);
157 if (aio
->efd
== -1) {
158 ALG_PERR("%s: Failed to get eventfd : ", __func__
);
159 AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION
,
160 AFALG_R_EVENTFD_FAILED
);
163 ret
= ASYNC_WAIT_CTX_set_wait_fd(waitctx
, engine_afalg_id
,
165 afalg_waitfd_cleanup
);
167 ALG_WARN("%s: Failed to set wait fd", __func__
);
171 /* make fd non-blocking in async mode */
172 if (fcntl(aio
->efd
, F_SETFL
, O_NONBLOCK
) != 0) {
173 ALG_WARN("%s: Failed to set event fd as NONBLOCKING",
177 aio
->mode
= MODE_ASYNC
;
180 aio
->efd
= eventfd(0);
181 if (aio
->efd
== -1) {
182 ALG_PERR("%s: Failed to get eventfd : ", __func__
);
183 AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION
,
184 AFALG_R_EVENTFD_FAILED
);
187 aio
->mode
= MODE_SYNC
;
192 int afalg_init_aio(afalg_aio
*aio
)
196 /* Initialise for AIO */
198 r
= io_setup(MAX_INFLIGHTS
, &aio
->aio_ctx
);
200 ALG_PERR("%s: io_setup error : ", __func__
);
201 AFALGerr(AFALG_F_AFALG_INIT_AIO
, AFALG_R_IO_SETUP_FAILED
);
205 memset(aio
->cbt
, 0, sizeof(aio
->cbt
));
207 aio
->mode
= MODE_UNINIT
;
212 int afalg_fin_cipher_aio(afalg_aio
*aio
, int sfd
, unsigned char *buf
,
217 unsigned int done
= 0;
219 struct timespec timeout
;
220 struct io_event events
[MAX_INFLIGHTS
];
226 /* if efd has not been initialised yet do it here */
227 if (aio
->mode
== MODE_UNINIT
) {
228 r
= afalg_setup_async_event_notification(aio
);
233 cb
= &(aio
->cbt
[0 % MAX_INFLIGHTS
]);
234 memset(cb
, '\0', sizeof(*cb
));
235 cb
->aio_fildes
= sfd
;
236 cb
->aio_lio_opcode
= IOCB_CMD_PREAD
;
238 * The pointer has to be converted to unsigned value first to avoid
239 * sign extension on cast to 64 bit value in 32-bit builds
241 cb
->aio_buf
= (size_t)buf
;
244 cb
->aio_nbytes
= len
;
245 cb
->aio_flags
= IOCB_FLAG_RESFD
;
246 cb
->aio_resfd
= aio
->efd
;
249 * Perform AIO read on AFALG socket, this in turn performs an async
250 * crypto operation in kernel space
252 r
= io_read(aio
->aio_ctx
, 1, &cb
);
254 ALG_PWARN("%s: io_read failed : ", __func__
);
259 /* While AIO read is being performed pause job */
262 /* Check for completion of AIO read */
263 r
= read(aio
->efd
, &eval
, sizeof(eval
));
265 if (errno
== EAGAIN
|| errno
== EWOULDBLOCK
)
267 ALG_PERR("%s: read failed for event fd : ", __func__
);
269 } else if (r
== 0 || eval
<= 0) {
270 ALG_WARN("%s: eventfd read %d bytes, eval = %lu\n", __func__
, r
,
275 /* Get results of AIO read */
276 r
= io_getevents(aio
->aio_ctx
, 1, MAX_INFLIGHTS
,
280 * events.res indicates the actual status of the operation.
281 * Handle the error condition first.
283 if (events
[0].res
< 0) {
285 * Underlying operation cannot be completed at the time
286 * of previous submission. Resubmit for the operation.
288 if (events
[0].res
== -EBUSY
&& retry
++ < 3) {
289 r
= io_read(aio
->aio_ctx
, 1, &cb
);
291 ALG_PERR("%s: retry %d for io_read failed : ",
298 * Retries exceed for -EBUSY or unrecoverable error
299 * condition for this instance of operation.
302 ("%s: Crypto Operation failed with code %lld\n",
303 __func__
, events
[0].res
);
307 /* Operation successful. */
310 ALG_PERR("%s: io_getevents failed : ", __func__
);
313 ALG_WARN("%s: io_geteventd read 0 bytes\n", __func__
);
321 static ossl_inline
void afalg_set_op_sk(struct cmsghdr
*cmsg
,
322 const ALG_OP_TYPE op
)
324 cmsg
->cmsg_level
= SOL_ALG
;
325 cmsg
->cmsg_type
= ALG_SET_OP
;
326 cmsg
->cmsg_len
= CMSG_LEN(ALG_OP_LEN
);
327 memcpy(CMSG_DATA(cmsg
), &op
, ALG_OP_LEN
);
330 static void afalg_set_iv_sk(struct cmsghdr
*cmsg
, const unsigned char *iv
,
331 const unsigned int len
)
333 struct af_alg_iv
*aiv
;
335 cmsg
->cmsg_level
= SOL_ALG
;
336 cmsg
->cmsg_type
= ALG_SET_IV
;
337 cmsg
->cmsg_len
= CMSG_LEN(ALG_IV_LEN(len
));
338 aiv
= (struct af_alg_iv
*)CMSG_DATA(cmsg
);
340 memcpy(aiv
->iv
, iv
, len
);
343 static ossl_inline
int afalg_set_key(afalg_ctx
*actx
, const unsigned char *key
,
347 ret
= setsockopt(actx
->bfd
, SOL_ALG
, ALG_SET_KEY
, key
, klen
);
349 ALG_PERR("%s: Failed to set socket option : ", __func__
);
350 AFALGerr(AFALG_F_AFALG_SET_KEY
, AFALG_R_SOCKET_SET_KEY_FAILED
);
357 static int afalg_create_sk(afalg_ctx
*actx
, const char *ciphertype
,
358 const char *ciphername
)
360 struct sockaddr_alg sa
;
363 actx
->bfd
= actx
->sfd
= -1;
365 memset(&sa
, 0, sizeof(sa
));
366 sa
.salg_family
= AF_ALG
;
367 strncpy((char *) sa
.salg_type
, ciphertype
, ALG_MAX_SALG_TYPE
);
368 sa
.salg_type
[ALG_MAX_SALG_TYPE
-1] = '\0';
369 strncpy((char *) sa
.salg_name
, ciphername
, ALG_MAX_SALG_NAME
);
370 sa
.salg_name
[ALG_MAX_SALG_NAME
-1] = '\0';
372 actx
->bfd
= socket(AF_ALG
, SOCK_SEQPACKET
, 0);
373 if (actx
->bfd
== -1) {
374 ALG_PERR("%s: Failed to open socket : ", __func__
);
375 AFALGerr(AFALG_F_AFALG_CREATE_SK
, AFALG_R_SOCKET_CREATE_FAILED
);
379 r
= bind(actx
->bfd
, (struct sockaddr
*)&sa
, sizeof(sa
));
381 ALG_PERR("%s: Failed to bind socket : ", __func__
);
382 AFALGerr(AFALG_F_AFALG_CREATE_SK
, AFALG_R_SOCKET_BIND_FAILED
);
386 actx
->sfd
= accept(actx
->bfd
, NULL
, 0);
388 ALG_PERR("%s: Socket Accept Failed : ", __func__
);
389 AFALGerr(AFALG_F_AFALG_CREATE_SK
, AFALG_R_SOCKET_ACCEPT_FAILED
);
400 actx
->bfd
= actx
->sfd
= -1;
404 static int afalg_start_cipher_sk(afalg_ctx
*actx
, const unsigned char *in
,
405 size_t inl
, const unsigned char *iv
,
408 struct msghdr msg
= { 0 };
409 struct cmsghdr
*cmsg
;
412 # ifdef ALG_ZERO_COPY
415 char cbuf
[CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN
)) + CMSG_SPACE(ALG_OP_LEN
)];
417 memset(cbuf
, 0, sizeof(cbuf
));
418 msg
.msg_control
= cbuf
;
419 msg
.msg_controllen
= sizeof(cbuf
);
422 * cipher direction (i.e. encrypt or decrypt) and iv are sent to the
423 * kernel as part of sendmsg()'s ancillary data
425 cmsg
= CMSG_FIRSTHDR(&msg
);
426 afalg_set_op_sk(cmsg
, enc
);
427 cmsg
= CMSG_NXTHDR(&msg
, cmsg
);
428 afalg_set_iv_sk(cmsg
, iv
, ALG_AES_IV_LEN
);
430 /* iov that describes input data */
431 iov
.iov_base
= (unsigned char *)in
;
434 msg
.msg_flags
= MSG_MORE
;
436 # ifdef ALG_ZERO_COPY
439 * Works best when buffer is 4k aligned
440 * OPENS: out of place processing (i.e. out != in)
443 /* Input data is not sent as part of call to sendmsg() */
447 /* Sendmsg() sends iv and cipher direction to the kernel */
448 sbytes
= sendmsg(actx
->sfd
, &msg
, 0);
450 ALG_PERR("%s: sendmsg failed for zero copy cipher operation : ",
456 * vmsplice and splice are used to pin the user space input buffer for
457 * kernel space processing avoiding copys from user to kernel space
459 ret
= vmsplice(actx
->zc_pipe
[1], &iov
, 1, SPLICE_F_GIFT
);
461 ALG_PERR("%s: vmsplice failed : ", __func__
);
465 ret
= splice(actx
->zc_pipe
[0], NULL
, actx
->sfd
, NULL
, inl
, 0);
467 ALG_PERR("%s: splice failed : ", __func__
);
474 /* Sendmsg() sends iv, cipher direction and input data to the kernel */
475 sbytes
= sendmsg(actx
->sfd
, &msg
, 0);
477 ALG_PERR("%s: sendmsg failed for cipher operation : ", __func__
);
481 if (sbytes
!= (ssize_t
) inl
) {
482 ALG_WARN("Cipher operation send bytes %zd != inlen %zd\n", sbytes
,
491 static int afalg_cipher_init(EVP_CIPHER_CTX
*ctx
, const unsigned char *key
,
492 const unsigned char *iv
, int enc
)
497 char ciphername
[ALG_MAX_SALG_NAME
];
499 if (ctx
== NULL
|| key
== NULL
) {
500 ALG_WARN("%s: Null Parameter\n", __func__
);
504 if (EVP_CIPHER_CTX_cipher(ctx
) == NULL
) {
505 ALG_WARN("%s: Cipher object NULL\n", __func__
);
509 actx
= EVP_CIPHER_CTX_get_cipher_data(ctx
);
511 ALG_WARN("%s: Cipher data NULL\n", __func__
);
515 ciphertype
= EVP_CIPHER_CTX_nid(ctx
);
516 switch (ciphertype
) {
517 case NID_aes_128_cbc
:
518 strncpy(ciphername
, "cbc(aes)", ALG_MAX_SALG_NAME
);
521 ALG_WARN("%s: Unsupported Cipher type %d\n", __func__
, ciphertype
);
524 ciphername
[ALG_MAX_SALG_NAME
-1]='\0';
526 if (ALG_AES_IV_LEN
!= EVP_CIPHER_CTX_iv_length(ctx
)) {
527 ALG_WARN("%s: Unsupported IV length :%d\n", __func__
,
528 EVP_CIPHER_CTX_iv_length(ctx
));
532 /* Setup AFALG socket for crypto processing */
533 ret
= afalg_create_sk(actx
, "skcipher", ciphername
);
538 ret
= afalg_set_key(actx
, key
, EVP_CIPHER_CTX_key_length(ctx
));
542 /* Setup AIO ctx to allow async AFALG crypto processing */
543 if (afalg_init_aio(&actx
->aio
) == 0)
546 # ifdef ALG_ZERO_COPY
550 actx
->init_done
= MAGIC_INIT_NUM
;
560 static int afalg_do_cipher(EVP_CIPHER_CTX
*ctx
, unsigned char *out
,
561 const unsigned char *in
, size_t inl
)
565 char nxtiv
[ALG_AES_IV_LEN
] = { 0 };
567 if (ctx
== NULL
|| out
== NULL
|| in
== NULL
) {
568 ALG_WARN("NULL parameter passed to function %s\n", __func__
);
572 actx
= (afalg_ctx
*) EVP_CIPHER_CTX_get_cipher_data(ctx
);
573 if (actx
== NULL
|| actx
->init_done
!= MAGIC_INIT_NUM
) {
574 ALG_WARN("%s afalg ctx passed\n",
575 ctx
== NULL
? "NULL" : "Uninitialised");
580 * set iv now for decrypt operation as the input buffer can be
581 * overwritten for inplace operation where in = out.
583 if (EVP_CIPHER_CTX_encrypting(ctx
) == 0) {
584 memcpy(nxtiv
, in
+ (inl
- ALG_AES_IV_LEN
), ALG_AES_IV_LEN
);
587 /* Send input data to kernel space */
588 ret
= afalg_start_cipher_sk(actx
, (unsigned char *)in
, inl
,
589 EVP_CIPHER_CTX_iv(ctx
),
590 EVP_CIPHER_CTX_encrypting(ctx
));
595 /* Perform async crypto operation in kernel space */
596 ret
= afalg_fin_cipher_aio(&actx
->aio
, actx
->sfd
, out
, inl
);
600 if (EVP_CIPHER_CTX_encrypting(ctx
)) {
601 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx
), out
+ (inl
- ALG_AES_IV_LEN
),
604 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx
), nxtiv
, ALG_AES_IV_LEN
);
610 static int afalg_cipher_cleanup(EVP_CIPHER_CTX
*ctx
)
615 ALG_WARN("NULL parameter passed to function %s\n", __func__
);
619 actx
= (afalg_ctx
*) EVP_CIPHER_CTX_get_cipher_data(ctx
);
620 if (actx
== NULL
|| actx
->init_done
!= MAGIC_INIT_NUM
) {
621 ALG_WARN("%s afalg ctx passed\n",
622 ctx
== NULL
? "NULL" : "Uninitialised");
628 # ifdef ALG_ZERO_COPY
629 close(actx
->zc_pipe
[0]);
630 close(actx
->zc_pipe
[1]);
632 /* close efd in sync mode, async mode is closed in afalg_waitfd_cleanup() */
633 if (actx
->aio
.mode
== MODE_SYNC
)
634 close(actx
->aio
.efd
);
635 io_destroy(actx
->aio
.aio_ctx
);
640 const EVP_CIPHER
*afalg_aes_128_cbc(void)
642 if (_hidden_aes_128_cbc
== NULL
643 && ((_hidden_aes_128_cbc
=
644 EVP_CIPHER_meth_new(NID_aes_128_cbc
,
646 AES_KEY_SIZE_128
)) == NULL
647 || !EVP_CIPHER_meth_set_iv_length(_hidden_aes_128_cbc
, AES_IV_LEN
)
648 || !EVP_CIPHER_meth_set_flags(_hidden_aes_128_cbc
,
650 EVP_CIPH_FLAG_DEFAULT_ASN1
)
651 || !EVP_CIPHER_meth_set_init(_hidden_aes_128_cbc
,
653 || !EVP_CIPHER_meth_set_do_cipher(_hidden_aes_128_cbc
,
655 || !EVP_CIPHER_meth_set_cleanup(_hidden_aes_128_cbc
,
656 afalg_cipher_cleanup
)
657 || !EVP_CIPHER_meth_set_impl_ctx_size(_hidden_aes_128_cbc
,
658 sizeof(afalg_ctx
)))) {
659 EVP_CIPHER_meth_free(_hidden_aes_128_cbc
);
660 _hidden_aes_128_cbc
= NULL
;
662 return _hidden_aes_128_cbc
;
665 static int afalg_ciphers(ENGINE
*e
, const EVP_CIPHER
**cipher
,
666 const int **nids
, int nid
)
670 if (cipher
== NULL
) {
671 *nids
= afalg_cipher_nids
;
672 return (sizeof(afalg_cipher_nids
) / sizeof(afalg_cipher_nids
[0]));
676 case NID_aes_128_cbc
:
677 *cipher
= afalg_aes_128_cbc();
687 static int bind_afalg(ENGINE
*e
)
689 /* Ensure the afalg error handling is set up */
690 ERR_load_AFALG_strings();
692 if (!ENGINE_set_id(e
, engine_afalg_id
)
693 || !ENGINE_set_name(e
, engine_afalg_name
)
694 || !ENGINE_set_destroy_function(e
, afalg_destroy
)
695 || !ENGINE_set_init_function(e
, afalg_init
)
696 || !ENGINE_set_finish_function(e
, afalg_finish
)) {
697 AFALGerr(AFALG_F_BIND_AFALG
, AFALG_R_INIT_FAILED
);
702 * Create _hidden_aes_128_cbc by calling afalg_aes_128_cbc
703 * now, as bind_aflag can only be called by one thread at a
706 if (afalg_aes_128_cbc() == NULL
) {
707 AFALGerr(AFALG_F_BIND_AFALG
, AFALG_R_INIT_FAILED
);
711 if (!ENGINE_set_ciphers(e
, afalg_ciphers
)) {
712 AFALGerr(AFALG_F_BIND_AFALG
, AFALG_R_INIT_FAILED
);
719 # ifndef OPENSSL_NO_DYNAMIC_ENGINE
720 static int bind_helper(ENGINE
*e
, const char *id
)
722 if (id
&& (strcmp(id
, engine_afalg_id
) != 0))
725 if (!afalg_chk_platform())
733 IMPLEMENT_DYNAMIC_CHECK_FN()
734 IMPLEMENT_DYNAMIC_BIND_FN(bind_helper
)
737 static int afalg_chk_platform(void)
741 int kver
[3] = { -1, -1, -1 };
748 AFALGerr(AFALG_F_AFALG_CHK_PLATFORM
,
749 AFALG_R_FAILED_TO_GET_PLATFORM_INFO
);
753 str
= strtok(ut
.release
, ".");
754 for (i
= 0; i
< 3 && str
!= NULL
; i
++) {
756 str
= strtok(NULL
, ".");
759 if (KERNEL_VERSION(kver
[0], kver
[1], kver
[2])
760 < KERNEL_VERSION(K_MAJ
, K_MIN1
, K_MIN2
)) {
761 ALG_ERR("ASYNC AFALG not supported this kernel(%d.%d.%d)\n",
762 kver
[0], kver
[1], kver
[2]);
763 ALG_ERR("ASYNC AFALG requires kernel version %d.%d.%d or later\n",
764 K_MAJ
, K_MIN1
, K_MIN2
);
765 AFALGerr(AFALG_F_AFALG_CHK_PLATFORM
,
766 AFALG_R_KERNEL_DOES_NOT_SUPPORT_ASYNC_AFALG
);
770 /* Test if we can actually create an AF_ALG socket */
771 sock
= socket(AF_ALG
, SOCK_SEQPACKET
, 0);
773 AFALGerr(AFALG_F_AFALG_CHK_PLATFORM
, AFALG_R_SOCKET_CREATE_FAILED
);
781 # ifdef OPENSSL_NO_DYNAMIC_ENGINE
782 static ENGINE
*engine_afalg(void)
784 ENGINE
*ret
= ENGINE_new();
787 if (!bind_afalg(ret
)) {
794 void engine_load_afalg_int(void)
798 if (!afalg_chk_platform())
801 toadd
= engine_afalg();
810 static int afalg_init(ENGINE
*e
)
815 static int afalg_finish(ENGINE
*e
)
820 static int afalg_destroy(ENGINE
*e
)
822 ERR_unload_AFALG_strings();
823 EVP_CIPHER_meth_free(_hidden_aes_128_cbc
);
824 _hidden_aes_128_cbc
= NULL
;
828 #endif /* KERNEL VERSION */