]>
Commit | Line | Data |
---|---|---|
440e5d80 | 1 | /* |
33388b44 | 2 | * Copyright 2016-2020 The OpenSSL Project Authors. All Rights Reserved. |
7f458a48 | 3 | * |
ab3fa1c0 | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
440e5d80 RS |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
7f458a48 | 8 | */ |
9 | ||
10 | /* Required for vmsplice */ | |
4e399729 AP |
11 | #ifndef _GNU_SOURCE |
12 | # define _GNU_SOURCE | |
13 | #endif | |
7f458a48 | 14 | #include <stdio.h> |
15 | #include <string.h> | |
16 | #include <unistd.h> | |
17 | ||
18 | #include <openssl/engine.h> | |
19 | #include <openssl/async.h> | |
6bf73e53 | 20 | #include <openssl/err.h> |
49ea0f09 | 21 | #include "internal/nelem.h" |
7f458a48 | 22 | |
4e399729 | 23 | #include <sys/socket.h> |
7f458a48 | 24 | #include <linux/version.h> |
25 | #define K_MAJ 4 | |
26 | #define K_MIN1 1 | |
27 | #define K_MIN2 0 | |
3ba70235 | 28 | #if LINUX_VERSION_CODE < KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2) || \ |
4e399729 | 29 | !defined(AF_ALG) |
97043e46 RL |
30 | # ifndef PEDANTIC |
31 | # warning "AFALG ENGINE requires Kernel Headers >= 4.1.0" | |
32 | # warning "Skipping Compilation of AFALG engine" | |
33 | # endif | |
a1fd1fb2 | 34 | void engine_load_afalg_int(void); |
627537dd MC |
35 | void engine_load_afalg_int(void) |
36 | { | |
37 | } | |
7f458a48 | 38 | #else |
39 | ||
40 | # include <linux/if_alg.h> | |
7f458a48 | 41 | # include <fcntl.h> |
42 | # include <sys/utsname.h> | |
43 | ||
44 | # include <linux/aio_abi.h> | |
45 | # include <sys/syscall.h> | |
46 | # include <errno.h> | |
47 | ||
48 | # include "e_afalg.h" | |
52df25cf | 49 | # include "e_afalg_err.c" |
7f458a48 | 50 | |
51 | # ifndef SOL_ALG | |
52 | # define SOL_ALG 279 | |
53 | # endif | |
54 | ||
55 | # ifdef ALG_ZERO_COPY | |
56 | # ifndef SPLICE_F_GIFT | |
57 | # define SPLICE_F_GIFT (0x08) | |
58 | # endif | |
59 | # endif | |
60 | ||
61 | # define ALG_AES_IV_LEN 16 | |
62 | # define ALG_IV_LEN(len) (sizeof(struct af_alg_iv) + (len)) | |
63 | # define ALG_OP_TYPE unsigned int | |
64 | # define ALG_OP_LEN (sizeof(ALG_OP_TYPE)) | |
65 | ||
6cba4a66 | 66 | # ifdef OPENSSL_NO_DYNAMIC_ENGINE |
b3599dbb | 67 | void engine_load_afalg_int(void); |
6cba4a66 | 68 | # endif |
69 | ||
7f458a48 | 70 | /* Local Linkage Functions */ |
71 | static int afalg_init_aio(afalg_aio *aio); | |
72 | static int afalg_fin_cipher_aio(afalg_aio *ptr, int sfd, | |
73 | unsigned char *buf, size_t len); | |
6cba4a66 | 74 | static int afalg_create_sk(afalg_ctx *actx, const char *ciphertype, |
75 | const char *ciphername); | |
7f458a48 | 76 | static int afalg_destroy(ENGINE *e); |
77 | static int afalg_init(ENGINE *e); | |
78 | static int afalg_finish(ENGINE *e); | |
3a7141df | 79 | static const EVP_CIPHER *afalg_aes_cbc(int nid); |
7e8a5e30 | 80 | static cbc_handles *get_cipher_handle(int nid); |
7f458a48 | 81 | static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher, |
82 | const int **nids, int nid); | |
83 | static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, | |
84 | const unsigned char *iv, int enc); | |
85 | static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, | |
86 | const unsigned char *in, size_t inl); | |
87 | static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx); | |
88 | static int afalg_chk_platform(void); | |
89 | ||
90 | /* Engine Id and Name */ | |
91 | static const char *engine_afalg_id = "afalg"; | |
538dbbc6 | 92 | static const char *engine_afalg_name = "AFALG engine support"; |
7f458a48 | 93 | |
a1933888 | 94 | static int afalg_cipher_nids[] = { |
49ea0f09 J |
95 | NID_aes_128_cbc, |
96 | NID_aes_192_cbc, | |
97 | NID_aes_256_cbc, | |
7f458a48 | 98 | }; |
99 | ||
49ea0f09 J |
100 | static cbc_handles cbc_handle[] = {{AES_KEY_SIZE_128, NULL}, |
101 | {AES_KEY_SIZE_192, NULL}, | |
102 | {AES_KEY_SIZE_256, NULL}}; | |
7f458a48 | 103 | |
2a7de0fd | 104 | static ossl_inline int io_setup(unsigned n, aio_context_t *ctx) |
7f458a48 | 105 | { |
106 | return syscall(__NR_io_setup, n, ctx); | |
107 | } | |
108 | ||
2a7de0fd | 109 | static ossl_inline int eventfd(int n) |
7f458a48 | 110 | { |
bee9c8a4 | 111 | return syscall(__NR_eventfd2, n, 0); |
7f458a48 | 112 | } |
113 | ||
2a7de0fd | 114 | static ossl_inline int io_destroy(aio_context_t ctx) |
7f458a48 | 115 | { |
116 | return syscall(__NR_io_destroy, ctx); | |
117 | } | |
118 | ||
2a7de0fd | 119 | static ossl_inline int io_read(aio_context_t ctx, long n, struct iocb **iocb) |
7f458a48 | 120 | { |
121 | return syscall(__NR_io_submit, ctx, n, iocb); | |
122 | } | |
123 | ||
2a7de0fd | 124 | static ossl_inline int io_getevents(aio_context_t ctx, long min, long max, |
7f458a48 | 125 | struct io_event *events, |
126 | struct timespec *timeout) | |
127 | { | |
5b5e2985 | 128 | #if defined(__NR_io_getevents) |
7f458a48 | 129 | return syscall(__NR_io_getevents, ctx, min, max, events, timeout); |
5b5e2985 AF |
130 | #elif defined(__NR_io_pgetevents_time64) |
131 | /* Let's only support the 64 suffix syscalls for 64-bit time_t. | |
132 | * This simplifies the code for us as we don't need to use a 64-bit | |
133 | * version of timespec with a 32-bit time_t and handle converting | |
134 | * between 64-bit and 32-bit times and check for overflows. | |
135 | */ | |
136 | if (sizeof(timeout->tv_sec) == 8) | |
137 | return syscall(__NR_io_pgetevents_time64, ctx, min, max, events, timeout, NULL); | |
138 | else { | |
139 | errno = ENOSYS; | |
140 | return -1; | |
141 | } | |
142 | #else | |
143 | # error "We require either the io_getevents syscall or __NR_io_pgetevents_time64." | |
144 | #endif | |
7f458a48 | 145 | } |
146 | ||
147 | static void afalg_waitfd_cleanup(ASYNC_WAIT_CTX *ctx, const void *key, | |
148 | OSSL_ASYNC_FD waitfd, void *custom) | |
149 | { | |
150 | close(waitfd); | |
151 | } | |
152 | ||
153 | static int afalg_setup_async_event_notification(afalg_aio *aio) | |
154 | { | |
155 | ASYNC_JOB *job; | |
156 | ASYNC_WAIT_CTX *waitctx; | |
157 | void *custom = NULL; | |
6cba4a66 | 158 | int ret; |
7f458a48 | 159 | |
160 | if ((job = ASYNC_get_current_job()) != NULL) { | |
161 | /* Async mode */ | |
162 | waitctx = ASYNC_get_wait_ctx(job); | |
163 | if (waitctx == NULL) { | |
24fa4b8d | 164 | ALG_WARN("%s(%d): ASYNC_get_wait_ctx error", __FILE__, __LINE__); |
7f458a48 | 165 | return 0; |
166 | } | |
46f4e1be | 167 | /* Get waitfd from ASYNC_WAIT_CTX if it is already set */ |
6cba4a66 | 168 | ret = ASYNC_WAIT_CTX_get_fd(waitctx, engine_afalg_id, |
169 | &aio->efd, &custom); | |
170 | if (ret == 0) { | |
171 | /* | |
172 | * waitfd is not set in ASYNC_WAIT_CTX, create a new one | |
173 | * and set it. efd will be signaled when AIO operation completes | |
174 | */ | |
175 | aio->efd = eventfd(0); | |
176 | if (aio->efd == -1) { | |
24fa4b8d BE |
177 | ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__, |
178 | __LINE__); | |
7f458a48 | 179 | AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION, |
180 | AFALG_R_EVENTFD_FAILED); | |
181 | return 0; | |
182 | } | |
6cba4a66 | 183 | ret = ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_afalg_id, |
184 | aio->efd, custom, | |
185 | afalg_waitfd_cleanup); | |
186 | if (ret == 0) { | |
24fa4b8d | 187 | ALG_WARN("%s(%d): Failed to set wait fd", __FILE__, __LINE__); |
6cba4a66 | 188 | close(aio->efd); |
7f458a48 | 189 | return 0; |
190 | } | |
191 | /* make fd non-blocking in async mode */ | |
6cba4a66 | 192 | if (fcntl(aio->efd, F_SETFL, O_NONBLOCK) != 0) { |
24fa4b8d BE |
193 | ALG_WARN("%s(%d): Failed to set event fd as NONBLOCKING", |
194 | __FILE__, __LINE__); | |
7f458a48 | 195 | } |
196 | } | |
6cba4a66 | 197 | aio->mode = MODE_ASYNC; |
7f458a48 | 198 | } else { |
199 | /* Sync mode */ | |
6cba4a66 | 200 | aio->efd = eventfd(0); |
201 | if (aio->efd == -1) { | |
24fa4b8d | 202 | ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__, __LINE__); |
7f458a48 | 203 | AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION, |
204 | AFALG_R_EVENTFD_FAILED); | |
205 | return 0; | |
206 | } | |
6cba4a66 | 207 | aio->mode = MODE_SYNC; |
7f458a48 | 208 | } |
209 | return 1; | |
210 | } | |
211 | ||
3a7141df | 212 | static int afalg_init_aio(afalg_aio *aio) |
7f458a48 | 213 | { |
214 | int r = -1; | |
215 | ||
216 | /* Initialise for AIO */ | |
217 | aio->aio_ctx = 0; | |
218 | r = io_setup(MAX_INFLIGHTS, &aio->aio_ctx); | |
219 | if (r < 0) { | |
24fa4b8d | 220 | ALG_PERR("%s(%d): io_setup error : ", __FILE__, __LINE__); |
7f458a48 | 221 | AFALGerr(AFALG_F_AFALG_INIT_AIO, AFALG_R_IO_SETUP_FAILED); |
222 | return 0; | |
223 | } | |
224 | ||
225 | memset(aio->cbt, 0, sizeof(aio->cbt)); | |
7f458a48 | 226 | aio->efd = -1; |
6cba4a66 | 227 | aio->mode = MODE_UNINIT; |
7f458a48 | 228 | |
229 | return 1; | |
230 | } | |
231 | ||
3a7141df RL |
232 | static int afalg_fin_cipher_aio(afalg_aio *aio, int sfd, unsigned char *buf, |
233 | size_t len) | |
7f458a48 | 234 | { |
235 | int r; | |
236 | int retry = 0; | |
237 | unsigned int done = 0; | |
238 | struct iocb *cb; | |
239 | struct timespec timeout; | |
240 | struct io_event events[MAX_INFLIGHTS]; | |
241 | u_int64_t eval = 0; | |
242 | ||
243 | timeout.tv_sec = 0; | |
244 | timeout.tv_nsec = 0; | |
245 | ||
246 | /* if efd has not been initialised yet do it here */ | |
6cba4a66 | 247 | if (aio->mode == MODE_UNINIT) { |
7f458a48 | 248 | r = afalg_setup_async_event_notification(aio); |
249 | if (r == 0) | |
250 | return 0; | |
251 | } | |
252 | ||
253 | cb = &(aio->cbt[0 % MAX_INFLIGHTS]); | |
254 | memset(cb, '\0', sizeof(*cb)); | |
255 | cb->aio_fildes = sfd; | |
256 | cb->aio_lio_opcode = IOCB_CMD_PREAD; | |
c27778d8 AP |
257 | /* |
258 | * The pointer has to be converted to unsigned value first to avoid | |
259 | * sign extension on cast to 64 bit value in 32-bit builds | |
260 | */ | |
261 | cb->aio_buf = (size_t)buf; | |
7f458a48 | 262 | cb->aio_offset = 0; |
263 | cb->aio_data = 0; | |
264 | cb->aio_nbytes = len; | |
265 | cb->aio_flags = IOCB_FLAG_RESFD; | |
266 | cb->aio_resfd = aio->efd; | |
267 | ||
268 | /* | |
269 | * Perform AIO read on AFALG socket, this in turn performs an async | |
270 | * crypto operation in kernel space | |
271 | */ | |
272 | r = io_read(aio->aio_ctx, 1, &cb); | |
273 | if (r < 0) { | |
24fa4b8d | 274 | ALG_PWARN("%s(%d): io_read failed : ", __FILE__, __LINE__); |
7f458a48 | 275 | return 0; |
276 | } | |
277 | ||
278 | do { | |
279 | /* While AIO read is being performed pause job */ | |
280 | ASYNC_pause_job(); | |
281 | ||
282 | /* Check for completion of AIO read */ | |
283 | r = read(aio->efd, &eval, sizeof(eval)); | |
284 | if (r < 0) { | |
285 | if (errno == EAGAIN || errno == EWOULDBLOCK) | |
286 | continue; | |
24fa4b8d | 287 | ALG_PERR("%s(%d): read failed for event fd : ", __FILE__, __LINE__); |
7f458a48 | 288 | return 0; |
289 | } else if (r == 0 || eval <= 0) { | |
24fa4b8d BE |
290 | ALG_WARN("%s(%d): eventfd read %d bytes, eval = %lu\n", __FILE__, |
291 | __LINE__, r, eval); | |
7f458a48 | 292 | } |
293 | if (eval > 0) { | |
294 | ||
295 | /* Get results of AIO read */ | |
6cba4a66 | 296 | r = io_getevents(aio->aio_ctx, 1, MAX_INFLIGHTS, |
297 | events, &timeout); | |
7f458a48 | 298 | if (r > 0) { |
299 | /* | |
300 | * events.res indicates the actual status of the operation. | |
301 | * Handle the error condition first. | |
302 | */ | |
303 | if (events[0].res < 0) { | |
304 | /* | |
305 | * Underlying operation cannot be completed at the time | |
306 | * of previous submission. Resubmit for the operation. | |
307 | */ | |
308 | if (events[0].res == -EBUSY && retry++ < 3) { | |
309 | r = io_read(aio->aio_ctx, 1, &cb); | |
310 | if (r < 0) { | |
24fa4b8d BE |
311 | ALG_PERR("%s(%d): retry %d for io_read failed : ", |
312 | __FILE__, __LINE__, retry); | |
7f458a48 | 313 | return 0; |
314 | } | |
315 | continue; | |
316 | } else { | |
317 | /* | |
318 | * Retries exceed for -EBUSY or unrecoverable error | |
319 | * condition for this instance of operation. | |
320 | */ | |
321 | ALG_WARN | |
24fa4b8d BE |
322 | ("%s(%d): Crypto Operation failed with code %lld\n", |
323 | __FILE__, __LINE__, events[0].res); | |
7f458a48 | 324 | return 0; |
325 | } | |
326 | } | |
327 | /* Operation successful. */ | |
328 | done = 1; | |
329 | } else if (r < 0) { | |
24fa4b8d | 330 | ALG_PERR("%s(%d): io_getevents failed : ", __FILE__, __LINE__); |
7f458a48 | 331 | return 0; |
332 | } else { | |
24fa4b8d BE |
333 | ALG_WARN("%s(%d): io_geteventd read 0 bytes\n", __FILE__, |
334 | __LINE__); | |
7f458a48 | 335 | } |
336 | } | |
337 | } while (!done); | |
338 | ||
339 | return 1; | |
340 | } | |
341 | ||
2a7de0fd | 342 | static ossl_inline void afalg_set_op_sk(struct cmsghdr *cmsg, |
574cffd5 | 343 | const ALG_OP_TYPE op) |
7f458a48 | 344 | { |
345 | cmsg->cmsg_level = SOL_ALG; | |
346 | cmsg->cmsg_type = ALG_SET_OP; | |
347 | cmsg->cmsg_len = CMSG_LEN(ALG_OP_LEN); | |
574cffd5 | 348 | memcpy(CMSG_DATA(cmsg), &op, ALG_OP_LEN); |
7f458a48 | 349 | } |
350 | ||
351 | static void afalg_set_iv_sk(struct cmsghdr *cmsg, const unsigned char *iv, | |
352 | const unsigned int len) | |
353 | { | |
354 | struct af_alg_iv *aiv; | |
355 | ||
356 | cmsg->cmsg_level = SOL_ALG; | |
357 | cmsg->cmsg_type = ALG_SET_IV; | |
358 | cmsg->cmsg_len = CMSG_LEN(ALG_IV_LEN(len)); | |
359 | aiv = (struct af_alg_iv *)CMSG_DATA(cmsg); | |
360 | aiv->ivlen = len; | |
361 | memcpy(aiv->iv, iv, len); | |
362 | } | |
363 | ||
2a7de0fd | 364 | static ossl_inline int afalg_set_key(afalg_ctx *actx, const unsigned char *key, |
6cba4a66 | 365 | const int klen) |
7f458a48 | 366 | { |
367 | int ret; | |
6cba4a66 | 368 | ret = setsockopt(actx->bfd, SOL_ALG, ALG_SET_KEY, key, klen); |
369 | if (ret < 0) { | |
24fa4b8d | 370 | ALG_PERR("%s(%d): Failed to set socket option : ", __FILE__, __LINE__); |
6cba4a66 | 371 | AFALGerr(AFALG_F_AFALG_SET_KEY, AFALG_R_SOCKET_SET_KEY_FAILED); |
372 | return 0; | |
373 | } | |
6cba4a66 | 374 | return 1; |
375 | } | |
376 | ||
377 | static int afalg_create_sk(afalg_ctx *actx, const char *ciphertype, | |
378 | const char *ciphername) | |
379 | { | |
380 | struct sockaddr_alg sa; | |
c27778d8 | 381 | int r = -1; |
7f458a48 | 382 | |
383 | actx->bfd = actx->sfd = -1; | |
384 | ||
6cba4a66 | 385 | memset(&sa, 0, sizeof(sa)); |
386 | sa.salg_family = AF_ALG; | |
62cc845f BE |
387 | OPENSSL_strlcpy((char *) sa.salg_type, ciphertype, sizeof(sa.salg_type)); |
388 | OPENSSL_strlcpy((char *) sa.salg_name, ciphername, sizeof(sa.salg_name)); | |
6cba4a66 | 389 | |
390 | actx->bfd = socket(AF_ALG, SOCK_SEQPACKET, 0); | |
391 | if (actx->bfd == -1) { | |
24fa4b8d | 392 | ALG_PERR("%s(%d): Failed to open socket : ", __FILE__, __LINE__); |
6cba4a66 | 393 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_CREATE_FAILED); |
394 | goto err; | |
7f458a48 | 395 | } |
396 | ||
6cba4a66 | 397 | r = bind(actx->bfd, (struct sockaddr *)&sa, sizeof(sa)); |
398 | if (r < 0) { | |
24fa4b8d | 399 | ALG_PERR("%s(%d): Failed to bind socket : ", __FILE__, __LINE__); |
6cba4a66 | 400 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_BIND_FAILED); |
7f458a48 | 401 | goto err; |
402 | } | |
403 | ||
7f458a48 | 404 | actx->sfd = accept(actx->bfd, NULL, 0); |
405 | if (actx->sfd < 0) { | |
24fa4b8d | 406 | ALG_PERR("%s(%d): Socket Accept Failed : ", __FILE__, __LINE__); |
6cba4a66 | 407 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_ACCEPT_FAILED); |
7f458a48 | 408 | goto err; |
409 | } | |
410 | ||
6cba4a66 | 411 | return 1; |
7f458a48 | 412 | |
413 | err: | |
414 | if (actx->bfd >= 0) | |
415 | close(actx->bfd); | |
416 | if (actx->sfd >= 0) | |
417 | close(actx->sfd); | |
418 | actx->bfd = actx->sfd = -1; | |
6cba4a66 | 419 | return 0; |
7f458a48 | 420 | } |
421 | ||
422 | static int afalg_start_cipher_sk(afalg_ctx *actx, const unsigned char *in, | |
423 | size_t inl, const unsigned char *iv, | |
424 | unsigned int enc) | |
425 | { | |
64a45882 | 426 | struct msghdr msg; |
7f458a48 | 427 | struct cmsghdr *cmsg; |
428 | struct iovec iov; | |
429 | ssize_t sbytes; | |
430 | # ifdef ALG_ZERO_COPY | |
431 | int ret; | |
432 | # endif | |
a1933888 | 433 | char cbuf[CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN)) + CMSG_SPACE(ALG_OP_LEN)]; |
7f458a48 | 434 | |
64a45882 | 435 | memset(&msg, 0, sizeof(msg)); |
a1933888 | 436 | memset(cbuf, 0, sizeof(cbuf)); |
7f458a48 | 437 | msg.msg_control = cbuf; |
a1933888 | 438 | msg.msg_controllen = sizeof(cbuf); |
7f458a48 | 439 | |
440 | /* | |
441 | * cipher direction (i.e. encrypt or decrypt) and iv are sent to the | |
442 | * kernel as part of sendmsg()'s ancillary data | |
443 | */ | |
444 | cmsg = CMSG_FIRSTHDR(&msg); | |
445 | afalg_set_op_sk(cmsg, enc); | |
446 | cmsg = CMSG_NXTHDR(&msg, cmsg); | |
447 | afalg_set_iv_sk(cmsg, iv, ALG_AES_IV_LEN); | |
448 | ||
449 | /* iov that describes input data */ | |
450 | iov.iov_base = (unsigned char *)in; | |
451 | iov.iov_len = inl; | |
452 | ||
453 | msg.msg_flags = MSG_MORE; | |
454 | ||
455 | # ifdef ALG_ZERO_COPY | |
456 | /* | |
457 | * ZERO_COPY mode | |
458 | * Works best when buffer is 4k aligned | |
459 | * OPENS: out of place processing (i.e. out != in) | |
460 | */ | |
461 | ||
462 | /* Input data is not sent as part of call to sendmsg() */ | |
463 | msg.msg_iovlen = 0; | |
464 | msg.msg_iov = NULL; | |
465 | ||
466 | /* Sendmsg() sends iv and cipher direction to the kernel */ | |
467 | sbytes = sendmsg(actx->sfd, &msg, 0); | |
468 | if (sbytes < 0) { | |
24fa4b8d BE |
469 | ALG_PERR("%s(%d): sendmsg failed for zero copy cipher operation : ", |
470 | __FILE__, __LINE__); | |
7f458a48 | 471 | return 0; |
472 | } | |
473 | ||
474 | /* | |
475 | * vmsplice and splice are used to pin the user space input buffer for | |
7fa8bcfe | 476 | * kernel space processing avoiding copies from user to kernel space |
7f458a48 | 477 | */ |
478 | ret = vmsplice(actx->zc_pipe[1], &iov, 1, SPLICE_F_GIFT); | |
479 | if (ret < 0) { | |
24fa4b8d | 480 | ALG_PERR("%s(%d): vmsplice failed : ", __FILE__, __LINE__); |
7f458a48 | 481 | return 0; |
482 | } | |
483 | ||
484 | ret = splice(actx->zc_pipe[0], NULL, actx->sfd, NULL, inl, 0); | |
485 | if (ret < 0) { | |
24fa4b8d | 486 | ALG_PERR("%s(%d): splice failed : ", __FILE__, __LINE__); |
7f458a48 | 487 | return 0; |
488 | } | |
489 | # else | |
490 | msg.msg_iovlen = 1; | |
491 | msg.msg_iov = &iov; | |
492 | ||
493 | /* Sendmsg() sends iv, cipher direction and input data to the kernel */ | |
494 | sbytes = sendmsg(actx->sfd, &msg, 0); | |
495 | if (sbytes < 0) { | |
24fa4b8d BE |
496 | ALG_PERR("%s(%d): sendmsg failed for cipher operation : ", __FILE__, |
497 | __LINE__); | |
7f458a48 | 498 | return 0; |
499 | } | |
500 | ||
501 | if (sbytes != (ssize_t) inl) { | |
6cba4a66 | 502 | ALG_WARN("Cipher operation send bytes %zd != inlen %zd\n", sbytes, |
7f458a48 | 503 | inl); |
504 | return 0; | |
505 | } | |
506 | # endif | |
507 | ||
508 | return 1; | |
509 | } | |
510 | ||
511 | static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, | |
512 | const unsigned char *iv, int enc) | |
513 | { | |
514 | int ciphertype; | |
6cba4a66 | 515 | int ret; |
7f458a48 | 516 | afalg_ctx *actx; |
62cc845f | 517 | const char *ciphername; |
7f458a48 | 518 | |
519 | if (ctx == NULL || key == NULL) { | |
24fa4b8d | 520 | ALG_WARN("%s(%d): Null Parameter\n", __FILE__, __LINE__); |
7f458a48 | 521 | return 0; |
522 | } | |
523 | ||
524 | if (EVP_CIPHER_CTX_cipher(ctx) == NULL) { | |
24fa4b8d | 525 | ALG_WARN("%s(%d): Cipher object NULL\n", __FILE__, __LINE__); |
7f458a48 | 526 | return 0; |
527 | } | |
528 | ||
44ab2dfd | 529 | actx = EVP_CIPHER_CTX_get_cipher_data(ctx); |
7f458a48 | 530 | if (actx == NULL) { |
24fa4b8d | 531 | ALG_WARN("%s(%d): Cipher data NULL\n", __FILE__, __LINE__); |
7f458a48 | 532 | return 0; |
533 | } | |
534 | ||
535 | ciphertype = EVP_CIPHER_CTX_nid(ctx); | |
536 | switch (ciphertype) { | |
537 | case NID_aes_128_cbc: | |
49ea0f09 J |
538 | case NID_aes_192_cbc: |
539 | case NID_aes_256_cbc: | |
62cc845f | 540 | ciphername = "cbc(aes)"; |
7f458a48 | 541 | break; |
542 | default: | |
24fa4b8d BE |
543 | ALG_WARN("%s(%d): Unsupported Cipher type %d\n", __FILE__, __LINE__, |
544 | ciphertype); | |
7f458a48 | 545 | return 0; |
546 | } | |
547 | ||
548 | if (ALG_AES_IV_LEN != EVP_CIPHER_CTX_iv_length(ctx)) { | |
24fa4b8d BE |
549 | ALG_WARN("%s(%d): Unsupported IV length :%d\n", __FILE__, __LINE__, |
550 | EVP_CIPHER_CTX_iv_length(ctx)); | |
7f458a48 | 551 | return 0; |
552 | } | |
553 | ||
554 | /* Setup AFALG socket for crypto processing */ | |
6cba4a66 | 555 | ret = afalg_create_sk(actx, "skcipher", ciphername); |
556 | if (ret < 1) | |
7f458a48 | 557 | return 0; |
6cba4a66 | 558 | |
559 | ||
560 | ret = afalg_set_key(actx, key, EVP_CIPHER_CTX_key_length(ctx)); | |
561 | if (ret < 1) | |
562 | goto err; | |
7f458a48 | 563 | |
564 | /* Setup AIO ctx to allow async AFALG crypto processing */ | |
6cba4a66 | 565 | if (afalg_init_aio(&actx->aio) == 0) |
566 | goto err; | |
567 | ||
7f458a48 | 568 | # ifdef ALG_ZERO_COPY |
569 | pipe(actx->zc_pipe); | |
570 | # endif | |
571 | ||
572 | actx->init_done = MAGIC_INIT_NUM; | |
573 | ||
574 | return 1; | |
6cba4a66 | 575 | |
576 | err: | |
577 | close(actx->sfd); | |
578 | close(actx->bfd); | |
579 | return 0; | |
7f458a48 | 580 | } |
581 | ||
582 | static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, | |
583 | const unsigned char *in, size_t inl) | |
584 | { | |
585 | afalg_ctx *actx; | |
586 | int ret; | |
587 | char nxtiv[ALG_AES_IV_LEN] = { 0 }; | |
588 | ||
589 | if (ctx == NULL || out == NULL || in == NULL) { | |
24fa4b8d BE |
590 | ALG_WARN("NULL parameter passed to function %s(%d)\n", __FILE__, |
591 | __LINE__); | |
7f458a48 | 592 | return 0; |
593 | } | |
594 | ||
44ab2dfd | 595 | actx = (afalg_ctx *) EVP_CIPHER_CTX_get_cipher_data(ctx); |
7f458a48 | 596 | if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) { |
597 | ALG_WARN("%s afalg ctx passed\n", | |
598 | ctx == NULL ? "NULL" : "Uninitialised"); | |
599 | return 0; | |
600 | } | |
601 | ||
602 | /* | |
603 | * set iv now for decrypt operation as the input buffer can be | |
604 | * overwritten for inplace operation where in = out. | |
605 | */ | |
606 | if (EVP_CIPHER_CTX_encrypting(ctx) == 0) { | |
607 | memcpy(nxtiv, in + (inl - ALG_AES_IV_LEN), ALG_AES_IV_LEN); | |
608 | } | |
609 | ||
610 | /* Send input data to kernel space */ | |
611 | ret = afalg_start_cipher_sk(actx, (unsigned char *)in, inl, | |
612 | EVP_CIPHER_CTX_iv(ctx), | |
613 | EVP_CIPHER_CTX_encrypting(ctx)); | |
614 | if (ret < 1) { | |
615 | return 0; | |
616 | } | |
617 | ||
618 | /* Perform async crypto operation in kernel space */ | |
619 | ret = afalg_fin_cipher_aio(&actx->aio, actx->sfd, out, inl); | |
6cba4a66 | 620 | if (ret < 1) |
7f458a48 | 621 | return 0; |
7f458a48 | 622 | |
623 | if (EVP_CIPHER_CTX_encrypting(ctx)) { | |
624 | memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), out + (inl - ALG_AES_IV_LEN), | |
625 | ALG_AES_IV_LEN); | |
626 | } else { | |
627 | memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), nxtiv, ALG_AES_IV_LEN); | |
628 | } | |
629 | ||
630 | return 1; | |
631 | } | |
632 | ||
633 | static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx) | |
634 | { | |
635 | afalg_ctx *actx; | |
636 | ||
637 | if (ctx == NULL) { | |
24fa4b8d BE |
638 | ALG_WARN("NULL parameter passed to function %s(%d)\n", __FILE__, |
639 | __LINE__); | |
7f458a48 | 640 | return 0; |
641 | } | |
642 | ||
44ab2dfd | 643 | actx = (afalg_ctx *) EVP_CIPHER_CTX_get_cipher_data(ctx); |
7f458a48 | 644 | if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) { |
645 | ALG_WARN("%s afalg ctx passed\n", | |
646 | ctx == NULL ? "NULL" : "Uninitialised"); | |
647 | return 0; | |
648 | } | |
649 | ||
650 | close(actx->sfd); | |
651 | close(actx->bfd); | |
652 | # ifdef ALG_ZERO_COPY | |
653 | close(actx->zc_pipe[0]); | |
654 | close(actx->zc_pipe[1]); | |
655 | # endif | |
6cba4a66 | 656 | /* close efd in sync mode, async mode is closed in afalg_waitfd_cleanup() */ |
657 | if (actx->aio.mode == MODE_SYNC) | |
658 | close(actx->aio.efd); | |
7f458a48 | 659 | io_destroy(actx->aio.aio_ctx); |
660 | ||
661 | return 1; | |
662 | } | |
663 | ||
f1138840 | 664 | static cbc_handles *get_cipher_handle(int nid) |
49ea0f09 J |
665 | { |
666 | switch (nid) { | |
667 | case NID_aes_128_cbc: | |
668 | return &cbc_handle[AES_CBC_128]; | |
669 | case NID_aes_192_cbc: | |
670 | return &cbc_handle[AES_CBC_192]; | |
671 | case NID_aes_256_cbc: | |
672 | return &cbc_handle[AES_CBC_256]; | |
673 | default: | |
674 | return NULL; | |
675 | } | |
676 | } | |
677 | ||
3a7141df | 678 | static const EVP_CIPHER *afalg_aes_cbc(int nid) |
7f458a48 | 679 | { |
49ea0f09 J |
680 | cbc_handles *cipher_handle = get_cipher_handle(nid); |
681 | if (cipher_handle->_hidden == NULL | |
682 | && ((cipher_handle->_hidden = | |
683 | EVP_CIPHER_meth_new(nid, | |
684 | AES_BLOCK_SIZE, | |
685 | cipher_handle->key_size)) == NULL | |
686 | || !EVP_CIPHER_meth_set_iv_length(cipher_handle->_hidden, | |
687 | AES_IV_LEN) | |
688 | || !EVP_CIPHER_meth_set_flags(cipher_handle->_hidden, | |
689 | EVP_CIPH_CBC_MODE | | |
690 | EVP_CIPH_FLAG_DEFAULT_ASN1) | |
691 | || !EVP_CIPHER_meth_set_init(cipher_handle->_hidden, | |
692 | afalg_cipher_init) | |
693 | || !EVP_CIPHER_meth_set_do_cipher(cipher_handle->_hidden, | |
694 | afalg_do_cipher) | |
695 | || !EVP_CIPHER_meth_set_cleanup(cipher_handle->_hidden, | |
696 | afalg_cipher_cleanup) | |
697 | || !EVP_CIPHER_meth_set_impl_ctx_size(cipher_handle->_hidden, | |
698 | sizeof(afalg_ctx)))) { | |
699 | EVP_CIPHER_meth_free(cipher_handle->_hidden); | |
700 | cipher_handle->_hidden= NULL; | |
701 | } | |
702 | return cipher_handle->_hidden; | |
7f458a48 | 703 | } |
704 | ||
705 | static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | |
706 | const int **nids, int nid) | |
707 | { | |
708 | int r = 1; | |
709 | ||
710 | if (cipher == NULL) { | |
711 | *nids = afalg_cipher_nids; | |
712 | return (sizeof(afalg_cipher_nids) / sizeof(afalg_cipher_nids[0])); | |
713 | } | |
714 | ||
715 | switch (nid) { | |
716 | case NID_aes_128_cbc: | |
49ea0f09 J |
717 | case NID_aes_192_cbc: |
718 | case NID_aes_256_cbc: | |
719 | *cipher = afalg_aes_cbc(nid); | |
7f458a48 | 720 | break; |
721 | default: | |
722 | *cipher = NULL; | |
723 | r = 0; | |
724 | } | |
7f458a48 | 725 | return r; |
726 | } | |
727 | ||
728 | static int bind_afalg(ENGINE *e) | |
729 | { | |
730 | /* Ensure the afalg error handling is set up */ | |
49ea0f09 | 731 | unsigned short i; |
7f458a48 | 732 | ERR_load_AFALG_strings(); |
733 | ||
734 | if (!ENGINE_set_id(e, engine_afalg_id) | |
735 | || !ENGINE_set_name(e, engine_afalg_name) | |
736 | || !ENGINE_set_destroy_function(e, afalg_destroy) | |
737 | || !ENGINE_set_init_function(e, afalg_init) | |
738 | || !ENGINE_set_finish_function(e, afalg_finish)) { | |
739 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
740 | return 0; | |
741 | } | |
742 | ||
6cba4a66 | 743 | /* |
49ea0f09 | 744 | * Create _hidden_aes_xxx_cbc by calling afalg_aes_xxx_cbc |
6cba4a66 | 745 | * now, as bind_aflag can only be called by one thread at a |
746 | * time. | |
747 | */ | |
49ea0f09 J |
748 | for(i = 0; i < OSSL_NELEM(afalg_cipher_nids); i++) { |
749 | if (afalg_aes_cbc(afalg_cipher_nids[i]) == NULL) { | |
750 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
751 | return 0; | |
752 | } | |
6cba4a66 | 753 | } |
754 | ||
7f458a48 | 755 | if (!ENGINE_set_ciphers(e, afalg_ciphers)) { |
756 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
757 | return 0; | |
758 | } | |
759 | ||
760 | return 1; | |
761 | } | |
762 | ||
763 | # ifndef OPENSSL_NO_DYNAMIC_ENGINE | |
764 | static int bind_helper(ENGINE *e, const char *id) | |
765 | { | |
766 | if (id && (strcmp(id, engine_afalg_id) != 0)) | |
767 | return 0; | |
768 | ||
769 | if (!afalg_chk_platform()) | |
770 | return 0; | |
771 | ||
772 | if (!bind_afalg(e)) | |
773 | return 0; | |
774 | return 1; | |
775 | } | |
776 | ||
777 | IMPLEMENT_DYNAMIC_CHECK_FN() | |
778 | IMPLEMENT_DYNAMIC_BIND_FN(bind_helper) | |
779 | # endif | |
6cba4a66 | 780 | |
7f458a48 | 781 | static int afalg_chk_platform(void) |
782 | { | |
783 | int ret; | |
784 | int i; | |
785 | int kver[3] = { -1, -1, -1 }; | |
25b9d11c | 786 | int sock; |
7f458a48 | 787 | char *str; |
788 | struct utsname ut; | |
789 | ||
790 | ret = uname(&ut); | |
791 | if (ret != 0) { | |
6cba4a66 | 792 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, |
793 | AFALG_R_FAILED_TO_GET_PLATFORM_INFO); | |
7f458a48 | 794 | return 0; |
795 | } | |
796 | ||
797 | str = strtok(ut.release, "."); | |
798 | for (i = 0; i < 3 && str != NULL; i++) { | |
799 | kver[i] = atoi(str); | |
800 | str = strtok(NULL, "."); | |
801 | } | |
802 | ||
803 | if (KERNEL_VERSION(kver[0], kver[1], kver[2]) | |
804 | < KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2)) { | |
6cba4a66 | 805 | ALG_ERR("ASYNC AFALG not supported this kernel(%d.%d.%d)\n", |
7f458a48 | 806 | kver[0], kver[1], kver[2]); |
6cba4a66 | 807 | ALG_ERR("ASYNC AFALG requires kernel version %d.%d.%d or later\n", |
7f458a48 | 808 | K_MAJ, K_MIN1, K_MIN2); |
809 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, | |
810 | AFALG_R_KERNEL_DOES_NOT_SUPPORT_ASYNC_AFALG); | |
811 | return 0; | |
812 | } | |
813 | ||
25b9d11c MC |
814 | /* Test if we can actually create an AF_ALG socket */ |
815 | sock = socket(AF_ALG, SOCK_SEQPACKET, 0); | |
816 | if (sock == -1) { | |
817 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, AFALG_R_SOCKET_CREATE_FAILED); | |
818 | return 0; | |
819 | } | |
820 | close(sock); | |
821 | ||
7f458a48 | 822 | return 1; |
823 | } | |
824 | ||
825 | # ifdef OPENSSL_NO_DYNAMIC_ENGINE | |
826 | static ENGINE *engine_afalg(void) | |
827 | { | |
828 | ENGINE *ret = ENGINE_new(); | |
829 | if (ret == NULL) | |
830 | return NULL; | |
831 | if (!bind_afalg(ret)) { | |
832 | ENGINE_free(ret); | |
833 | return NULL; | |
834 | } | |
835 | return ret; | |
836 | } | |
837 | ||
b3599dbb | 838 | void engine_load_afalg_int(void) |
7f458a48 | 839 | { |
840 | ENGINE *toadd; | |
841 | ||
842 | if (!afalg_chk_platform()) | |
843 | return; | |
844 | ||
845 | toadd = engine_afalg(); | |
846 | if (toadd == NULL) | |
847 | return; | |
848 | ENGINE_add(toadd); | |
849 | ENGINE_free(toadd); | |
850 | ERR_clear_error(); | |
851 | } | |
852 | # endif | |
853 | ||
854 | static int afalg_init(ENGINE *e) | |
855 | { | |
856 | return 1; | |
857 | } | |
858 | ||
859 | static int afalg_finish(ENGINE *e) | |
860 | { | |
861 | return 1; | |
862 | } | |
863 | ||
49ea0f09 J |
864 | static int free_cbc(void) |
865 | { | |
a3d7fd28 | 866 | short unsigned int i; |
49ea0f09 J |
867 | for(i = 0; i < OSSL_NELEM(afalg_cipher_nids); i++) { |
868 | EVP_CIPHER_meth_free(cbc_handle[i]._hidden); | |
869 | cbc_handle[i]._hidden = NULL; | |
870 | } | |
871 | return 1; | |
872 | } | |
873 | ||
7f458a48 | 874 | static int afalg_destroy(ENGINE *e) |
875 | { | |
876 | ERR_unload_AFALG_strings(); | |
49ea0f09 | 877 | free_cbc(); |
7f458a48 | 878 | return 1; |
879 | } | |
880 | ||
881 | #endif /* KERNEL VERSION */ |