]>
Commit | Line | Data |
---|---|---|
440e5d80 | 1 | /* |
8020d79b | 2 | * Copyright 2016-2021 The OpenSSL Project Authors. All Rights Reserved. |
7f458a48 | 3 | * |
ab3fa1c0 | 4 | * Licensed under the Apache License 2.0 (the "License"). You may not use |
440e5d80 RS |
5 | * this file except in compliance with the License. You can obtain a copy |
6 | * in the file LICENSE in the source distribution or at | |
7 | * https://www.openssl.org/source/license.html | |
7f458a48 | 8 | */ |
9 | ||
77ae4f6f | 10 | /* We need to use some deprecated APIs */ |
cf8e8cba P |
11 | #define OPENSSL_SUPPRESS_DEPRECATED |
12 | ||
7f458a48 | 13 | /* Required for vmsplice */ |
4e399729 AP |
14 | #ifndef _GNU_SOURCE |
15 | # define _GNU_SOURCE | |
16 | #endif | |
7f458a48 | 17 | #include <stdio.h> |
18 | #include <string.h> | |
19 | #include <unistd.h> | |
20 | ||
21 | #include <openssl/engine.h> | |
22 | #include <openssl/async.h> | |
6bf73e53 | 23 | #include <openssl/err.h> |
49ea0f09 | 24 | #include "internal/nelem.h" |
7f458a48 | 25 | |
4e399729 | 26 | #include <sys/socket.h> |
7f458a48 | 27 | #include <linux/version.h> |
28 | #define K_MAJ 4 | |
29 | #define K_MIN1 1 | |
30 | #define K_MIN2 0 | |
3ba70235 | 31 | #if LINUX_VERSION_CODE < KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2) || \ |
4e399729 | 32 | !defined(AF_ALG) |
97043e46 RL |
33 | # ifndef PEDANTIC |
34 | # warning "AFALG ENGINE requires Kernel Headers >= 4.1.0" | |
35 | # warning "Skipping Compilation of AFALG engine" | |
36 | # endif | |
a1fd1fb2 | 37 | void engine_load_afalg_int(void); |
627537dd MC |
38 | void engine_load_afalg_int(void) |
39 | { | |
40 | } | |
7f458a48 | 41 | #else |
42 | ||
43 | # include <linux/if_alg.h> | |
7f458a48 | 44 | # include <fcntl.h> |
45 | # include <sys/utsname.h> | |
46 | ||
47 | # include <linux/aio_abi.h> | |
48 | # include <sys/syscall.h> | |
49 | # include <errno.h> | |
50 | ||
51 | # include "e_afalg.h" | |
52df25cf | 52 | # include "e_afalg_err.c" |
7f458a48 | 53 | |
54 | # ifndef SOL_ALG | |
55 | # define SOL_ALG 279 | |
56 | # endif | |
57 | ||
58 | # ifdef ALG_ZERO_COPY | |
59 | # ifndef SPLICE_F_GIFT | |
60 | # define SPLICE_F_GIFT (0x08) | |
61 | # endif | |
62 | # endif | |
63 | ||
64 | # define ALG_AES_IV_LEN 16 | |
65 | # define ALG_IV_LEN(len) (sizeof(struct af_alg_iv) + (len)) | |
66 | # define ALG_OP_TYPE unsigned int | |
67 | # define ALG_OP_LEN (sizeof(ALG_OP_TYPE)) | |
68 | ||
6cba4a66 | 69 | # ifdef OPENSSL_NO_DYNAMIC_ENGINE |
b3599dbb | 70 | void engine_load_afalg_int(void); |
6cba4a66 | 71 | # endif |
72 | ||
7f458a48 | 73 | /* Local Linkage Functions */ |
74 | static int afalg_init_aio(afalg_aio *aio); | |
75 | static int afalg_fin_cipher_aio(afalg_aio *ptr, int sfd, | |
76 | unsigned char *buf, size_t len); | |
6cba4a66 | 77 | static int afalg_create_sk(afalg_ctx *actx, const char *ciphertype, |
78 | const char *ciphername); | |
7f458a48 | 79 | static int afalg_destroy(ENGINE *e); |
80 | static int afalg_init(ENGINE *e); | |
81 | static int afalg_finish(ENGINE *e); | |
3a7141df | 82 | static const EVP_CIPHER *afalg_aes_cbc(int nid); |
7e8a5e30 | 83 | static cbc_handles *get_cipher_handle(int nid); |
7f458a48 | 84 | static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher, |
85 | const int **nids, int nid); | |
86 | static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, | |
87 | const unsigned char *iv, int enc); | |
88 | static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, | |
89 | const unsigned char *in, size_t inl); | |
90 | static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx); | |
91 | static int afalg_chk_platform(void); | |
92 | ||
93 | /* Engine Id and Name */ | |
94 | static const char *engine_afalg_id = "afalg"; | |
538dbbc6 | 95 | static const char *engine_afalg_name = "AFALG engine support"; |
7f458a48 | 96 | |
a1933888 | 97 | static int afalg_cipher_nids[] = { |
49ea0f09 J |
98 | NID_aes_128_cbc, |
99 | NID_aes_192_cbc, | |
100 | NID_aes_256_cbc, | |
7f458a48 | 101 | }; |
102 | ||
49ea0f09 J |
103 | static cbc_handles cbc_handle[] = {{AES_KEY_SIZE_128, NULL}, |
104 | {AES_KEY_SIZE_192, NULL}, | |
105 | {AES_KEY_SIZE_256, NULL}}; | |
7f458a48 | 106 | |
2a7de0fd | 107 | static ossl_inline int io_setup(unsigned n, aio_context_t *ctx) |
7f458a48 | 108 | { |
109 | return syscall(__NR_io_setup, n, ctx); | |
110 | } | |
111 | ||
2a7de0fd | 112 | static ossl_inline int eventfd(int n) |
7f458a48 | 113 | { |
bee9c8a4 | 114 | return syscall(__NR_eventfd2, n, 0); |
7f458a48 | 115 | } |
116 | ||
2a7de0fd | 117 | static ossl_inline int io_destroy(aio_context_t ctx) |
7f458a48 | 118 | { |
119 | return syscall(__NR_io_destroy, ctx); | |
120 | } | |
121 | ||
2a7de0fd | 122 | static ossl_inline int io_read(aio_context_t ctx, long n, struct iocb **iocb) |
7f458a48 | 123 | { |
124 | return syscall(__NR_io_submit, ctx, n, iocb); | |
125 | } | |
126 | ||
e5499a3c AF |
127 | /* A version of 'struct timespec' with 32-bit time_t and nanoseconds. */ |
128 | struct __timespec32 | |
129 | { | |
130 | __kernel_long_t tv_sec; | |
131 | __kernel_long_t tv_nsec; | |
132 | }; | |
133 | ||
2a7de0fd | 134 | static ossl_inline int io_getevents(aio_context_t ctx, long min, long max, |
7f458a48 | 135 | struct io_event *events, |
136 | struct timespec *timeout) | |
137 | { | |
e5499a3c AF |
138 | #if defined(__NR_io_pgetevents_time64) |
139 | /* Check if we are a 32-bit architecture with a 64-bit time_t */ | |
140 | if (sizeof(*timeout) != sizeof(struct __timespec32)) { | |
141 | int ret = syscall(__NR_io_pgetevents_time64, ctx, min, max, events, | |
142 | timeout, NULL); | |
143 | if (ret == 0 || errno != ENOSYS) | |
144 | return ret; | |
145 | } | |
146 | #endif | |
147 | ||
5b5e2985 | 148 | #if defined(__NR_io_getevents) |
e5499a3c AF |
149 | if (sizeof(*timeout) == sizeof(struct __timespec32)) |
150 | /* | |
151 | * time_t matches our architecture length, we can just use | |
152 | * __NR_io_getevents | |
153 | */ | |
154 | return syscall(__NR_io_getevents, ctx, min, max, events, timeout); | |
5b5e2985 | 155 | else { |
e5499a3c AF |
156 | /* |
157 | * We don't have __NR_io_pgetevents_time64, but we are using a | |
158 | * 64-bit time_t on a 32-bit architecture. If we can fit the | |
159 | * timeout value in a 32-bit time_t, then let's do that | |
160 | * and then use the __NR_io_getevents syscall. | |
161 | */ | |
162 | if (timeout && timeout->tv_sec == (long)timeout->tv_sec) { | |
163 | struct __timespec32 ts32; | |
164 | ||
165 | ts32.tv_sec = (__kernel_long_t) timeout->tv_sec; | |
166 | ts32.tv_nsec = (__kernel_long_t) timeout->tv_nsec; | |
167 | ||
168 | return syscall(__NR_io_getevents, ctx, min, max, events, ts32); | |
169 | } else { | |
170 | return syscall(__NR_io_getevents, ctx, min, max, events, NULL); | |
171 | } | |
5b5e2985 | 172 | } |
5b5e2985 | 173 | #endif |
e5499a3c AF |
174 | |
175 | errno = ENOSYS; | |
176 | return -1; | |
7f458a48 | 177 | } |
178 | ||
179 | static void afalg_waitfd_cleanup(ASYNC_WAIT_CTX *ctx, const void *key, | |
180 | OSSL_ASYNC_FD waitfd, void *custom) | |
181 | { | |
182 | close(waitfd); | |
183 | } | |
184 | ||
185 | static int afalg_setup_async_event_notification(afalg_aio *aio) | |
186 | { | |
187 | ASYNC_JOB *job; | |
188 | ASYNC_WAIT_CTX *waitctx; | |
189 | void *custom = NULL; | |
6cba4a66 | 190 | int ret; |
7f458a48 | 191 | |
192 | if ((job = ASYNC_get_current_job()) != NULL) { | |
193 | /* Async mode */ | |
194 | waitctx = ASYNC_get_wait_ctx(job); | |
195 | if (waitctx == NULL) { | |
24fa4b8d | 196 | ALG_WARN("%s(%d): ASYNC_get_wait_ctx error", __FILE__, __LINE__); |
7f458a48 | 197 | return 0; |
198 | } | |
46f4e1be | 199 | /* Get waitfd from ASYNC_WAIT_CTX if it is already set */ |
6cba4a66 | 200 | ret = ASYNC_WAIT_CTX_get_fd(waitctx, engine_afalg_id, |
201 | &aio->efd, &custom); | |
202 | if (ret == 0) { | |
203 | /* | |
204 | * waitfd is not set in ASYNC_WAIT_CTX, create a new one | |
205 | * and set it. efd will be signaled when AIO operation completes | |
206 | */ | |
207 | aio->efd = eventfd(0); | |
208 | if (aio->efd == -1) { | |
24fa4b8d BE |
209 | ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__, |
210 | __LINE__); | |
7f458a48 | 211 | AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION, |
212 | AFALG_R_EVENTFD_FAILED); | |
213 | return 0; | |
214 | } | |
6cba4a66 | 215 | ret = ASYNC_WAIT_CTX_set_wait_fd(waitctx, engine_afalg_id, |
216 | aio->efd, custom, | |
217 | afalg_waitfd_cleanup); | |
218 | if (ret == 0) { | |
24fa4b8d | 219 | ALG_WARN("%s(%d): Failed to set wait fd", __FILE__, __LINE__); |
6cba4a66 | 220 | close(aio->efd); |
7f458a48 | 221 | return 0; |
222 | } | |
223 | /* make fd non-blocking in async mode */ | |
6cba4a66 | 224 | if (fcntl(aio->efd, F_SETFL, O_NONBLOCK) != 0) { |
24fa4b8d BE |
225 | ALG_WARN("%s(%d): Failed to set event fd as NONBLOCKING", |
226 | __FILE__, __LINE__); | |
7f458a48 | 227 | } |
228 | } | |
6cba4a66 | 229 | aio->mode = MODE_ASYNC; |
7f458a48 | 230 | } else { |
231 | /* Sync mode */ | |
6cba4a66 | 232 | aio->efd = eventfd(0); |
233 | if (aio->efd == -1) { | |
24fa4b8d | 234 | ALG_PERR("%s(%d): Failed to get eventfd : ", __FILE__, __LINE__); |
7f458a48 | 235 | AFALGerr(AFALG_F_AFALG_SETUP_ASYNC_EVENT_NOTIFICATION, |
236 | AFALG_R_EVENTFD_FAILED); | |
237 | return 0; | |
238 | } | |
6cba4a66 | 239 | aio->mode = MODE_SYNC; |
7f458a48 | 240 | } |
241 | return 1; | |
242 | } | |
243 | ||
3a7141df | 244 | static int afalg_init_aio(afalg_aio *aio) |
7f458a48 | 245 | { |
246 | int r = -1; | |
247 | ||
248 | /* Initialise for AIO */ | |
249 | aio->aio_ctx = 0; | |
250 | r = io_setup(MAX_INFLIGHTS, &aio->aio_ctx); | |
251 | if (r < 0) { | |
24fa4b8d | 252 | ALG_PERR("%s(%d): io_setup error : ", __FILE__, __LINE__); |
7f458a48 | 253 | AFALGerr(AFALG_F_AFALG_INIT_AIO, AFALG_R_IO_SETUP_FAILED); |
254 | return 0; | |
255 | } | |
256 | ||
257 | memset(aio->cbt, 0, sizeof(aio->cbt)); | |
7f458a48 | 258 | aio->efd = -1; |
6cba4a66 | 259 | aio->mode = MODE_UNINIT; |
7f458a48 | 260 | |
261 | return 1; | |
262 | } | |
263 | ||
3a7141df RL |
264 | static int afalg_fin_cipher_aio(afalg_aio *aio, int sfd, unsigned char *buf, |
265 | size_t len) | |
7f458a48 | 266 | { |
267 | int r; | |
268 | int retry = 0; | |
269 | unsigned int done = 0; | |
270 | struct iocb *cb; | |
271 | struct timespec timeout; | |
272 | struct io_event events[MAX_INFLIGHTS]; | |
273 | u_int64_t eval = 0; | |
274 | ||
275 | timeout.tv_sec = 0; | |
276 | timeout.tv_nsec = 0; | |
277 | ||
278 | /* if efd has not been initialised yet do it here */ | |
6cba4a66 | 279 | if (aio->mode == MODE_UNINIT) { |
7f458a48 | 280 | r = afalg_setup_async_event_notification(aio); |
281 | if (r == 0) | |
282 | return 0; | |
283 | } | |
284 | ||
285 | cb = &(aio->cbt[0 % MAX_INFLIGHTS]); | |
286 | memset(cb, '\0', sizeof(*cb)); | |
287 | cb->aio_fildes = sfd; | |
288 | cb->aio_lio_opcode = IOCB_CMD_PREAD; | |
c27778d8 AP |
289 | /* |
290 | * The pointer has to be converted to unsigned value first to avoid | |
291 | * sign extension on cast to 64 bit value in 32-bit builds | |
292 | */ | |
293 | cb->aio_buf = (size_t)buf; | |
7f458a48 | 294 | cb->aio_offset = 0; |
295 | cb->aio_data = 0; | |
296 | cb->aio_nbytes = len; | |
297 | cb->aio_flags = IOCB_FLAG_RESFD; | |
298 | cb->aio_resfd = aio->efd; | |
299 | ||
300 | /* | |
301 | * Perform AIO read on AFALG socket, this in turn performs an async | |
302 | * crypto operation in kernel space | |
303 | */ | |
304 | r = io_read(aio->aio_ctx, 1, &cb); | |
305 | if (r < 0) { | |
24fa4b8d | 306 | ALG_PWARN("%s(%d): io_read failed : ", __FILE__, __LINE__); |
7f458a48 | 307 | return 0; |
308 | } | |
309 | ||
310 | do { | |
311 | /* While AIO read is being performed pause job */ | |
312 | ASYNC_pause_job(); | |
313 | ||
314 | /* Check for completion of AIO read */ | |
315 | r = read(aio->efd, &eval, sizeof(eval)); | |
316 | if (r < 0) { | |
317 | if (errno == EAGAIN || errno == EWOULDBLOCK) | |
318 | continue; | |
24fa4b8d | 319 | ALG_PERR("%s(%d): read failed for event fd : ", __FILE__, __LINE__); |
7f458a48 | 320 | return 0; |
321 | } else if (r == 0 || eval <= 0) { | |
24fa4b8d BE |
322 | ALG_WARN("%s(%d): eventfd read %d bytes, eval = %lu\n", __FILE__, |
323 | __LINE__, r, eval); | |
7f458a48 | 324 | } |
325 | if (eval > 0) { | |
326 | ||
514b7691 P |
327 | #ifdef OSSL_SANITIZE_MEMORY |
328 | /* | |
329 | * In a memory sanitiser build, the changes to memory made by the | |
330 | * system call aren't reliably detected. By initialising the | |
331 | * memory here, the sanitiser is told that they are okay. | |
332 | */ | |
333 | memset(events, 0, sizeof(events)); | |
334 | #endif | |
335 | ||
7f458a48 | 336 | /* Get results of AIO read */ |
6cba4a66 | 337 | r = io_getevents(aio->aio_ctx, 1, MAX_INFLIGHTS, |
338 | events, &timeout); | |
7f458a48 | 339 | if (r > 0) { |
340 | /* | |
341 | * events.res indicates the actual status of the operation. | |
342 | * Handle the error condition first. | |
343 | */ | |
344 | if (events[0].res < 0) { | |
345 | /* | |
346 | * Underlying operation cannot be completed at the time | |
347 | * of previous submission. Resubmit for the operation. | |
348 | */ | |
349 | if (events[0].res == -EBUSY && retry++ < 3) { | |
350 | r = io_read(aio->aio_ctx, 1, &cb); | |
351 | if (r < 0) { | |
24fa4b8d BE |
352 | ALG_PERR("%s(%d): retry %d for io_read failed : ", |
353 | __FILE__, __LINE__, retry); | |
7f458a48 | 354 | return 0; |
355 | } | |
356 | continue; | |
357 | } else { | |
bd19999b | 358 | char strbuf[32]; |
359 | /* | |
360 | * sometimes __s64 is defined as long long int | |
361 | * but on some archs ( like mips64 or powerpc64 ) it's just long int | |
362 | * | |
363 | * to be able to use BIO_snprintf() with %lld without warnings | |
364 | * copy events[0].res to an long long int variable | |
365 | * | |
366 | * because long long int should always be at least 64 bit this should work | |
367 | */ | |
368 | long long int op_ret = events[0].res; | |
369 | ||
7f458a48 | 370 | /* |
371 | * Retries exceed for -EBUSY or unrecoverable error | |
372 | * condition for this instance of operation. | |
373 | */ | |
374 | ALG_WARN | |
24fa4b8d BE |
375 | ("%s(%d): Crypto Operation failed with code %lld\n", |
376 | __FILE__, __LINE__, events[0].res); | |
bd19999b | 377 | BIO_snprintf(strbuf, sizeof(strbuf), "%lld", op_ret); |
378 | switch (events[0].res) { | |
379 | case -ENOMEM: | |
380 | AFALGerr(0, AFALG_R_KERNEL_OP_FAILED); | |
381 | ERR_add_error_data(3, "-ENOMEM ( code ", strbuf, " )"); | |
382 | break; | |
383 | default: | |
384 | AFALGerr(0, AFALG_R_KERNEL_OP_FAILED); | |
385 | ERR_add_error_data(2, "code ", strbuf); | |
386 | break; | |
387 | } | |
7f458a48 | 388 | return 0; |
389 | } | |
390 | } | |
391 | /* Operation successful. */ | |
392 | done = 1; | |
393 | } else if (r < 0) { | |
24fa4b8d | 394 | ALG_PERR("%s(%d): io_getevents failed : ", __FILE__, __LINE__); |
7f458a48 | 395 | return 0; |
396 | } else { | |
24fa4b8d BE |
397 | ALG_WARN("%s(%d): io_geteventd read 0 bytes\n", __FILE__, |
398 | __LINE__); | |
7f458a48 | 399 | } |
400 | } | |
401 | } while (!done); | |
402 | ||
403 | return 1; | |
404 | } | |
405 | ||
2a7de0fd | 406 | static ossl_inline void afalg_set_op_sk(struct cmsghdr *cmsg, |
574cffd5 | 407 | const ALG_OP_TYPE op) |
7f458a48 | 408 | { |
409 | cmsg->cmsg_level = SOL_ALG; | |
410 | cmsg->cmsg_type = ALG_SET_OP; | |
411 | cmsg->cmsg_len = CMSG_LEN(ALG_OP_LEN); | |
574cffd5 | 412 | memcpy(CMSG_DATA(cmsg), &op, ALG_OP_LEN); |
7f458a48 | 413 | } |
414 | ||
415 | static void afalg_set_iv_sk(struct cmsghdr *cmsg, const unsigned char *iv, | |
416 | const unsigned int len) | |
417 | { | |
418 | struct af_alg_iv *aiv; | |
419 | ||
420 | cmsg->cmsg_level = SOL_ALG; | |
421 | cmsg->cmsg_type = ALG_SET_IV; | |
422 | cmsg->cmsg_len = CMSG_LEN(ALG_IV_LEN(len)); | |
423 | aiv = (struct af_alg_iv *)CMSG_DATA(cmsg); | |
424 | aiv->ivlen = len; | |
425 | memcpy(aiv->iv, iv, len); | |
426 | } | |
427 | ||
2a7de0fd | 428 | static ossl_inline int afalg_set_key(afalg_ctx *actx, const unsigned char *key, |
6cba4a66 | 429 | const int klen) |
7f458a48 | 430 | { |
431 | int ret; | |
6cba4a66 | 432 | ret = setsockopt(actx->bfd, SOL_ALG, ALG_SET_KEY, key, klen); |
433 | if (ret < 0) { | |
24fa4b8d | 434 | ALG_PERR("%s(%d): Failed to set socket option : ", __FILE__, __LINE__); |
6cba4a66 | 435 | AFALGerr(AFALG_F_AFALG_SET_KEY, AFALG_R_SOCKET_SET_KEY_FAILED); |
436 | return 0; | |
437 | } | |
6cba4a66 | 438 | return 1; |
439 | } | |
440 | ||
441 | static int afalg_create_sk(afalg_ctx *actx, const char *ciphertype, | |
442 | const char *ciphername) | |
443 | { | |
444 | struct sockaddr_alg sa; | |
c27778d8 | 445 | int r = -1; |
7f458a48 | 446 | |
447 | actx->bfd = actx->sfd = -1; | |
448 | ||
6cba4a66 | 449 | memset(&sa, 0, sizeof(sa)); |
450 | sa.salg_family = AF_ALG; | |
62cc845f BE |
451 | OPENSSL_strlcpy((char *) sa.salg_type, ciphertype, sizeof(sa.salg_type)); |
452 | OPENSSL_strlcpy((char *) sa.salg_name, ciphername, sizeof(sa.salg_name)); | |
6cba4a66 | 453 | |
454 | actx->bfd = socket(AF_ALG, SOCK_SEQPACKET, 0); | |
455 | if (actx->bfd == -1) { | |
24fa4b8d | 456 | ALG_PERR("%s(%d): Failed to open socket : ", __FILE__, __LINE__); |
6cba4a66 | 457 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_CREATE_FAILED); |
458 | goto err; | |
7f458a48 | 459 | } |
460 | ||
6cba4a66 | 461 | r = bind(actx->bfd, (struct sockaddr *)&sa, sizeof(sa)); |
462 | if (r < 0) { | |
24fa4b8d | 463 | ALG_PERR("%s(%d): Failed to bind socket : ", __FILE__, __LINE__); |
6cba4a66 | 464 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_BIND_FAILED); |
7f458a48 | 465 | goto err; |
466 | } | |
467 | ||
7f458a48 | 468 | actx->sfd = accept(actx->bfd, NULL, 0); |
469 | if (actx->sfd < 0) { | |
24fa4b8d | 470 | ALG_PERR("%s(%d): Socket Accept Failed : ", __FILE__, __LINE__); |
6cba4a66 | 471 | AFALGerr(AFALG_F_AFALG_CREATE_SK, AFALG_R_SOCKET_ACCEPT_FAILED); |
7f458a48 | 472 | goto err; |
473 | } | |
474 | ||
6cba4a66 | 475 | return 1; |
7f458a48 | 476 | |
477 | err: | |
478 | if (actx->bfd >= 0) | |
479 | close(actx->bfd); | |
480 | if (actx->sfd >= 0) | |
481 | close(actx->sfd); | |
482 | actx->bfd = actx->sfd = -1; | |
6cba4a66 | 483 | return 0; |
7f458a48 | 484 | } |
485 | ||
486 | static int afalg_start_cipher_sk(afalg_ctx *actx, const unsigned char *in, | |
487 | size_t inl, const unsigned char *iv, | |
488 | unsigned int enc) | |
489 | { | |
64a45882 | 490 | struct msghdr msg; |
7f458a48 | 491 | struct cmsghdr *cmsg; |
492 | struct iovec iov; | |
493 | ssize_t sbytes; | |
494 | # ifdef ALG_ZERO_COPY | |
495 | int ret; | |
496 | # endif | |
a1933888 | 497 | char cbuf[CMSG_SPACE(ALG_IV_LEN(ALG_AES_IV_LEN)) + CMSG_SPACE(ALG_OP_LEN)]; |
7f458a48 | 498 | |
64a45882 | 499 | memset(&msg, 0, sizeof(msg)); |
a1933888 | 500 | memset(cbuf, 0, sizeof(cbuf)); |
7f458a48 | 501 | msg.msg_control = cbuf; |
a1933888 | 502 | msg.msg_controllen = sizeof(cbuf); |
7f458a48 | 503 | |
504 | /* | |
505 | * cipher direction (i.e. encrypt or decrypt) and iv are sent to the | |
506 | * kernel as part of sendmsg()'s ancillary data | |
507 | */ | |
508 | cmsg = CMSG_FIRSTHDR(&msg); | |
509 | afalg_set_op_sk(cmsg, enc); | |
510 | cmsg = CMSG_NXTHDR(&msg, cmsg); | |
511 | afalg_set_iv_sk(cmsg, iv, ALG_AES_IV_LEN); | |
512 | ||
513 | /* iov that describes input data */ | |
514 | iov.iov_base = (unsigned char *)in; | |
515 | iov.iov_len = inl; | |
516 | ||
517 | msg.msg_flags = MSG_MORE; | |
518 | ||
519 | # ifdef ALG_ZERO_COPY | |
520 | /* | |
521 | * ZERO_COPY mode | |
522 | * Works best when buffer is 4k aligned | |
523 | * OPENS: out of place processing (i.e. out != in) | |
524 | */ | |
525 | ||
526 | /* Input data is not sent as part of call to sendmsg() */ | |
527 | msg.msg_iovlen = 0; | |
528 | msg.msg_iov = NULL; | |
529 | ||
530 | /* Sendmsg() sends iv and cipher direction to the kernel */ | |
531 | sbytes = sendmsg(actx->sfd, &msg, 0); | |
532 | if (sbytes < 0) { | |
24fa4b8d BE |
533 | ALG_PERR("%s(%d): sendmsg failed for zero copy cipher operation : ", |
534 | __FILE__, __LINE__); | |
7f458a48 | 535 | return 0; |
536 | } | |
537 | ||
538 | /* | |
539 | * vmsplice and splice are used to pin the user space input buffer for | |
7fa8bcfe | 540 | * kernel space processing avoiding copies from user to kernel space |
7f458a48 | 541 | */ |
542 | ret = vmsplice(actx->zc_pipe[1], &iov, 1, SPLICE_F_GIFT); | |
543 | if (ret < 0) { | |
24fa4b8d | 544 | ALG_PERR("%s(%d): vmsplice failed : ", __FILE__, __LINE__); |
7f458a48 | 545 | return 0; |
546 | } | |
547 | ||
548 | ret = splice(actx->zc_pipe[0], NULL, actx->sfd, NULL, inl, 0); | |
549 | if (ret < 0) { | |
24fa4b8d | 550 | ALG_PERR("%s(%d): splice failed : ", __FILE__, __LINE__); |
7f458a48 | 551 | return 0; |
552 | } | |
553 | # else | |
554 | msg.msg_iovlen = 1; | |
555 | msg.msg_iov = &iov; | |
556 | ||
557 | /* Sendmsg() sends iv, cipher direction and input data to the kernel */ | |
558 | sbytes = sendmsg(actx->sfd, &msg, 0); | |
559 | if (sbytes < 0) { | |
24fa4b8d BE |
560 | ALG_PERR("%s(%d): sendmsg failed for cipher operation : ", __FILE__, |
561 | __LINE__); | |
7f458a48 | 562 | return 0; |
563 | } | |
564 | ||
565 | if (sbytes != (ssize_t) inl) { | |
6cba4a66 | 566 | ALG_WARN("Cipher operation send bytes %zd != inlen %zd\n", sbytes, |
7f458a48 | 567 | inl); |
568 | return 0; | |
569 | } | |
570 | # endif | |
571 | ||
572 | return 1; | |
573 | } | |
574 | ||
575 | static int afalg_cipher_init(EVP_CIPHER_CTX *ctx, const unsigned char *key, | |
576 | const unsigned char *iv, int enc) | |
577 | { | |
578 | int ciphertype; | |
b0a0ab07 | 579 | int ret, len; |
7f458a48 | 580 | afalg_ctx *actx; |
62cc845f | 581 | const char *ciphername; |
7f458a48 | 582 | |
583 | if (ctx == NULL || key == NULL) { | |
24fa4b8d | 584 | ALG_WARN("%s(%d): Null Parameter\n", __FILE__, __LINE__); |
7f458a48 | 585 | return 0; |
586 | } | |
587 | ||
f6c95e46 | 588 | if (EVP_CIPHER_CTX_get0_cipher(ctx) == NULL) { |
24fa4b8d | 589 | ALG_WARN("%s(%d): Cipher object NULL\n", __FILE__, __LINE__); |
7f458a48 | 590 | return 0; |
591 | } | |
592 | ||
44ab2dfd | 593 | actx = EVP_CIPHER_CTX_get_cipher_data(ctx); |
7f458a48 | 594 | if (actx == NULL) { |
24fa4b8d | 595 | ALG_WARN("%s(%d): Cipher data NULL\n", __FILE__, __LINE__); |
7f458a48 | 596 | return 0; |
597 | } | |
598 | ||
ed576acd | 599 | ciphertype = EVP_CIPHER_CTX_get_nid(ctx); |
7f458a48 | 600 | switch (ciphertype) { |
601 | case NID_aes_128_cbc: | |
49ea0f09 J |
602 | case NID_aes_192_cbc: |
603 | case NID_aes_256_cbc: | |
62cc845f | 604 | ciphername = "cbc(aes)"; |
7f458a48 | 605 | break; |
606 | default: | |
24fa4b8d BE |
607 | ALG_WARN("%s(%d): Unsupported Cipher type %d\n", __FILE__, __LINE__, |
608 | ciphertype); | |
7f458a48 | 609 | return 0; |
610 | } | |
611 | ||
ed576acd | 612 | if (ALG_AES_IV_LEN != EVP_CIPHER_CTX_get_iv_length(ctx)) { |
24fa4b8d | 613 | ALG_WARN("%s(%d): Unsupported IV length :%d\n", __FILE__, __LINE__, |
ed576acd | 614 | EVP_CIPHER_CTX_get_iv_length(ctx)); |
7f458a48 | 615 | return 0; |
616 | } | |
617 | ||
618 | /* Setup AFALG socket for crypto processing */ | |
6cba4a66 | 619 | ret = afalg_create_sk(actx, "skcipher", ciphername); |
620 | if (ret < 1) | |
7f458a48 | 621 | return 0; |
6cba4a66 | 622 | |
b0a0ab07 P |
623 | if ((len = EVP_CIPHER_CTX_get_key_length(ctx)) <= 0) |
624 | goto err; | |
625 | ret = afalg_set_key(actx, key, len); | |
6cba4a66 | 626 | if (ret < 1) |
627 | goto err; | |
7f458a48 | 628 | |
629 | /* Setup AIO ctx to allow async AFALG crypto processing */ | |
6cba4a66 | 630 | if (afalg_init_aio(&actx->aio) == 0) |
631 | goto err; | |
632 | ||
7f458a48 | 633 | # ifdef ALG_ZERO_COPY |
634 | pipe(actx->zc_pipe); | |
635 | # endif | |
636 | ||
637 | actx->init_done = MAGIC_INIT_NUM; | |
638 | ||
639 | return 1; | |
6cba4a66 | 640 | |
641 | err: | |
642 | close(actx->sfd); | |
643 | close(actx->bfd); | |
644 | return 0; | |
7f458a48 | 645 | } |
646 | ||
647 | static int afalg_do_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out, | |
648 | const unsigned char *in, size_t inl) | |
649 | { | |
650 | afalg_ctx *actx; | |
651 | int ret; | |
652 | char nxtiv[ALG_AES_IV_LEN] = { 0 }; | |
653 | ||
654 | if (ctx == NULL || out == NULL || in == NULL) { | |
24fa4b8d BE |
655 | ALG_WARN("NULL parameter passed to function %s(%d)\n", __FILE__, |
656 | __LINE__); | |
7f458a48 | 657 | return 0; |
658 | } | |
659 | ||
44ab2dfd | 660 | actx = (afalg_ctx *) EVP_CIPHER_CTX_get_cipher_data(ctx); |
7f458a48 | 661 | if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) { |
662 | ALG_WARN("%s afalg ctx passed\n", | |
663 | ctx == NULL ? "NULL" : "Uninitialised"); | |
664 | return 0; | |
665 | } | |
666 | ||
667 | /* | |
668 | * set iv now for decrypt operation as the input buffer can be | |
669 | * overwritten for inplace operation where in = out. | |
670 | */ | |
ed576acd | 671 | if (EVP_CIPHER_CTX_is_encrypting(ctx) == 0) { |
7f458a48 | 672 | memcpy(nxtiv, in + (inl - ALG_AES_IV_LEN), ALG_AES_IV_LEN); |
673 | } | |
674 | ||
675 | /* Send input data to kernel space */ | |
676 | ret = afalg_start_cipher_sk(actx, (unsigned char *)in, inl, | |
677 | EVP_CIPHER_CTX_iv(ctx), | |
ed576acd | 678 | EVP_CIPHER_CTX_is_encrypting(ctx)); |
7f458a48 | 679 | if (ret < 1) { |
680 | return 0; | |
681 | } | |
682 | ||
683 | /* Perform async crypto operation in kernel space */ | |
684 | ret = afalg_fin_cipher_aio(&actx->aio, actx->sfd, out, inl); | |
6cba4a66 | 685 | if (ret < 1) |
7f458a48 | 686 | return 0; |
7f458a48 | 687 | |
ed576acd | 688 | if (EVP_CIPHER_CTX_is_encrypting(ctx)) { |
7f458a48 | 689 | memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), out + (inl - ALG_AES_IV_LEN), |
690 | ALG_AES_IV_LEN); | |
691 | } else { | |
692 | memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), nxtiv, ALG_AES_IV_LEN); | |
693 | } | |
694 | ||
695 | return 1; | |
696 | } | |
697 | ||
698 | static int afalg_cipher_cleanup(EVP_CIPHER_CTX *ctx) | |
699 | { | |
700 | afalg_ctx *actx; | |
701 | ||
702 | if (ctx == NULL) { | |
24fa4b8d BE |
703 | ALG_WARN("NULL parameter passed to function %s(%d)\n", __FILE__, |
704 | __LINE__); | |
7f458a48 | 705 | return 0; |
706 | } | |
707 | ||
44ab2dfd | 708 | actx = (afalg_ctx *) EVP_CIPHER_CTX_get_cipher_data(ctx); |
6f6a5e0c BE |
709 | if (actx == NULL || actx->init_done != MAGIC_INIT_NUM) |
710 | return 1; | |
7f458a48 | 711 | |
712 | close(actx->sfd); | |
713 | close(actx->bfd); | |
714 | # ifdef ALG_ZERO_COPY | |
715 | close(actx->zc_pipe[0]); | |
716 | close(actx->zc_pipe[1]); | |
717 | # endif | |
6cba4a66 | 718 | /* close efd in sync mode, async mode is closed in afalg_waitfd_cleanup() */ |
719 | if (actx->aio.mode == MODE_SYNC) | |
720 | close(actx->aio.efd); | |
7f458a48 | 721 | io_destroy(actx->aio.aio_ctx); |
722 | ||
723 | return 1; | |
724 | } | |
725 | ||
f1138840 | 726 | static cbc_handles *get_cipher_handle(int nid) |
49ea0f09 J |
727 | { |
728 | switch (nid) { | |
729 | case NID_aes_128_cbc: | |
730 | return &cbc_handle[AES_CBC_128]; | |
731 | case NID_aes_192_cbc: | |
732 | return &cbc_handle[AES_CBC_192]; | |
733 | case NID_aes_256_cbc: | |
734 | return &cbc_handle[AES_CBC_256]; | |
735 | default: | |
736 | return NULL; | |
737 | } | |
738 | } | |
739 | ||
3a7141df | 740 | static const EVP_CIPHER *afalg_aes_cbc(int nid) |
7f458a48 | 741 | { |
49ea0f09 | 742 | cbc_handles *cipher_handle = get_cipher_handle(nid); |
728d03b5 P |
743 | |
744 | if (cipher_handle == NULL) | |
745 | return NULL; | |
49ea0f09 J |
746 | if (cipher_handle->_hidden == NULL |
747 | && ((cipher_handle->_hidden = | |
748 | EVP_CIPHER_meth_new(nid, | |
749 | AES_BLOCK_SIZE, | |
750 | cipher_handle->key_size)) == NULL | |
751 | || !EVP_CIPHER_meth_set_iv_length(cipher_handle->_hidden, | |
752 | AES_IV_LEN) | |
753 | || !EVP_CIPHER_meth_set_flags(cipher_handle->_hidden, | |
754 | EVP_CIPH_CBC_MODE | | |
755 | EVP_CIPH_FLAG_DEFAULT_ASN1) | |
756 | || !EVP_CIPHER_meth_set_init(cipher_handle->_hidden, | |
757 | afalg_cipher_init) | |
758 | || !EVP_CIPHER_meth_set_do_cipher(cipher_handle->_hidden, | |
759 | afalg_do_cipher) | |
760 | || !EVP_CIPHER_meth_set_cleanup(cipher_handle->_hidden, | |
761 | afalg_cipher_cleanup) | |
762 | || !EVP_CIPHER_meth_set_impl_ctx_size(cipher_handle->_hidden, | |
763 | sizeof(afalg_ctx)))) { | |
764 | EVP_CIPHER_meth_free(cipher_handle->_hidden); | |
765 | cipher_handle->_hidden= NULL; | |
766 | } | |
767 | return cipher_handle->_hidden; | |
7f458a48 | 768 | } |
769 | ||
770 | static int afalg_ciphers(ENGINE *e, const EVP_CIPHER **cipher, | |
771 | const int **nids, int nid) | |
772 | { | |
773 | int r = 1; | |
774 | ||
775 | if (cipher == NULL) { | |
776 | *nids = afalg_cipher_nids; | |
777 | return (sizeof(afalg_cipher_nids) / sizeof(afalg_cipher_nids[0])); | |
778 | } | |
779 | ||
780 | switch (nid) { | |
781 | case NID_aes_128_cbc: | |
49ea0f09 J |
782 | case NID_aes_192_cbc: |
783 | case NID_aes_256_cbc: | |
784 | *cipher = afalg_aes_cbc(nid); | |
7f458a48 | 785 | break; |
786 | default: | |
787 | *cipher = NULL; | |
788 | r = 0; | |
789 | } | |
7f458a48 | 790 | return r; |
791 | } | |
792 | ||
793 | static int bind_afalg(ENGINE *e) | |
794 | { | |
795 | /* Ensure the afalg error handling is set up */ | |
49ea0f09 | 796 | unsigned short i; |
7f458a48 | 797 | ERR_load_AFALG_strings(); |
798 | ||
799 | if (!ENGINE_set_id(e, engine_afalg_id) | |
800 | || !ENGINE_set_name(e, engine_afalg_name) | |
801 | || !ENGINE_set_destroy_function(e, afalg_destroy) | |
802 | || !ENGINE_set_init_function(e, afalg_init) | |
803 | || !ENGINE_set_finish_function(e, afalg_finish)) { | |
804 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
805 | return 0; | |
806 | } | |
807 | ||
6cba4a66 | 808 | /* |
49ea0f09 | 809 | * Create _hidden_aes_xxx_cbc by calling afalg_aes_xxx_cbc |
6cba4a66 | 810 | * now, as bind_aflag can only be called by one thread at a |
811 | * time. | |
812 | */ | |
1287dabd | 813 | for (i = 0; i < OSSL_NELEM(afalg_cipher_nids); i++) { |
49ea0f09 J |
814 | if (afalg_aes_cbc(afalg_cipher_nids[i]) == NULL) { |
815 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
816 | return 0; | |
817 | } | |
6cba4a66 | 818 | } |
819 | ||
7f458a48 | 820 | if (!ENGINE_set_ciphers(e, afalg_ciphers)) { |
821 | AFALGerr(AFALG_F_BIND_AFALG, AFALG_R_INIT_FAILED); | |
822 | return 0; | |
823 | } | |
824 | ||
825 | return 1; | |
826 | } | |
827 | ||
828 | # ifndef OPENSSL_NO_DYNAMIC_ENGINE | |
829 | static int bind_helper(ENGINE *e, const char *id) | |
830 | { | |
831 | if (id && (strcmp(id, engine_afalg_id) != 0)) | |
832 | return 0; | |
833 | ||
834 | if (!afalg_chk_platform()) | |
835 | return 0; | |
836 | ||
837 | if (!bind_afalg(e)) | |
838 | return 0; | |
839 | return 1; | |
840 | } | |
841 | ||
842 | IMPLEMENT_DYNAMIC_CHECK_FN() | |
843 | IMPLEMENT_DYNAMIC_BIND_FN(bind_helper) | |
844 | # endif | |
6cba4a66 | 845 | |
7f458a48 | 846 | static int afalg_chk_platform(void) |
847 | { | |
848 | int ret; | |
849 | int i; | |
850 | int kver[3] = { -1, -1, -1 }; | |
25b9d11c | 851 | int sock; |
7f458a48 | 852 | char *str; |
853 | struct utsname ut; | |
854 | ||
855 | ret = uname(&ut); | |
856 | if (ret != 0) { | |
6cba4a66 | 857 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, |
858 | AFALG_R_FAILED_TO_GET_PLATFORM_INFO); | |
7f458a48 | 859 | return 0; |
860 | } | |
861 | ||
862 | str = strtok(ut.release, "."); | |
863 | for (i = 0; i < 3 && str != NULL; i++) { | |
864 | kver[i] = atoi(str); | |
865 | str = strtok(NULL, "."); | |
866 | } | |
867 | ||
868 | if (KERNEL_VERSION(kver[0], kver[1], kver[2]) | |
869 | < KERNEL_VERSION(K_MAJ, K_MIN1, K_MIN2)) { | |
6cba4a66 | 870 | ALG_ERR("ASYNC AFALG not supported this kernel(%d.%d.%d)\n", |
7f458a48 | 871 | kver[0], kver[1], kver[2]); |
6cba4a66 | 872 | ALG_ERR("ASYNC AFALG requires kernel version %d.%d.%d or later\n", |
7f458a48 | 873 | K_MAJ, K_MIN1, K_MIN2); |
874 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, | |
875 | AFALG_R_KERNEL_DOES_NOT_SUPPORT_ASYNC_AFALG); | |
876 | return 0; | |
877 | } | |
878 | ||
25b9d11c MC |
879 | /* Test if we can actually create an AF_ALG socket */ |
880 | sock = socket(AF_ALG, SOCK_SEQPACKET, 0); | |
881 | if (sock == -1) { | |
882 | AFALGerr(AFALG_F_AFALG_CHK_PLATFORM, AFALG_R_SOCKET_CREATE_FAILED); | |
883 | return 0; | |
884 | } | |
885 | close(sock); | |
886 | ||
7f458a48 | 887 | return 1; |
888 | } | |
889 | ||
890 | # ifdef OPENSSL_NO_DYNAMIC_ENGINE | |
891 | static ENGINE *engine_afalg(void) | |
892 | { | |
893 | ENGINE *ret = ENGINE_new(); | |
894 | if (ret == NULL) | |
895 | return NULL; | |
896 | if (!bind_afalg(ret)) { | |
897 | ENGINE_free(ret); | |
898 | return NULL; | |
899 | } | |
900 | return ret; | |
901 | } | |
902 | ||
b3599dbb | 903 | void engine_load_afalg_int(void) |
7f458a48 | 904 | { |
905 | ENGINE *toadd; | |
906 | ||
907 | if (!afalg_chk_platform()) | |
908 | return; | |
909 | ||
910 | toadd = engine_afalg(); | |
911 | if (toadd == NULL) | |
912 | return; | |
b9b2135d | 913 | ERR_set_mark(); |
7f458a48 | 914 | ENGINE_add(toadd); |
b9b2135d MC |
915 | /* |
916 | * If the "add" worked, it gets a structural reference. So either way, we | |
917 | * release our just-created reference. | |
918 | */ | |
7f458a48 | 919 | ENGINE_free(toadd); |
b9b2135d MC |
920 | /* |
921 | * If the "add" didn't work, it was probably a conflict because it was | |
922 | * already added (eg. someone calling ENGINE_load_blah then calling | |
923 | * ENGINE_load_builtin_engines() perhaps). | |
924 | */ | |
925 | ERR_pop_to_mark(); | |
7f458a48 | 926 | } |
927 | # endif | |
928 | ||
929 | static int afalg_init(ENGINE *e) | |
930 | { | |
931 | return 1; | |
932 | } | |
933 | ||
934 | static int afalg_finish(ENGINE *e) | |
935 | { | |
936 | return 1; | |
937 | } | |
938 | ||
49ea0f09 J |
939 | static int free_cbc(void) |
940 | { | |
a3d7fd28 | 941 | short unsigned int i; |
1287dabd | 942 | for (i = 0; i < OSSL_NELEM(afalg_cipher_nids); i++) { |
49ea0f09 J |
943 | EVP_CIPHER_meth_free(cbc_handle[i]._hidden); |
944 | cbc_handle[i]._hidden = NULL; | |
945 | } | |
946 | return 1; | |
947 | } | |
948 | ||
7f458a48 | 949 | static int afalg_destroy(ENGINE *e) |
950 | { | |
951 | ERR_unload_AFALG_strings(); | |
49ea0f09 | 952 | free_cbc(); |
7f458a48 | 953 | return 1; |
954 | } | |
955 | ||
956 | #endif /* KERNEL VERSION */ |