]> git.ipfire.org Git - thirdparty/openssl.git/blame - crypto/evp/e_aes.c
projects / thirdparty / openssl.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Move the AES-XTS mode duplicated key check into the init_key function rather
[thirdparty/openssl.git] / crypto / evp / e_aes.c
CommitLineData
aa6bb135 1/*
5516c19b 2 * Copyright 2001-2019 The OpenSSL Project Authors. All Rights Reserved.
deb2c1a1 3 *
4a8b0c55 4 * Licensed under the Apache License 2.0 (the "License"). You may not use
aa6bb135
RS		 5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
deb2c1a1
DSH		 8 */
9
8c84b677 10#include <openssl/opensslconf.h>
5158c763
MC		 11#include <openssl/crypto.h>
12#include <openssl/evp.h>
13#include <openssl/err.h>
14#include <string.h>
15#include <assert.h>
16#include <openssl/aes.h>
17#include "internal/evp_int.h"
18#include "modes_lcl.h"
19#include <openssl/rand.h>
b1ceb439 20#include <openssl/cmac.h>
7141ba31 21#include "evp_locl.h"
0f113f3e
MC		 22
23typedef struct {
24 union {
25 double align;
26 AES_KEY ks;
27 } ks;
28 block128_f block;
29 union {
30 cbc128_f cbc;
31 ctr128_f ctr;
32 } stream;
33} EVP_AES_KEY;
34
35typedef struct {
36 union {
37 double align;
38 AES_KEY ks;
39 } ks; /* AES key schedule to use */
40 int key_set; /* Set if key initialised */
41 int iv_set; /* Set if an iv is set */
42 GCM128_CONTEXT gcm;
43 unsigned char *iv; /* Temporary IV store */
44 int ivlen; /* IV length */
45 int taglen;
46 int iv_gen; /* It is OK to generate IVs */
bcf082d1 47 int iv_gen_rand; /* No IV was specified, so generate a rand IV */
0f113f3e 48 int tls_aad_len; /* TLS AAD length */
d6b34570 49 uint64_t tls_enc_records; /* Number of TLS records encrypted */
0f113f3e
MC		 50 ctr128_f ctr;
51} EVP_AES_GCM_CTX;
52
53typedef struct {
54 union {
55 double align;
56 AES_KEY ks;
57 } ks1, ks2; /* AES key schedules to use */
58 XTS128_CONTEXT xts;
59 void (*stream) (const unsigned char *in,
60 unsigned char *out, size_t length,
61 const AES_KEY *key1, const AES_KEY *key2,
62 const unsigned char iv[16]);
63} EVP_AES_XTS_CTX;
64
65typedef struct {
66 union {
67 double align;
68 AES_KEY ks;
69 } ks; /* AES key schedule to use */
70 int key_set; /* Set if key initialised */
71 int iv_set; /* Set if an iv is set */
72 int tag_set; /* Set if tag is valid */
73 int len_set; /* Set if message length set */
74 int L, M; /* L and M parameters from RFC3610 */
e75c5a79 75 int tls_aad_len; /* TLS AAD length */
0f113f3e
MC		 76 CCM128_CONTEXT ccm;
77 ccm128_f str;
78} EVP_AES_CCM_CTX;
79
5158c763 80#ifndef OPENSSL_NO_OCB
0f113f3e 81typedef struct {
bdc985b1
AP		 82 union {
83 double align;
84 AES_KEY ks;
85 } ksenc; /* AES key schedule to use for encryption */
86 union {
87 double align;
88 AES_KEY ks;
89 } ksdec; /* AES key schedule to use for decryption */
0f113f3e
MC		 90 int key_set; /* Set if key initialised */
91 int iv_set; /* Set if an iv is set */
92 OCB128_CONTEXT ocb;
93 unsigned char *iv; /* Temporary IV store */
94 unsigned char tag[16];
95 unsigned char data_buf[16]; /* Store partial data blocks */
96 unsigned char aad_buf[16]; /* Store partial AAD blocks */
97 int data_buf_len;
98 int aad_buf_len;
99 int ivlen; /* IV length */
100 int taglen;
101} EVP_AES_OCB_CTX;
5158c763 102#endif
e6b336ef 103
5158c763 104#define MAXBITCHUNK ((size_t)1<<(sizeof(size_t)*8-4))
17f121de 105
5158c763 106#ifdef VPAES_ASM
8ca28da0 107int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
0f113f3e 108 AES_KEY *key);
8ca28da0 109int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
0f113f3e 110 AES_KEY *key);
8ca28da0
AP		 111
112void vpaes_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e 113 const AES_KEY *key);
8ca28da0 114void vpaes_decrypt(const unsigned char *in, unsigned char *out,
0f113f3e 115 const AES_KEY *key);
8ca28da0
AP		 116
117void vpaes_cbc_encrypt(const unsigned char *in,
0f113f3e
MC		 118 unsigned char *out,
119 size_t length,
120 const AES_KEY *key, unsigned char *ivec, int enc);
5158c763
MC		 121#endif
122#ifdef BSAES_ASM
a75a52a4 123void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e
MC		 124 size_t length, const AES_KEY *key,
125 unsigned char ivec[16], int enc);
993adc05 126void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
0f113f3e
MC		 127 size_t len, const AES_KEY *key,
128 const unsigned char ivec[16]);
60d4e99c 129void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
0f113f3e
MC		 130 size_t len, const AES_KEY *key1,
131 const AES_KEY *key2, const unsigned char iv[16]);
60d4e99c 132void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
0f113f3e
MC		 133 size_t len, const AES_KEY *key1,
134 const AES_KEY *key2, const unsigned char iv[16]);
5158c763
MC		 135#endif
136#ifdef AES_CTR_ASM
07904e0c 137void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e
MC		 138 size_t blocks, const AES_KEY *key,
139 const unsigned char ivec[AES_BLOCK_SIZE]);
5158c763
MC		 140#endif
141#ifdef AES_XTS_ASM
96cce820 142void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
0f113f3e
MC		 143 const AES_KEY *key1, const AES_KEY *key2,
144 const unsigned char iv[16]);
96cce820 145void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
0f113f3e
MC		 146 const AES_KEY *key1, const AES_KEY *key2,
147 const unsigned char iv[16]);
5158c763 148#endif
8ca28da0 149
03a5e5ae
PS		 150/* increment counter (64-bit int) by 1 */
151static void ctr64_inc(unsigned char *counter)
152{
153 int n = 8;
154 unsigned char c;
155
156 do {
157 --n;
158 c = counter[n];
159 ++c;
160 counter[n] = c;
161 if (c)
162 return;
163 } while (n);
164}
165
6944565b 166#if defined(OPENSSL_CPUID_OBJ) && (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
5158c763
MC		 167# include "ppc_arch.h"
168# ifdef VPAES_ASM
169# define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
de51e830 170# endif
5158c763
MC		 171# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
172# define HWAES_set_encrypt_key aes_p8_set_encrypt_key
173# define HWAES_set_decrypt_key aes_p8_set_decrypt_key
174# define HWAES_encrypt aes_p8_encrypt
175# define HWAES_decrypt aes_p8_decrypt
176# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
177# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
46f047d7
AP		 178# define HWAES_xts_encrypt aes_p8_xts_encrypt
179# define HWAES_xts_decrypt aes_p8_xts_decrypt
5158c763 180#endif
07f3e4f3 181
5158c763 182#if defined(AES_ASM) && !defined(I386_ONLY) && ( \
0f113f3e
MC		 183 ((defined(__i386) || defined(__i386__) || \
184 defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
185 defined(__x86_64) || defined(__x86_64__) || \
b1a07c38 186 defined(_M_AMD64) || defined(_M_X64) )
8ca28da0 187
c5f6da54 188extern unsigned int OPENSSL_ia32cap_P[];
8ca28da0 189
5158c763
MC		 190# ifdef VPAES_ASM
191# define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
192# endif
193# ifdef BSAES_ASM
194# define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
195# endif
17f121de
AP		 196/*
197 * AES-NI section
198 */
5158c763 199# define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
d1fff483
AP		 200
201int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
0f113f3e 202 AES_KEY *key);
d1fff483 203int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
0f113f3e 204 AES_KEY *key);
d1fff483
AP		 205
206void aesni_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e 207 const AES_KEY *key);
d1fff483 208void aesni_decrypt(const unsigned char *in, unsigned char *out,
0f113f3e 209 const AES_KEY *key);
d1fff483
AP		 210
211void aesni_ecb_encrypt(const unsigned char *in,
0f113f3e
MC		 212 unsigned char *out,
213 size_t length, const AES_KEY *key, int enc);
d1fff483 214void aesni_cbc_encrypt(const unsigned char *in,
0f113f3e
MC		 215 unsigned char *out,
216 size_t length,
217 const AES_KEY *key, unsigned char *ivec, int enc);
d1fff483
AP		 218
219void aesni_ctr32_encrypt_blocks(const unsigned char *in,
0f113f3e
MC		 220 unsigned char *out,
221 size_t blocks,
222 const void *key, const unsigned char *ivec);
17f121de
AP		 223
224void aesni_xts_encrypt(const unsigned char *in,
0f113f3e
MC		 225 unsigned char *out,
226 size_t length,
227 const AES_KEY *key1, const AES_KEY *key2,
228 const unsigned char iv[16]);
17f121de
AP		 229
230void aesni_xts_decrypt(const unsigned char *in,
0f113f3e
MC		 231 unsigned char *out,
232 size_t length,
233 const AES_KEY *key1, const AES_KEY *key2,
234 const unsigned char iv[16]);
235
236void aesni_ccm64_encrypt_blocks(const unsigned char *in,
237 unsigned char *out,
238 size_t blocks,
239 const void *key,
240 const unsigned char ivec[16],
241 unsigned char cmac[16]);
242
243void aesni_ccm64_decrypt_blocks(const unsigned char *in,
244 unsigned char *out,
245 size_t blocks,
246 const void *key,
247 const unsigned char ivec[16],
248 unsigned char cmac[16]);
249
5158c763 250# if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
4e049c52 251size_t aesni_gcm_encrypt(const unsigned char *in,
0f113f3e
MC		 252 unsigned char *out,
253 size_t len,
254 const void *key, unsigned char ivec[16], u64 *Xi);
5158c763 255# define AES_gcm_encrypt aesni_gcm_encrypt
4e049c52 256size_t aesni_gcm_decrypt(const unsigned char *in,
0f113f3e
MC		 257 unsigned char *out,
258 size_t len,
259 const void *key, unsigned char ivec[16], u64 *Xi);
5158c763 260# define AES_gcm_decrypt aesni_gcm_decrypt
0f113f3e
MC		 261void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in,
262 size_t len);
5158c763 263# define AES_GCM_ASM(gctx) (gctx->ctr==aesni_ctr32_encrypt_blocks && \
0f113f3e 264 gctx->gcm.ghash==gcm_ghash_avx)
5158c763 265# define AES_GCM_ASM2(gctx) (gctx->gcm.block==(block128_f)aesni_encrypt && \
0f113f3e 266 gctx->gcm.ghash==gcm_ghash_avx)
5158c763
MC		 267# undef AES_GCM_ASM2 /* minor size optimization */
268# endif
4e049c52 269
17f121de 270static int aesni_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 271 const unsigned char *iv, int enc)
272{
273 int ret, mode;
6435f0f6 274 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
0f113f3e 275
6435f0f6 276 mode = EVP_CIPHER_CTX_mode(ctx);
0f113f3e
MC		 277 if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
278 && !enc) {
6435f0f6
RL		 279 ret = aesni_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
280 &dat->ks.ks);
0f113f3e
MC		 281 dat->block = (block128_f) aesni_decrypt;
282 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
283 (cbc128_f) aesni_cbc_encrypt : NULL;
284 } else {
6435f0f6
RL		 285 ret = aesni_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
286 &dat->ks.ks);
0f113f3e
MC		 287 dat->block = (block128_f) aesni_encrypt;
288 if (mode == EVP_CIPH_CBC_MODE)
289 dat->stream.cbc = (cbc128_f) aesni_cbc_encrypt;
290 else if (mode == EVP_CIPH_CTR_MODE)
291 dat->stream.ctr = (ctr128_f) aesni_ctr32_encrypt_blocks;
292 else
293 dat->stream.cbc = NULL;
294 }
295
296 if (ret < 0) {
297 EVPerr(EVP_F_AESNI_INIT_KEY, EVP_R_AES_KEY_SETUP_FAILED);
298 return 0;
299 }
300
301 return 1;
302}
303
304static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
305 const unsigned char *in, size_t len)
d1fff483 306{
6435f0f6
RL		 307 aesni_cbc_encrypt(in, out, len, &EVP_C_DATA(EVP_AES_KEY,ctx)->ks.ks,
308 EVP_CIPHER_CTX_iv_noconst(ctx),
309 EVP_CIPHER_CTX_encrypting(ctx));
d1fff483 310
0f113f3e 311 return 1;
d1fff483
AP		 312}
313
0f113f3e
MC		 314static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
315 const unsigned char *in, size_t len)
d1fff483 316{
6435f0f6 317 size_t bl = EVP_CIPHER_CTX_block_size(ctx);
d1fff483 318
0f113f3e
MC		 319 if (len < bl)
320 return 1;
d1fff483 321
6435f0f6
RL		 322 aesni_ecb_encrypt(in, out, len, &EVP_C_DATA(EVP_AES_KEY,ctx)->ks.ks,
323 EVP_CIPHER_CTX_encrypting(ctx));
d1fff483 324
0f113f3e 325 return 1;
d1fff483
AP		 326}
327
5158c763 328# define aesni_ofb_cipher aes_ofb_cipher
0f113f3e
MC		 329static int aesni_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
330 const unsigned char *in, size_t len);
d1fff483 331
5158c763 332# define aesni_cfb_cipher aes_cfb_cipher
0f113f3e
MC		 333static int aesni_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
334 const unsigned char *in, size_t len);
d1fff483 335
5158c763 336# define aesni_cfb8_cipher aes_cfb8_cipher
0f113f3e
MC		 337static int aesni_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
338 const unsigned char *in, size_t len);
d1fff483 339
5158c763 340# define aesni_cfb1_cipher aes_cfb1_cipher
0f113f3e
MC		 341static int aesni_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
342 const unsigned char *in, size_t len);
d1fff483 343
5158c763 344# define aesni_ctr_cipher aes_ctr_cipher
17f121de 345static int aesni_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 346 const unsigned char *in, size_t len);
d1fff483 347
17f121de 348static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 349 const unsigned char *iv, int enc)
350{
6435f0f6 351 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
0f113f3e
MC		 352 if (!iv && !key)
353 return 1;
354 if (key) {
6435f0f6
RL		 355 aesni_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
356 &gctx->ks.ks);
0f113f3e
MC		 357 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f) aesni_encrypt);
358 gctx->ctr = (ctr128_f) aesni_ctr32_encrypt_blocks;
359 /*
360 * If we have an iv can set it directly, otherwise use saved IV.
361 */
362 if (iv == NULL && gctx->iv_set)
363 iv = gctx->iv;
364 if (iv) {
365 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
366 gctx->iv_set = 1;
367 }
368 gctx->key_set = 1;
369 } else {
370 /* If key set use IV, otherwise copy */
371 if (gctx->key_set)
372 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
373 else
374 memcpy(gctx->iv, iv, gctx->ivlen);
375 gctx->iv_set = 1;
376 gctx->iv_gen = 0;
377 }
378 return 1;
379}
380
5158c763 381# define aesni_gcm_cipher aes_gcm_cipher
17f121de 382static int aesni_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 383 const unsigned char *in, size_t len);
17f121de
AP		 384
385static int aesni_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 386 const unsigned char *iv, int enc)
387{
6435f0f6 388 EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
0f113f3e
MC		 389 if (!iv && !key)
390 return 1;
391
392 if (key) {
3538b0f7
P		 393 /* The key is two half length keys in reality */
394 const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
395 const int bits = bytes * 8;
396
397 /*
398 * Verify that the two keys are different.
399 *
400 * This addresses Rogaway's vulnerability.
401 * See comment in aes_xts_init_key() below.
402 */
403 if (memcmp(key, key + bytes, bytes) == 0) {
404 EVPerr(EVP_F_AESNI_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
405 return 0;
406 }
407
0f113f3e
MC		 408 /* key_len is two AES keys */
409 if (enc) {
3538b0f7 410 aesni_set_encrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 411 xctx->xts.block1 = (block128_f) aesni_encrypt;
412 xctx->stream = aesni_xts_encrypt;
413 } else {
3538b0f7 414 aesni_set_decrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 415 xctx->xts.block1 = (block128_f) aesni_decrypt;
416 xctx->stream = aesni_xts_decrypt;
417 }
418
3538b0f7 419 aesni_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
0f113f3e
MC		 420 xctx->xts.block2 = (block128_f) aesni_encrypt;
421
422 xctx->xts.key1 = &xctx->ks1;
423 }
424
425 if (iv) {
426 xctx->xts.key2 = &xctx->ks2;
6435f0f6 427 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 16);
0f113f3e
MC		 428 }
429
430 return 1;
431}
432
5158c763 433# define aesni_xts_cipher aes_xts_cipher
17f121de 434static int aesni_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 435 const unsigned char *in, size_t len);
17f121de
AP		 436
437static int aesni_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 438 const unsigned char *iv, int enc)
439{
6435f0f6 440 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx);
0f113f3e
MC		 441 if (!iv && !key)
442 return 1;
443 if (key) {
6435f0f6
RL		 444 aesni_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
445 &cctx->ks.ks);
0f113f3e
MC		 446 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
447 &cctx->ks, (block128_f) aesni_encrypt);
448 cctx->str = enc ? (ccm128_f) aesni_ccm64_encrypt_blocks :
449 (ccm128_f) aesni_ccm64_decrypt_blocks;
450 cctx->key_set = 1;
451 }
452 if (iv) {
6435f0f6 453 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 15 - cctx->L);
0f113f3e
MC		 454 cctx->iv_set = 1;
455 }
456 return 1;
457}
458
5158c763 459# define aesni_ccm_cipher aes_ccm_cipher
17f121de 460static int aesni_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 461 const unsigned char *in, size_t len);
17f121de 462
5158c763 463# ifndef OPENSSL_NO_OCB
bd30091c
AP		 464void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
465 size_t blocks, const void *key,
466 size_t start_block_num,
467 unsigned char offset_i[16],
468 const unsigned char L_[][16],
469 unsigned char checksum[16]);
470void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
471 size_t blocks, const void *key,
472 size_t start_block_num,
473 unsigned char offset_i[16],
474 const unsigned char L_[][16],
475 unsigned char checksum[16]);
476
e6b336ef 477static int aesni_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 478 const unsigned char *iv, int enc)
479{
6435f0f6 480 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx);
0f113f3e
MC		 481 if (!iv && !key)
482 return 1;
483 if (key) {
484 do {
485 /*
486 * We set both the encrypt and decrypt key here because decrypt
487 * needs both. We could possibly optimise to remove setting the
488 * decrypt for an encryption operation.
489 */
6435f0f6
RL		 490 aesni_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
491 &octx->ksenc.ks);
492 aesni_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
493 &octx->ksdec.ks);
bdc985b1
AP		 494 if (!CRYPTO_ocb128_init(&octx->ocb,
495 &octx->ksenc.ks, &octx->ksdec.ks,
0f113f3e 496 (block128_f) aesni_encrypt,
bd30091c
AP		 497 (block128_f) aesni_decrypt,
498 enc ? aesni_ocb_encrypt
499 : aesni_ocb_decrypt))
0f113f3e
MC		 500 return 0;
501 }
502 while (0);
503
504 /*
505 * If we have an iv we can set it directly, otherwise use saved IV.
506 */
507 if (iv == NULL && octx->iv_set)
508 iv = octx->iv;
509 if (iv) {
510 if (CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen)
511 != 1)
512 return 0;
513 octx->iv_set = 1;
514 }
515 octx->key_set = 1;
516 } else {
517 /* If key set use IV, otherwise copy */
518 if (octx->key_set)
519 CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
520 else
521 memcpy(octx->iv, iv, octx->ivlen);
522 octx->iv_set = 1;
523 }
524 return 1;
525}
526
5158c763 527# define aesni_ocb_cipher aes_ocb_cipher
e6b336ef 528static int aesni_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 529 const unsigned char *in, size_t len);
5158c763 530# endif /* OPENSSL_NO_OCB */
e6b336ef 531
5158c763 532# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
17f121de 533static const EVP_CIPHER aesni_##keylen##_##mode = { \
0f113f3e
MC		 534 nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
535 flags|EVP_CIPH_##MODE##_MODE, \
536 aesni_init_key, \
537 aesni_##mode##_cipher, \
538 NULL, \
539 sizeof(EVP_AES_KEY), \
540 NULL,NULL,NULL,NULL }; \
17f121de 541static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e
MC		 542 nid##_##keylen##_##nmode,blocksize, \
543 keylen/8,ivlen, \
544 flags|EVP_CIPH_##MODE##_MODE, \
545 aes_init_key, \
546 aes_##mode##_cipher, \
547 NULL, \
548 sizeof(EVP_AES_KEY), \
549 NULL,NULL,NULL,NULL }; \
17f121de 550const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
8ca28da0 551{ return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
17f121de 552
5158c763 553# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
17f121de 554static const EVP_CIPHER aesni_##keylen##_##mode = { \
0f113f3e 555 nid##_##keylen##_##mode,blocksize, \
b1ceb439
TS		 556 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
557 ivlen, \
0f113f3e
MC		 558 flags|EVP_CIPH_##MODE##_MODE, \
559 aesni_##mode##_init_key, \
560 aesni_##mode##_cipher, \
561 aes_##mode##_cleanup, \
562 sizeof(EVP_AES_##MODE##_CTX), \
563 NULL,NULL,aes_##mode##_ctrl,NULL }; \
17f121de 564static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e 565 nid##_##keylen##_##mode,blocksize, \
b1ceb439
TS		 566 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
567 ivlen, \
0f113f3e
MC		 568 flags|EVP_CIPH_##MODE##_MODE, \
569 aes_##mode##_init_key, \
570 aes_##mode##_cipher, \
571 aes_##mode##_cleanup, \
572 sizeof(EVP_AES_##MODE##_CTX), \
573 NULL,NULL,aes_##mode##_ctrl,NULL }; \
17f121de 574const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
8ca28da0 575{ return AESNI_CAPABLE?&aesni_##keylen##_##mode:&aes_##keylen##_##mode; }
d1fff483 576
5158c763 577#elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
c5f6da54 578
5158c763 579# include "sparc_arch.h"
c5f6da54
AP		 580
581extern unsigned int OPENSSL_sparcv9cap_P[];
582
6944565b
AP		 583/*
584 * Initial Fujitsu SPARC64 X support
585 */
586# define HWAES_CAPABLE (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
587# define HWAES_set_encrypt_key aes_fx_set_encrypt_key
588# define HWAES_set_decrypt_key aes_fx_set_decrypt_key
589# define HWAES_encrypt aes_fx_encrypt
590# define HWAES_decrypt aes_fx_decrypt
365f95ad
AP		 591# define HWAES_cbc_encrypt aes_fx_cbc_encrypt
592# define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
6944565b 593
5158c763 594# define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES)
c5f6da54 595
0f113f3e
MC		 596void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
597void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
598void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
599 const AES_KEY *key);
600void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
601 const AES_KEY *key);
c5f6da54
AP		 602/*
603 * Key-length specific subroutines were chosen for following reason.
604 * Each SPARC T4 core can execute up to 8 threads which share core's
605 * resources. Loading as much key material to registers allows to
606 * minimize references to shared memory interface, as well as amount
607 * of instructions in inner loops [much needed on T4]. But then having
608 * non-key-length specific routines would require conditional branches
609 * either in inner loops or on subroutines' entries. Former is hardly
610 * acceptable, while latter means code size increase to size occupied
0d4fb843 611 * by multiple key-length specific subroutines, so why fight?
c5f6da54 612 */
0f113f3e
MC		 613void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
614 size_t len, const AES_KEY *key,
615 unsigned char *ivec);
616void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
617 size_t len, const AES_KEY *key,
618 unsigned char *ivec);
619void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
620 size_t len, const AES_KEY *key,
621 unsigned char *ivec);
622void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
623 size_t len, const AES_KEY *key,
624 unsigned char *ivec);
625void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
626 size_t len, const AES_KEY *key,
627 unsigned char *ivec);
628void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
629 size_t len, const AES_KEY *key,
630 unsigned char *ivec);
631void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
632 size_t blocks, const AES_KEY *key,
633 unsigned char *ivec);
634void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
635 size_t blocks, const AES_KEY *key,
636 unsigned char *ivec);
637void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
638 size_t blocks, const AES_KEY *key,
639 unsigned char *ivec);
640void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
641 size_t blocks, const AES_KEY *key1,
642 const AES_KEY *key2, const unsigned char *ivec);
643void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
644 size_t blocks, const AES_KEY *key1,
645 const AES_KEY *key2, const unsigned char *ivec);
646void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
647 size_t blocks, const AES_KEY *key1,
648 const AES_KEY *key2, const unsigned char *ivec);
649void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
650 size_t blocks, const AES_KEY *key1,
651 const AES_KEY *key2, const unsigned char *ivec);
c5f6da54
AP		 652
653static int aes_t4_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 654 const unsigned char *iv, int enc)
655{
656 int ret, mode, bits;
6435f0f6 657 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
0f113f3e 658
6435f0f6
RL		 659 mode = EVP_CIPHER_CTX_mode(ctx);
660 bits = EVP_CIPHER_CTX_key_length(ctx) * 8;
0f113f3e
MC		 661 if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
662 && !enc) {
663 ret = 0;
6435f0f6 664 aes_t4_set_decrypt_key(key, bits, &dat->ks.ks);
0f113f3e
MC		 665 dat->block = (block128_f) aes_t4_decrypt;
666 switch (bits) {
667 case 128:
668 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
669 (cbc128_f) aes128_t4_cbc_decrypt : NULL;
670 break;
671 case 192:
672 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
673 (cbc128_f) aes192_t4_cbc_decrypt : NULL;
674 break;
675 case 256:
676 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
677 (cbc128_f) aes256_t4_cbc_decrypt : NULL;
678 break;
679 default:
680 ret = -1;
681 }
682 } else {
683 ret = 0;
6435f0f6 684 aes_t4_set_encrypt_key(key, bits, &dat->ks.ks);
0f113f3e
MC		 685 dat->block = (block128_f) aes_t4_encrypt;
686 switch (bits) {
687 case 128:
688 if (mode == EVP_CIPH_CBC_MODE)
689 dat->stream.cbc = (cbc128_f) aes128_t4_cbc_encrypt;
690 else if (mode == EVP_CIPH_CTR_MODE)
691 dat->stream.ctr = (ctr128_f) aes128_t4_ctr32_encrypt;
692 else
693 dat->stream.cbc = NULL;
694 break;
695 case 192:
696 if (mode == EVP_CIPH_CBC_MODE)
697 dat->stream.cbc = (cbc128_f) aes192_t4_cbc_encrypt;
698 else if (mode == EVP_CIPH_CTR_MODE)
699 dat->stream.ctr = (ctr128_f) aes192_t4_ctr32_encrypt;
700 else
701 dat->stream.cbc = NULL;
702 break;
703 case 256:
704 if (mode == EVP_CIPH_CBC_MODE)
705 dat->stream.cbc = (cbc128_f) aes256_t4_cbc_encrypt;
706 else if (mode == EVP_CIPH_CTR_MODE)
707 dat->stream.ctr = (ctr128_f) aes256_t4_ctr32_encrypt;
708 else
709 dat->stream.cbc = NULL;
710 break;
711 default:
712 ret = -1;
713 }
714 }
715
716 if (ret < 0) {
717 EVPerr(EVP_F_AES_T4_INIT_KEY, EVP_R_AES_KEY_SETUP_FAILED);
718 return 0;
719 }
720
721 return 1;
722}
723
5158c763 724# define aes_t4_cbc_cipher aes_cbc_cipher
0f113f3e
MC		 725static int aes_t4_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
726 const unsigned char *in, size_t len);
727
5158c763 728# define aes_t4_ecb_cipher aes_ecb_cipher
0f113f3e
MC		 729static int aes_t4_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
730 const unsigned char *in, size_t len);
731
5158c763 732# define aes_t4_ofb_cipher aes_ofb_cipher
0f113f3e
MC		 733static int aes_t4_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
734 const unsigned char *in, size_t len);
735
5158c763 736# define aes_t4_cfb_cipher aes_cfb_cipher
0f113f3e
MC		 737static int aes_t4_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
738 const unsigned char *in, size_t len);
739
5158c763 740# define aes_t4_cfb8_cipher aes_cfb8_cipher
0f113f3e
MC		 741static int aes_t4_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
742 const unsigned char *in, size_t len);
743
5158c763 744# define aes_t4_cfb1_cipher aes_cfb1_cipher
0f113f3e
MC		 745static int aes_t4_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
746 const unsigned char *in, size_t len);
747
5158c763 748# define aes_t4_ctr_cipher aes_ctr_cipher
c5f6da54 749static int aes_t4_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 750 const unsigned char *in, size_t len);
c5f6da54
AP		 751
752static int aes_t4_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 753 const unsigned char *iv, int enc)
754{
6435f0f6 755 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
0f113f3e
MC		 756 if (!iv && !key)
757 return 1;
758 if (key) {
6435f0f6 759 int bits = EVP_CIPHER_CTX_key_length(ctx) * 8;
0f113f3e
MC		 760 aes_t4_set_encrypt_key(key, bits, &gctx->ks.ks);
761 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
762 (block128_f) aes_t4_encrypt);
763 switch (bits) {
764 case 128:
765 gctx->ctr = (ctr128_f) aes128_t4_ctr32_encrypt;
766 break;
767 case 192:
768 gctx->ctr = (ctr128_f) aes192_t4_ctr32_encrypt;
769 break;
770 case 256:
771 gctx->ctr = (ctr128_f) aes256_t4_ctr32_encrypt;
772 break;
773 default:
774 return 0;
775 }
776 /*
777 * If we have an iv can set it directly, otherwise use saved IV.
778 */
779 if (iv == NULL && gctx->iv_set)
780 iv = gctx->iv;
781 if (iv) {
782 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
783 gctx->iv_set = 1;
784 }
785 gctx->key_set = 1;
786 } else {
787 /* If key set use IV, otherwise copy */
788 if (gctx->key_set)
789 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
790 else
791 memcpy(gctx->iv, iv, gctx->ivlen);
792 gctx->iv_set = 1;
793 gctx->iv_gen = 0;
794 }
795 return 1;
796}
797
5158c763 798# define aes_t4_gcm_cipher aes_gcm_cipher
c5f6da54 799static int aes_t4_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 800 const unsigned char *in, size_t len);
c5f6da54
AP		 801
802static int aes_t4_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 803 const unsigned char *iv, int enc)
804{
6435f0f6 805 EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
0f113f3e
MC		 806 if (!iv && !key)
807 return 1;
808
809 if (key) {
3538b0f7
P		 810 /* The key is two half length keys in reality */
811 const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
812 const int bits = bytes * 8;
813
814 /*
815 * Verify that the two keys are different.
816 *
817 * This addresses Rogaway's vulnerability.
818 * See comment in aes_xts_init_key() below.
819 */
820 if (memcmp(key, key + bytes, bytes) == 0) {
821 EVPerr(EVP_F_AES_T4_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
822 return 0;
823 }
824
0f113f3e
MC		 825 xctx->stream = NULL;
826 /* key_len is two AES keys */
827 if (enc) {
828 aes_t4_set_encrypt_key(key, bits, &xctx->ks1.ks);
829 xctx->xts.block1 = (block128_f) aes_t4_encrypt;
830 switch (bits) {
831 case 128:
832 xctx->stream = aes128_t4_xts_encrypt;
833 break;
0f113f3e
MC		 834 case 256:
835 xctx->stream = aes256_t4_xts_encrypt;
836 break;
837 default:
838 return 0;
839 }
840 } else {
3538b0f7 841 aes_t4_set_decrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 842 xctx->xts.block1 = (block128_f) aes_t4_decrypt;
843 switch (bits) {
844 case 128:
845 xctx->stream = aes128_t4_xts_decrypt;
846 break;
0f113f3e
MC		 847 case 256:
848 xctx->stream = aes256_t4_xts_decrypt;
849 break;
850 default:
851 return 0;
852 }
853 }
854
3538b0f7 855 aes_t4_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
0f113f3e
MC		 856 xctx->xts.block2 = (block128_f) aes_t4_encrypt;
857
858 xctx->xts.key1 = &xctx->ks1;
859 }
860
861 if (iv) {
862 xctx->xts.key2 = &xctx->ks2;
6435f0f6 863 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 16);
0f113f3e
MC		 864 }
865
866 return 1;
867}
868
5158c763 869# define aes_t4_xts_cipher aes_xts_cipher
c5f6da54 870static int aes_t4_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 871 const unsigned char *in, size_t len);
c5f6da54
AP		 872
873static int aes_t4_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 874 const unsigned char *iv, int enc)
875{
6435f0f6 876 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx);
0f113f3e
MC		 877 if (!iv && !key)
878 return 1;
879 if (key) {
6435f0f6 880 int bits = EVP_CIPHER_CTX_key_length(ctx) * 8;
0f113f3e
MC		 881 aes_t4_set_encrypt_key(key, bits, &cctx->ks.ks);
882 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
883 &cctx->ks, (block128_f) aes_t4_encrypt);
bdc985b1 884 cctx->str = NULL;
0f113f3e
MC		 885 cctx->key_set = 1;
886 }
887 if (iv) {
6435f0f6 888 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 15 - cctx->L);
0f113f3e
MC		 889 cctx->iv_set = 1;
890 }
891 return 1;
892}
893
5158c763 894# define aes_t4_ccm_cipher aes_ccm_cipher
c5f6da54 895static int aes_t4_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 896 const unsigned char *in, size_t len);
c5f6da54 897
5158c763 898# ifndef OPENSSL_NO_OCB
e6b336ef 899static int aes_t4_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 900 const unsigned char *iv, int enc)
901{
6435f0f6 902 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx);
0f113f3e
MC		 903 if (!iv && !key)
904 return 1;
905 if (key) {
906 do {
907 /*
908 * We set both the encrypt and decrypt key here because decrypt
909 * needs both. We could possibly optimise to remove setting the
910 * decrypt for an encryption operation.
911 */
6435f0f6
RL		 912 aes_t4_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
913 &octx->ksenc.ks);
914 aes_t4_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
915 &octx->ksdec.ks);
bdc985b1
AP		 916 if (!CRYPTO_ocb128_init(&octx->ocb,
917 &octx->ksenc.ks, &octx->ksdec.ks,
0f113f3e 918 (block128_f) aes_t4_encrypt,
02dc0b82
AP		 919 (block128_f) aes_t4_decrypt,
920 NULL))
0f113f3e
MC		 921 return 0;
922 }
923 while (0);
924
925 /*
926 * If we have an iv we can set it directly, otherwise use saved IV.
927 */
928 if (iv == NULL && octx->iv_set)
929 iv = octx->iv;
930 if (iv) {
931 if (CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen)
932 != 1)
933 return 0;
934 octx->iv_set = 1;
935 }
936 octx->key_set = 1;
937 } else {
938 /* If key set use IV, otherwise copy */
939 if (octx->key_set)
940 CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
941 else
942 memcpy(octx->iv, iv, octx->ivlen);
943 octx->iv_set = 1;
944 }
945 return 1;
946}
947
5158c763 948# define aes_t4_ocb_cipher aes_ocb_cipher
e6b336ef 949static int aes_t4_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e 950 const unsigned char *in, size_t len);
5158c763 951# endif /* OPENSSL_NO_OCB */
e6b336ef 952
87d06aed
MC		 953# ifndef OPENSSL_NO_SIV
954# define aes_t4_siv_init_key aes_siv_init_key
955# define aes_t4_siv_cipher aes_siv_cipher
956# endif /* OPENSSL_NO_SIV */
957
5158c763 958# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
c5f6da54 959static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
0f113f3e
MC		 960 nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
961 flags|EVP_CIPH_##MODE##_MODE, \
962 aes_t4_init_key, \
963 aes_t4_##mode##_cipher, \
964 NULL, \
965 sizeof(EVP_AES_KEY), \
966 NULL,NULL,NULL,NULL }; \
c5f6da54 967static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e
MC		 968 nid##_##keylen##_##nmode,blocksize, \
969 keylen/8,ivlen, \
970 flags|EVP_CIPH_##MODE##_MODE, \
971 aes_init_key, \
972 aes_##mode##_cipher, \
973 NULL, \
974 sizeof(EVP_AES_KEY), \
975 NULL,NULL,NULL,NULL }; \
c5f6da54
AP		 976const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
977{ return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
978
5158c763 979# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
c5f6da54 980static const EVP_CIPHER aes_t4_##keylen##_##mode = { \
0f113f3e 981 nid##_##keylen##_##mode,blocksize, \
b1ceb439
TS		 982 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
983 ivlen, \
0f113f3e
MC		 984 flags|EVP_CIPH_##MODE##_MODE, \
985 aes_t4_##mode##_init_key, \
986 aes_t4_##mode##_cipher, \
987 aes_##mode##_cleanup, \
988 sizeof(EVP_AES_##MODE##_CTX), \
989 NULL,NULL,aes_##mode##_ctrl,NULL }; \
c5f6da54 990static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e 991 nid##_##keylen##_##mode,blocksize, \
b1ceb439
TS		 992 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
993 ivlen, \
0f113f3e
MC		 994 flags|EVP_CIPH_##MODE##_MODE, \
995 aes_##mode##_init_key, \
996 aes_##mode##_cipher, \
997 aes_##mode##_cleanup, \
998 sizeof(EVP_AES_##MODE##_CTX), \
999 NULL,NULL,aes_##mode##_ctrl,NULL }; \
c5f6da54
AP		 1000const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
1001{ return SPARC_AES_CAPABLE?&aes_t4_##keylen##_##mode:&aes_##keylen##_##mode; }
1002
96530eea
PS		 1003#elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
1004/*
1005 * IBM S390X support
1006 */
1007# include "s390x_arch.h"
1008
55bd169f
PS		 1009typedef struct {
1010 union {
1011 double align;
1012 /*-
1013 * KM-AES parameter block - begin
1014 * (see z/Architecture Principles of Operation >= SA22-7832-06)
1015 */
1016 struct {
1017 unsigned char k[32];
1018 } param;
1019 /* KM-AES parameter block - end */
1020 } km;
1021 unsigned int fc;
1022} S390X_AES_ECB_CTX;
1023
dacd2a87
PS		 1024typedef struct {
1025 union {
1026 double align;
1027 /*-
1028 * KMO-AES parameter block - begin
1029 * (see z/Architecture Principles of Operation >= SA22-7832-08)
1030 */
1031 struct {
1032 unsigned char cv[16];
1033 unsigned char k[32];
1034 } param;
1035 /* KMO-AES parameter block - end */
1036 } kmo;
1037 unsigned int fc;
1038
1039 int res;
1040} S390X_AES_OFB_CTX;
1041
74d38a86
PS		 1042typedef struct {
1043 union {
1044 double align;
1045 /*-
1046 * KMF-AES parameter block - begin
1047 * (see z/Architecture Principles of Operation >= SA22-7832-08)
1048 */
1049 struct {
1050 unsigned char cv[16];
1051 unsigned char k[32];
1052 } param;
1053 /* KMF-AES parameter block - end */
1054 } kmf;
1055 unsigned int fc;
1056
1057 int res;
1058} S390X_AES_CFB_CTX;
1059
96530eea
PS		 1060typedef struct {
1061 union {
1062 double align;
1063 /*-
5d2a6f4b
PS		 1064 * KMA-GCM-AES parameter block - begin
1065 * (see z/Architecture Principles of Operation >= SA22-7832-11)
96530eea
PS		 1066 */
1067 struct {
1068 unsigned char reserved[12];
1069 union {
1070 unsigned int w;
1071 unsigned char b[4];
1072 } cv;
1073 union {
1074 unsigned long long g[2];
1075 unsigned char b[16];
1076 } t;
1077 unsigned char h[16];
1078 unsigned long long taadl;
1079 unsigned long long tpcl;
1080 union {
1081 unsigned long long g[2];
1082 unsigned int w[4];
1083 } j0;
1084 unsigned char k[32];
1085 } param;
5d2a6f4b 1086 /* KMA-GCM-AES parameter block - end */
96530eea
PS		 1087 } kma;
1088 unsigned int fc;
1089 int key_set;
1090
1091 unsigned char *iv;
1092 int ivlen;
1093 int iv_set;
1094 int iv_gen;
1095
1096 int taglen;
1097
1098 unsigned char ares[16];
1099 unsigned char mres[16];
1100 unsigned char kres[16];
1101 int areslen;
1102 int mreslen;
1103 int kreslen;
1104
1105 int tls_aad_len;
d6b34570 1106 uint64_t tls_enc_records; /* Number of TLS records encrypted */
96530eea
PS		 1107} S390X_AES_GCM_CTX;
1108
39f5b069
PS		 1109typedef struct {
1110 union {
1111 double align;
1112 /*-
1113 * Padding is chosen so that ccm.kmac_param.k overlaps with key.k and
1114 * ccm.fc with key.k.rounds. Remember that on s390x, an AES_KEY's
1115 * rounds field is used to store the function code and that the key
1116 * schedule is not stored (if aes hardware support is detected).
1117 */
1118 struct {
1119 unsigned char pad[16];
1120 AES_KEY k;
1121 } key;
1122
1123 struct {
1124 /*-
1125 * KMAC-AES parameter block - begin
1126 * (see z/Architecture Principles of Operation >= SA22-7832-08)
1127 */
1128 struct {
1129 union {
1130 unsigned long long g[2];
1131 unsigned char b[16];
1132 } icv;
1133 unsigned char k[32];
1134 } kmac_param;
1135 /* KMAC-AES paramater block - end */
1136
1137 union {
1138 unsigned long long g[2];
1139 unsigned char b[16];
1140 } nonce;
1141 union {
1142 unsigned long long g[2];
1143 unsigned char b[16];
1144 } buf;
1145
1146 unsigned long long blocks;
1147 int l;
1148 int m;
1149 int tls_aad_len;
1150 int iv_set;
1151 int tag_set;
1152 int len_set;
1153 int key_set;
1154
1155 unsigned char pad[140];
1156 unsigned int fc;
1157 } ccm;
1158 } aes;
1159} S390X_AES_CCM_CTX;
1160
55bd169f
PS		 1161/* Convert key size to function code: [16,24,32] -> [18,19,20]. */
1162# define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
1163
1164/* Most modes of operation need km for partial block processing. */
1165# define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
1166 S390X_CAPBIT(S390X_AES_128))
1167# define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
1168 S390X_CAPBIT(S390X_AES_192))
1169# define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
1170 S390X_CAPBIT(S390X_AES_256))
96530eea
PS		 1171
1172# define s390x_aes_init_key aes_init_key
1173static int s390x_aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
1174 const unsigned char *iv, int enc);
1175
1176# define S390X_aes_128_cbc_CAPABLE 1 /* checked by callee */
1177# define S390X_aes_192_cbc_CAPABLE 1
1178# define S390X_aes_256_cbc_CAPABLE 1
55bd169f
PS		 1179# define S390X_AES_CBC_CTX EVP_AES_KEY
1180
1181# define s390x_aes_cbc_init_key aes_init_key
96530eea
PS		 1182
1183# define s390x_aes_cbc_cipher aes_cbc_cipher
1184static int s390x_aes_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1185 const unsigned char *in, size_t len);
1186
55bd169f
PS		 1187# define S390X_aes_128_ecb_CAPABLE S390X_aes_128_CAPABLE
1188# define S390X_aes_192_ecb_CAPABLE S390X_aes_192_CAPABLE
1189# define S390X_aes_256_ecb_CAPABLE S390X_aes_256_CAPABLE
1190
1191static int s390x_aes_ecb_init_key(EVP_CIPHER_CTX *ctx,
1192 const unsigned char *key,
1193 const unsigned char *iv, int enc)
1194{
1195 S390X_AES_ECB_CTX *cctx = EVP_C_DATA(S390X_AES_ECB_CTX, ctx);
1196 const int keylen = EVP_CIPHER_CTX_key_length(ctx);
1197
1198 cctx->fc = S390X_AES_FC(keylen);
1199 if (!enc)
1200 cctx->fc |= S390X_DECRYPT;
1201
1202 memcpy(cctx->km.param.k, key, keylen);
1203 return 1;
1204}
96530eea 1205
96530eea 1206static int s390x_aes_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
55bd169f
PS		 1207 const unsigned char *in, size_t len)
1208{
1209 S390X_AES_ECB_CTX *cctx = EVP_C_DATA(S390X_AES_ECB_CTX, ctx);
1210
1211 s390x_km(in, len, out, cctx->fc, &cctx->km.param);
1212 return 1;
1213}
96530eea 1214
dacd2a87
PS		 1215# define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE && \
1216 (OPENSSL_s390xcap_P.kmo[0] & \
1217 S390X_CAPBIT(S390X_AES_128)))
1218# define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE && \
1219 (OPENSSL_s390xcap_P.kmo[0] & \
1220 S390X_CAPBIT(S390X_AES_192)))
1221# define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE && \
1222 (OPENSSL_s390xcap_P.kmo[0] & \
1223 S390X_CAPBIT(S390X_AES_256)))
1224
1225static int s390x_aes_ofb_init_key(EVP_CIPHER_CTX *ctx,
1226 const unsigned char *key,
1227 const unsigned char *ivec, int enc)
1228{
1229 S390X_AES_OFB_CTX *cctx = EVP_C_DATA(S390X_AES_OFB_CTX, ctx);
1230 const unsigned char *iv = EVP_CIPHER_CTX_original_iv(ctx);
1231 const int keylen = EVP_CIPHER_CTX_key_length(ctx);
1232 const int ivlen = EVP_CIPHER_CTX_iv_length(ctx);
55bd169f 1233
dacd2a87
PS		 1234 memcpy(cctx->kmo.param.cv, iv, ivlen);
1235 memcpy(cctx->kmo.param.k, key, keylen);
1236 cctx->fc = S390X_AES_FC(keylen);
1237 cctx->res = 0;
1238 return 1;
1239}
96530eea 1240
96530eea 1241static int s390x_aes_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
dacd2a87
PS		 1242 const unsigned char *in, size_t len)
1243{
1244 S390X_AES_OFB_CTX *cctx = EVP_C_DATA(S390X_AES_OFB_CTX, ctx);
1245 int n = cctx->res;
1246 int rem;
1247
1248 while (n && len) {
1249 *out = *in ^ cctx->kmo.param.cv[n];
1250 n = (n + 1) & 0xf;
1251 --len;
1252 ++in;
1253 ++out;
1254 }
1255
1256 rem = len & 0xf;
1257
1258 len &= ~(size_t)0xf;
1259 if (len) {
1260 s390x_kmo(in, len, out, cctx->fc, &cctx->kmo.param);
1261
1262 out += len;
1263 in += len;
1264 }
1265
1266 if (rem) {
1267 s390x_km(cctx->kmo.param.cv, 16, cctx->kmo.param.cv, cctx->fc,
1268 cctx->kmo.param.k);
1269
1270 while (rem--) {
1271 out[n] = in[n] ^ cctx->kmo.param.cv[n];
1272 ++n;
1273 }
1274 }
1275
1276 cctx->res = n;
1277 return 1;
1278}
96530eea 1279
74d38a86
PS		 1280# define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE && \
1281 (OPENSSL_s390xcap_P.kmf[0] & \
1282 S390X_CAPBIT(S390X_AES_128)))
1283# define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE && \
1284 (OPENSSL_s390xcap_P.kmf[0] & \
1285 S390X_CAPBIT(S390X_AES_192)))
1286# define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE && \
1287 (OPENSSL_s390xcap_P.kmf[0] & \
1288 S390X_CAPBIT(S390X_AES_256)))
1289
1290static int s390x_aes_cfb_init_key(EVP_CIPHER_CTX *ctx,
1291 const unsigned char *key,
1292 const unsigned char *ivec, int enc)
1293{
1294 S390X_AES_CFB_CTX *cctx = EVP_C_DATA(S390X_AES_CFB_CTX, ctx);
1295 const unsigned char *iv = EVP_CIPHER_CTX_original_iv(ctx);
1296 const int keylen = EVP_CIPHER_CTX_key_length(ctx);
1297 const int ivlen = EVP_CIPHER_CTX_iv_length(ctx);
1298
1299 cctx->fc = S390X_AES_FC(keylen);
1300 cctx->fc |= 16 << 24; /* 16 bytes cipher feedback */
1301 if (!enc)
1302 cctx->fc |= S390X_DECRYPT;
55bd169f 1303
74d38a86
PS		 1304 cctx->res = 0;
1305 memcpy(cctx->kmf.param.cv, iv, ivlen);
1306 memcpy(cctx->kmf.param.k, key, keylen);
1307 return 1;
1308}
96530eea 1309
96530eea 1310static int s390x_aes_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
74d38a86
PS		 1311 const unsigned char *in, size_t len)
1312{
1313 S390X_AES_CFB_CTX *cctx = EVP_C_DATA(S390X_AES_CFB_CTX, ctx);
1314 const int keylen = EVP_CIPHER_CTX_key_length(ctx);
1315 const int enc = EVP_CIPHER_CTX_encrypting(ctx);
1316 int n = cctx->res;
1317 int rem;
1318 unsigned char tmp;
1319
1320 while (n && len) {
1321 tmp = *in;
1322 *out = cctx->kmf.param.cv[n] ^ tmp;
1323 cctx->kmf.param.cv[n] = enc ? *out : tmp;
1324 n = (n + 1) & 0xf;
1325 --len;
1326 ++in;
1327 ++out;
1328 }
1329
1330 rem = len & 0xf;
1331
1332 len &= ~(size_t)0xf;
1333 if (len) {
1334 s390x_kmf(in, len, out, cctx->fc, &cctx->kmf.param);
1335
1336 out += len;
1337 in += len;
1338 }
1339
1340 if (rem) {
1341 s390x_km(cctx->kmf.param.cv, 16, cctx->kmf.param.cv,
1342 S390X_AES_FC(keylen), cctx->kmf.param.k);
1343
1344 while (rem--) {
1345 tmp = in[n];
1346 out[n] = cctx->kmf.param.cv[n] ^ tmp;
1347 cctx->kmf.param.cv[n] = enc ? out[n] : tmp;
1348 ++n;
1349 }
1350 }
96530eea 1351
74d38a86
PS		 1352 cctx->res = n;
1353 return 1;
1354}
1355
1356# define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
1357 S390X_CAPBIT(S390X_AES_128))
1358# define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
1359 S390X_CAPBIT(S390X_AES_192))
1360# define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
1361 S390X_CAPBIT(S390X_AES_256))
1362
1363static int s390x_aes_cfb8_init_key(EVP_CIPHER_CTX *ctx,
1364 const unsigned char *key,
1365 const unsigned char *ivec, int enc)
1366{
1367 S390X_AES_CFB_CTX *cctx = EVP_C_DATA(S390X_AES_CFB_CTX, ctx);
1368 const unsigned char *iv = EVP_CIPHER_CTX_original_iv(ctx);
1369 const int keylen = EVP_CIPHER_CTX_key_length(ctx);
1370 const int ivlen = EVP_CIPHER_CTX_iv_length(ctx);
1371
1372 cctx->fc = S390X_AES_FC(keylen);
1373 cctx->fc |= 1 << 24; /* 1 byte cipher feedback */
1374 if (!enc)
1375 cctx->fc |= S390X_DECRYPT;
96530eea 1376
74d38a86
PS		 1377 memcpy(cctx->kmf.param.cv, iv, ivlen);
1378 memcpy(cctx->kmf.param.k, key, keylen);
1379 return 1;
1380}
55bd169f 1381
96530eea 1382static int s390x_aes_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
74d38a86
PS		 1383 const unsigned char *in, size_t len)
1384{
1385 S390X_AES_CFB_CTX *cctx = EVP_C_DATA(S390X_AES_CFB_CTX, ctx);
1386
1387 s390x_kmf(in, len, out, cctx->fc, &cctx->kmf.param);
1388 return 1;
1389}
96530eea
PS		 1390
1391# define S390X_aes_128_cfb1_CAPABLE 0
1392# define S390X_aes_192_cfb1_CAPABLE 0
1393# define S390X_aes_256_cfb1_CAPABLE 0
1394
55bd169f
PS		 1395# define s390x_aes_cfb1_init_key aes_init_key
1396
96530eea
PS		 1397# define s390x_aes_cfb1_cipher aes_cfb1_cipher
1398static int s390x_aes_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1399 const unsigned char *in, size_t len);
1400
1401# define S390X_aes_128_ctr_CAPABLE 1 /* checked by callee */
1402# define S390X_aes_192_ctr_CAPABLE 1
1403# define S390X_aes_256_ctr_CAPABLE 1
55bd169f
PS		 1404# define S390X_AES_CTR_CTX EVP_AES_KEY
1405
1406# define s390x_aes_ctr_init_key aes_init_key
96530eea
PS		 1407
1408# define s390x_aes_ctr_cipher aes_ctr_cipher
1409static int s390x_aes_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1410 const unsigned char *in, size_t len);
1411
1412# define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE && \
1413 (OPENSSL_s390xcap_P.kma[0] & \
1414 S390X_CAPBIT(S390X_AES_128)))
1415# define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE && \
1416 (OPENSSL_s390xcap_P.kma[0] & \
1417 S390X_CAPBIT(S390X_AES_192)))
1418# define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE && \
1419 (OPENSSL_s390xcap_P.kma[0] & \
1420 S390X_CAPBIT(S390X_AES_256)))
1421
bcf082d1 1422/* iv + padding length for iv lengths != 12 */
96530eea
PS		 1423# define S390X_gcm_ivpadlen(i) ((((i) + 15) >> 4 << 4) + 16)
1424
5d2a6f4b
PS		 1425/*-
1426 * Process additional authenticated data. Returns 0 on success. Code is
1427 * big-endian.
1428 */
96530eea
PS		 1429static int s390x_aes_gcm_aad(S390X_AES_GCM_CTX *ctx, const unsigned char *aad,
1430 size_t len)
1431{
1432 unsigned long long alen;
1433 int n, rem;
1434
1435 if (ctx->kma.param.tpcl)
1436 return -2;
1437
1438 alen = ctx->kma.param.taadl + len;
1439 if (alen > (U64(1) << 61) || (sizeof(len) == 8 && alen < len))
1440 return -1;
1441 ctx->kma.param.taadl = alen;
1442
1443 n = ctx->areslen;
1444 if (n) {
1445 while (n && len) {
1446 ctx->ares[n] = *aad;
1447 n = (n + 1) & 0xf;
1448 ++aad;
1449 --len;
1450 }
1451 /* ctx->ares contains a complete block if offset has wrapped around */
1452 if (!n) {
1453 s390x_kma(ctx->ares, 16, NULL, 0, NULL, ctx->fc, &ctx->kma.param);
1454 ctx->fc |= S390X_KMA_HS;
1455 }
1456 ctx->areslen = n;
1457 }
1458
1459 rem = len & 0xf;
1460
25868993 1461 len &= ~(size_t)0xf;
96530eea
PS		 1462 if (len) {
1463 s390x_kma(aad, len, NULL, 0, NULL, ctx->fc, &ctx->kma.param);
1464 aad += len;
1465 ctx->fc |= S390X_KMA_HS;
1466 }
1467
1468 if (rem) {
1469 ctx->areslen = rem;
1470
1471 do {
1472 --rem;
1473 ctx->ares[rem] = aad[rem];
1474 } while (rem);
1475 }
1476 return 0;
1477}
1478
5d2a6f4b
PS		 1479/*-
1480 * En/de-crypt plain/cipher-text and authenticate ciphertext. Returns 0 for
1481 * success. Code is big-endian.
1482 */
96530eea
PS		 1483static int s390x_aes_gcm(S390X_AES_GCM_CTX *ctx, const unsigned char *in,
1484 unsigned char *out, size_t len)
1485{
1486 const unsigned char *inptr;
1487 unsigned long long mlen;
1488 union {
1489 unsigned int w[4];
1490 unsigned char b[16];
1491 } buf;
1492 size_t inlen;
1493 int n, rem, i;
1494
1495 mlen = ctx->kma.param.tpcl + len;
1496 if (mlen > ((U64(1) << 36) - 32) || (sizeof(len) == 8 && mlen < len))
1497 return -1;
1498 ctx->kma.param.tpcl = mlen;
1499
1500 n = ctx->mreslen;
1501 if (n) {
1502 inptr = in;
1503 inlen = len;
1504 while (n && inlen) {
1505 ctx->mres[n] = *inptr;
1506 n = (n + 1) & 0xf;
1507 ++inptr;
1508 --inlen;
1509 }
1510 /* ctx->mres contains a complete block if offset has wrapped around */
1511 if (!n) {
1512 s390x_kma(ctx->ares, ctx->areslen, ctx->mres, 16, buf.b,
1513 ctx->fc | S390X_KMA_LAAD, &ctx->kma.param);
1514 ctx->fc |= S390X_KMA_HS;
1515 ctx->areslen = 0;
1516
1517 /* previous call already encrypted/decrypted its remainder,
1518 * see comment below */
1519 n = ctx->mreslen;
1520 while (n) {
1521 *out = buf.b[n];
1522 n = (n + 1) & 0xf;
1523 ++out;
1524 ++in;
1525 --len;
1526 }
1527 ctx->mreslen = 0;
1528 }
1529 }
1530
1531 rem = len & 0xf;
1532
25868993 1533 len &= ~(size_t)0xf;
96530eea
PS		 1534 if (len) {
1535 s390x_kma(ctx->ares, ctx->areslen, in, len, out,
1536 ctx->fc | S390X_KMA_LAAD, &ctx->kma.param);
1537 in += len;
1538 out += len;
1539 ctx->fc |= S390X_KMA_HS;
1540 ctx->areslen = 0;
1541 }
1542
1543 /*-
1544 * If there is a remainder, it has to be saved such that it can be
1545 * processed by kma later. However, we also have to do the for-now
1546 * unauthenticated encryption/decryption part here and now...
1547 */
1548 if (rem) {
1549 if (!ctx->mreslen) {
1550 buf.w[0] = ctx->kma.param.j0.w[0];
1551 buf.w[1] = ctx->kma.param.j0.w[1];
1552 buf.w[2] = ctx->kma.param.j0.w[2];
1553 buf.w[3] = ctx->kma.param.cv.w + 1;
1554 s390x_km(buf.b, 16, ctx->kres, ctx->fc & 0x1f, &ctx->kma.param.k);
1555 }
1556
1557 n = ctx->mreslen;
1558 for (i = 0; i < rem; i++) {
1559 ctx->mres[n + i] = in[i];
1560 out[i] = in[i] ^ ctx->kres[n + i];
1561 }
1562
1563 ctx->mreslen += rem;
1564 }
1565 return 0;
1566}
1567
5d2a6f4b
PS		 1568/*-
1569 * Initialize context structure. Code is big-endian.
1570 */
96530eea
PS		 1571static void s390x_aes_gcm_setiv(S390X_AES_GCM_CTX *ctx,
1572 const unsigned char *iv)
1573{
1574 ctx->kma.param.t.g[0] = 0;
1575 ctx->kma.param.t.g[1] = 0;
1576 ctx->kma.param.tpcl = 0;
1577 ctx->kma.param.taadl = 0;
1578 ctx->mreslen = 0;
1579 ctx->areslen = 0;
1580 ctx->kreslen = 0;
1581
1582 if (ctx->ivlen == 12) {
1583 memcpy(&ctx->kma.param.j0, iv, ctx->ivlen);
1584 ctx->kma.param.j0.w[3] = 1;
1585 ctx->kma.param.cv.w = 1;
1586 } else {
1587 /* ctx->iv has the right size and is already padded. */
1588 memcpy(ctx->iv, iv, ctx->ivlen);
1589 s390x_kma(ctx->iv, S390X_gcm_ivpadlen(ctx->ivlen), NULL, 0, NULL,
1590 ctx->fc, &ctx->kma.param);
1591 ctx->fc |= S390X_KMA_HS;
1592
1593 ctx->kma.param.j0.g[0] = ctx->kma.param.t.g[0];
1594 ctx->kma.param.j0.g[1] = ctx->kma.param.t.g[1];
1595 ctx->kma.param.cv.w = ctx->kma.param.j0.w[3];
1596 ctx->kma.param.t.g[0] = 0;
1597 ctx->kma.param.t.g[1] = 0;
1598 }
1599}
1600
5d2a6f4b
PS		 1601/*-
1602 * Performs various operations on the context structure depending on control
1603 * type. Returns 1 for success, 0 for failure and -1 for unknown control type.
1604 * Code is big-endian.
1605 */
96530eea
PS		 1606static int s390x_aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
1607{
1608 S390X_AES_GCM_CTX *gctx = EVP_C_DATA(S390X_AES_GCM_CTX, c);
1609 S390X_AES_GCM_CTX *gctx_out;
1610 EVP_CIPHER_CTX *out;
1611 unsigned char *buf, *iv;
1612 int ivlen, enc, len;
1613
1614 switch (type) {
1615 case EVP_CTRL_INIT:
1616 ivlen = EVP_CIPHER_CTX_iv_length(c);
1617 iv = EVP_CIPHER_CTX_iv_noconst(c);
1618 gctx->key_set = 0;
1619 gctx->iv_set = 0;
1620 gctx->ivlen = ivlen;
1621 gctx->iv = iv;
1622 gctx->taglen = -1;
1623 gctx->iv_gen = 0;
1624 gctx->tls_aad_len = -1;
1625 return 1;
1626
1627 case EVP_CTRL_AEAD_SET_IVLEN:
1628 if (arg <= 0)
1629 return 0;
1630
1631 if (arg != 12) {
1632 iv = EVP_CIPHER_CTX_iv_noconst(c);
1633 len = S390X_gcm_ivpadlen(arg);
1634
1635 /* Allocate memory for iv if needed. */
1636 if (gctx->ivlen == 12 || len > S390X_gcm_ivpadlen(gctx->ivlen)) {
1637 if (gctx->iv != iv)
1638 OPENSSL_free(gctx->iv);
1639
cdb10bae
RS		 1640 if ((gctx->iv = OPENSSL_malloc(len)) == NULL) {
1641 EVPerr(EVP_F_S390X_AES_GCM_CTRL, ERR_R_MALLOC_FAILURE);
96530eea 1642 return 0;
cdb10bae 1643 }
96530eea
PS		 1644 }
1645 /* Add padding. */
1646 memset(gctx->iv + arg, 0, len - arg - 8);
1647 *((unsigned long long *)(gctx->iv + len - 8)) = arg << 3;
1648 }
1649 gctx->ivlen = arg;
1650 return 1;
1651
1652 case EVP_CTRL_AEAD_SET_TAG:
1653 buf = EVP_CIPHER_CTX_buf_noconst(c);
1654 enc = EVP_CIPHER_CTX_encrypting(c);
1655 if (arg <= 0 || arg > 16 || enc)
1656 return 0;
1657
1658 memcpy(buf, ptr, arg);
1659 gctx->taglen = arg;
1660 return 1;
1661
1662 case EVP_CTRL_AEAD_GET_TAG:
1663 enc = EVP_CIPHER_CTX_encrypting(c);
1664 if (arg <= 0 || arg > 16 || !enc || gctx->taglen < 0)
1665 return 0;
1666
1667 memcpy(ptr, gctx->kma.param.t.b, arg);
1668 return 1;
1669
1670 case EVP_CTRL_GCM_SET_IV_FIXED:
1671 /* Special case: -1 length restores whole iv */
1672 if (arg == -1) {
1673 memcpy(gctx->iv, ptr, gctx->ivlen);
1674 gctx->iv_gen = 1;
1675 return 1;
1676 }
1677 /*
1678 * Fixed field must be at least 4 bytes and invocation field at least
1679 * 8.
1680 */
1681 if ((arg < 4) || (gctx->ivlen - arg) < 8)
1682 return 0;
1683
1684 if (arg)
1685 memcpy(gctx->iv, ptr, arg);
1686
1687 enc = EVP_CIPHER_CTX_encrypting(c);
16cfc2c9
KR		 1688 if (enc && RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0)
1689 return 0;
96530eea
PS		 1690
1691 gctx->iv_gen = 1;
1692 return 1;
1693
1694 case EVP_CTRL_GCM_IV_GEN:
1695 if (gctx->iv_gen == 0 || gctx->key_set == 0)
1696 return 0;
1697
1698 s390x_aes_gcm_setiv(gctx, gctx->iv);
1699
1700 if (arg <= 0 || arg > gctx->ivlen)
1701 arg = gctx->ivlen;
1702
1703 memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg);
1704 /*
1705 * Invocation field will be at least 8 bytes in size and so no need
1706 * to check wrap around or increment more than last 8 bytes.
1707 */
03a5e5ae 1708 ctr64_inc(gctx->iv + gctx->ivlen - 8);
96530eea
PS		 1709 gctx->iv_set = 1;
1710 return 1;
1711
1712 case EVP_CTRL_GCM_SET_IV_INV:
1713 enc = EVP_CIPHER_CTX_encrypting(c);
1714 if (gctx->iv_gen == 0 || gctx->key_set == 0 || enc)
1715 return 0;
1716
1717 memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg);
1718 s390x_aes_gcm_setiv(gctx, gctx->iv);
1719 gctx->iv_set = 1;
1720 return 1;
1721
1722 case EVP_CTRL_AEAD_TLS1_AAD:
1723 /* Save the aad for later use. */
1724 if (arg != EVP_AEAD_TLS1_AAD_LEN)
1725 return 0;
1726
1727 buf = EVP_CIPHER_CTX_buf_noconst(c);
1728 memcpy(buf, ptr, arg);
1729 gctx->tls_aad_len = arg;
d6b34570 1730 gctx->tls_enc_records = 0;
96530eea
PS		 1731
1732 len = buf[arg - 2] << 8 | buf[arg - 1];
1733 /* Correct length for explicit iv. */
1734 if (len < EVP_GCM_TLS_EXPLICIT_IV_LEN)
1735 return 0;
1736 len -= EVP_GCM_TLS_EXPLICIT_IV_LEN;
1737
1738 /* If decrypting correct for tag too. */
1739 enc = EVP_CIPHER_CTX_encrypting(c);
1740 if (!enc) {
1741 if (len < EVP_GCM_TLS_TAG_LEN)
1742 return 0;
1743 len -= EVP_GCM_TLS_TAG_LEN;
1744 }
1745 buf[arg - 2] = len >> 8;
1746 buf[arg - 1] = len & 0xff;
1747 /* Extra padding: tag appended to record. */
1748 return EVP_GCM_TLS_TAG_LEN;
1749
1750 case EVP_CTRL_COPY:
1751 out = ptr;
1752 gctx_out = EVP_C_DATA(S390X_AES_GCM_CTX, out);
1753 iv = EVP_CIPHER_CTX_iv_noconst(c);
1754
1755 if (gctx->iv == iv) {
1756 gctx_out->iv = EVP_CIPHER_CTX_iv_noconst(out);
1757 } else {
1758 len = S390X_gcm_ivpadlen(gctx->ivlen);
1759
cdb10bae
RS		 1760 if ((gctx_out->iv = OPENSSL_malloc(len)) == NULL) {
1761 EVPerr(EVP_F_S390X_AES_GCM_CTRL, ERR_R_MALLOC_FAILURE);
96530eea 1762 return 0;
cdb10bae 1763 }
96530eea
PS		 1764
1765 memcpy(gctx_out->iv, gctx->iv, len);
1766 }
1767 return 1;
1768
1769 default:
1770 return -1;
1771 }
1772}
1773
5d2a6f4b
PS		 1774/*-
1775 * Set key and/or iv. Returns 1 on success. Otherwise 0 is returned.
1776 */
96530eea
PS		 1777static int s390x_aes_gcm_init_key(EVP_CIPHER_CTX *ctx,
1778 const unsigned char *key,
1779 const unsigned char *iv, int enc)
1780{
1781 S390X_AES_GCM_CTX *gctx = EVP_C_DATA(S390X_AES_GCM_CTX, ctx);
1782 int keylen;
1783
1784 if (iv == NULL && key == NULL)
1785 return 1;
1786
1787 if (key != NULL) {
1788 keylen = EVP_CIPHER_CTX_key_length(ctx);
1789 memcpy(&gctx->kma.param.k, key, keylen);
1790
8eb399fb 1791 gctx->fc = S390X_AES_FC(keylen);
96530eea
PS		 1792 if (!enc)
1793 gctx->fc |= S390X_DECRYPT;
1794
1795 if (iv == NULL && gctx->iv_set)
1796 iv = gctx->iv;
1797
1798 if (iv != NULL) {
1799 s390x_aes_gcm_setiv(gctx, iv);
1800 gctx->iv_set = 1;
1801 }
1802 gctx->key_set = 1;
1803 } else {
1804 if (gctx->key_set)
1805 s390x_aes_gcm_setiv(gctx, iv);
1806 else
1807 memcpy(gctx->iv, iv, gctx->ivlen);
1808
1809 gctx->iv_set = 1;
1810 gctx->iv_gen = 0;
1811 }
1812 return 1;
1813}
1814
5d2a6f4b
PS		 1815/*-
1816 * En/de-crypt and authenticate TLS packet. Returns the number of bytes written
1817 * if successful. Otherwise -1 is returned. Code is big-endian.
1818 */
96530eea
PS		 1819static int s390x_aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1820 const unsigned char *in, size_t len)
1821{
1822 S390X_AES_GCM_CTX *gctx = EVP_C_DATA(S390X_AES_GCM_CTX, ctx);
1823 const unsigned char *buf = EVP_CIPHER_CTX_buf_noconst(ctx);
1824 const int enc = EVP_CIPHER_CTX_encrypting(ctx);
1825 int rv = -1;
1826
1827 if (out != in || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN))
1828 return -1;
1829
d6b34570
P		 1830 /*
1831 * Check for too many keys as per FIPS 140-2 IG A.5 "Key/IV Pair Uniqueness
1832 * Requirements from SP 800-38D". The requirements is for one party to the
1833 * communication to fail after 2^64 - 1 keys. We do this on the encrypting
1834 * side only.
1835 */
1836 if (ctx->encrypt && ++gctx->tls_enc_records == 0) {
1837 EVPerr(EVP_F_S390X_AES_GCM_TLS_CIPHER, EVP_R_TOO_MANY_RECORDS);
1838 goto err;
1839 }
1840
96530eea
PS		 1841 if (EVP_CIPHER_CTX_ctrl(ctx, enc ? EVP_CTRL_GCM_IV_GEN
1842 : EVP_CTRL_GCM_SET_IV_INV,
1843 EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0)
1844 goto err;
1845
1846 in += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1847 out += EVP_GCM_TLS_EXPLICIT_IV_LEN;
1848 len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1849
1850 gctx->kma.param.taadl = gctx->tls_aad_len << 3;
1851 gctx->kma.param.tpcl = len << 3;
1852 s390x_kma(buf, gctx->tls_aad_len, in, len, out,
1853 gctx->fc | S390X_KMA_LAAD | S390X_KMA_LPC, &gctx->kma.param);
1854
1855 if (enc) {
1856 memcpy(out + len, gctx->kma.param.t.b, EVP_GCM_TLS_TAG_LEN);
1857 rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
1858 } else {
1859 if (CRYPTO_memcmp(gctx->kma.param.t.b, in + len,
1860 EVP_GCM_TLS_TAG_LEN)) {
1861 OPENSSL_cleanse(out, len);
1862 goto err;
1863 }
1864 rv = len;
1865 }
1866err:
1867 gctx->iv_set = 0;
1868 gctx->tls_aad_len = -1;
1869 return rv;
1870}
1871
5d2a6f4b
PS		 1872/*-
1873 * Called from EVP layer to initialize context, process additional
1874 * authenticated data, en/de-crypt plain/cipher-text and authenticate
1875 * ciphertext or process a TLS packet, depending on context. Returns bytes
1876 * written on success. Otherwise -1 is returned. Code is big-endian.
1877 */
96530eea
PS		 1878static int s390x_aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1879 const unsigned char *in, size_t len)
1880{
1881 S390X_AES_GCM_CTX *gctx = EVP_C_DATA(S390X_AES_GCM_CTX, ctx);
1882 unsigned char *buf, tmp[16];
1883 int enc;
1884
1885 if (!gctx->key_set)
1886 return -1;
1887
1888 if (gctx->tls_aad_len >= 0)
1889 return s390x_aes_gcm_tls_cipher(ctx, out, in, len);
1890
1891 if (!gctx->iv_set)
1892 return -1;
1893
1894 if (in != NULL) {
1895 if (out == NULL) {
1896 if (s390x_aes_gcm_aad(gctx, in, len))
1897 return -1;
1898 } else {
1899 if (s390x_aes_gcm(gctx, in, out, len))
1900 return -1;
1901 }
1902 return len;
1903 } else {
1904 gctx->kma.param.taadl <<= 3;
1905 gctx->kma.param.tpcl <<= 3;
1906 s390x_kma(gctx->ares, gctx->areslen, gctx->mres, gctx->mreslen, tmp,
1907 gctx->fc | S390X_KMA_LAAD | S390X_KMA_LPC, &gctx->kma.param);
1908 /* recall that we already did en-/decrypt gctx->mres
1909 * and returned it to caller... */
1910 OPENSSL_cleanse(tmp, gctx->mreslen);
1911 gctx->iv_set = 0;
1912
1913 enc = EVP_CIPHER_CTX_encrypting(ctx);
1914 if (enc) {
1915 gctx->taglen = 16;
1916 } else {
1917 if (gctx->taglen < 0)
1918 return -1;
1919
1920 buf = EVP_CIPHER_CTX_buf_noconst(ctx);
1921 if (CRYPTO_memcmp(buf, gctx->kma.param.t.b, gctx->taglen))
1922 return -1;
1923 }
1924 return 0;
1925 }
1926}
1927
1928static int s390x_aes_gcm_cleanup(EVP_CIPHER_CTX *c)
1929{
1930 S390X_AES_GCM_CTX *gctx = EVP_C_DATA(S390X_AES_GCM_CTX, c);
1931 const unsigned char *iv;
1932
1933 if (gctx == NULL)
1934 return 0;
1935
1936 iv = EVP_CIPHER_CTX_iv(c);
1937 if (iv != gctx->iv)
1938 OPENSSL_free(gctx->iv);
1939
1940 OPENSSL_cleanse(gctx, sizeof(*gctx));
1941 return 1;
1942}
1943
1944# define S390X_AES_XTS_CTX EVP_AES_XTS_CTX
1945# define S390X_aes_128_xts_CAPABLE 1 /* checked by callee */
1946# define S390X_aes_256_xts_CAPABLE 1
1947
1948# define s390x_aes_xts_init_key aes_xts_init_key
1949static int s390x_aes_xts_init_key(EVP_CIPHER_CTX *ctx,
1950 const unsigned char *key,
1951 const unsigned char *iv, int enc);
1952# define s390x_aes_xts_cipher aes_xts_cipher
1953static int s390x_aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
1954 const unsigned char *in, size_t len);
1955# define s390x_aes_xts_ctrl aes_xts_ctrl
1956static int s390x_aes_xts_ctrl(EVP_CIPHER_CTX *, int type, int arg, void *ptr);
1957# define s390x_aes_xts_cleanup aes_xts_cleanup
1958
39f5b069
PS		 1959# define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE && \
1960 (OPENSSL_s390xcap_P.kmac[0] & \
1961 S390X_CAPBIT(S390X_AES_128)))
1962# define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE && \
1963 (OPENSSL_s390xcap_P.kmac[0] & \
1964 S390X_CAPBIT(S390X_AES_192)))
1965# define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE && \
1966 (OPENSSL_s390xcap_P.kmac[0] & \
1967 S390X_CAPBIT(S390X_AES_256)))
1968
1969# define S390X_CCM_AAD_FLAG 0x40
1970
1971/*-
1972 * Set nonce and length fields. Code is big-endian.
1973 */
1974static inline void s390x_aes_ccm_setiv(S390X_AES_CCM_CTX *ctx,
1975 const unsigned char *nonce,
1976 size_t mlen)
1977{
1978 ctx->aes.ccm.nonce.b[0] &= ~S390X_CCM_AAD_FLAG;
1979 ctx->aes.ccm.nonce.g[1] = mlen;
1980 memcpy(ctx->aes.ccm.nonce.b + 1, nonce, 15 - ctx->aes.ccm.l);
1981}
1982
1983/*-
1984 * Process additional authenticated data. Code is big-endian.
1985 */
1986static void s390x_aes_ccm_aad(S390X_AES_CCM_CTX *ctx, const unsigned char *aad,
1987 size_t alen)
1988{
1989 unsigned char *ptr;
1990 int i, rem;
1991
1992 if (!alen)
1993 return;
1994
1995 ctx->aes.ccm.nonce.b[0] |= S390X_CCM_AAD_FLAG;
1996
1997 /* Suppress 'type-punned pointer dereference' warning. */
1998 ptr = ctx->aes.ccm.buf.b;
1999
2000 if (alen < ((1 << 16) - (1 << 8))) {
2001 *(uint16_t *)ptr = alen;
2002 i = 2;
2003 } else if (sizeof(alen) == 8
2004 && alen >= (size_t)1 << (32 % (sizeof(alen) * 8))) {
2005 *(uint16_t *)ptr = 0xffff;
2006 *(uint64_t *)(ptr + 2) = alen;
2007 i = 10;
2008 } else {
2009 *(uint16_t *)ptr = 0xfffe;
2010 *(uint32_t *)(ptr + 2) = alen;
2011 i = 6;
2012 }
2013
2014 while (i < 16 && alen) {
2015 ctx->aes.ccm.buf.b[i] = *aad;
2016 ++aad;
2017 --alen;
2018 ++i;
2019 }
2020 while (i < 16) {
2021 ctx->aes.ccm.buf.b[i] = 0;
2022 ++i;
2023 }
2024
2025 ctx->aes.ccm.kmac_param.icv.g[0] = 0;
2026 ctx->aes.ccm.kmac_param.icv.g[1] = 0;
2027 s390x_kmac(ctx->aes.ccm.nonce.b, 32, ctx->aes.ccm.fc,
2028 &ctx->aes.ccm.kmac_param);
2029 ctx->aes.ccm.blocks += 2;
2030
2031 rem = alen & 0xf;
25868993 2032 alen &= ~(size_t)0xf;
39f5b069
PS		 2033 if (alen) {
2034 s390x_kmac(aad, alen, ctx->aes.ccm.fc, &ctx->aes.ccm.kmac_param);
2035 ctx->aes.ccm.blocks += alen >> 4;
2036 aad += alen;
2037 }
2038 if (rem) {
2039 for (i = 0; i < rem; i++)
2040 ctx->aes.ccm.kmac_param.icv.b[i] ^= aad[i];
2041
2042 s390x_km(ctx->aes.ccm.kmac_param.icv.b, 16,
2043 ctx->aes.ccm.kmac_param.icv.b, ctx->aes.ccm.fc,
2044 ctx->aes.ccm.kmac_param.k);
2045 ctx->aes.ccm.blocks++;
2046 }
2047}
2048
2049/*-
2050 * En/de-crypt plain/cipher-text. Compute tag from plaintext. Returns 0 for
2051 * success.
2052 */
2053static int s390x_aes_ccm(S390X_AES_CCM_CTX *ctx, const unsigned char *in,
2054 unsigned char *out, size_t len, int enc)
2055{
2056 size_t n, rem;
2057 unsigned int i, l, num;
2058 unsigned char flags;
2059
2060 flags = ctx->aes.ccm.nonce.b[0];
2061 if (!(flags & S390X_CCM_AAD_FLAG)) {
2062 s390x_km(ctx->aes.ccm.nonce.b, 16, ctx->aes.ccm.kmac_param.icv.b,
2063 ctx->aes.ccm.fc, ctx->aes.ccm.kmac_param.k);
2064 ctx->aes.ccm.blocks++;
2065 }
2066 l = flags & 0x7;
2067 ctx->aes.ccm.nonce.b[0] = l;
2068
2069 /*-
2070 * Reconstruct length from encoded length field
2071 * and initialize it with counter value.
2072 */
2073 n = 0;
2074 for (i = 15 - l; i < 15; i++) {
2075 n |= ctx->aes.ccm.nonce.b[i];
2076 ctx->aes.ccm.nonce.b[i] = 0;
2077 n <<= 8;
2078 }
2079 n |= ctx->aes.ccm.nonce.b[15];
2080 ctx->aes.ccm.nonce.b[15] = 1;
2081
2082 if (n != len)
2083 return -1; /* length mismatch */
2084
2085 if (enc) {
2086 /* Two operations per block plus one for tag encryption */
2087 ctx->aes.ccm.blocks += (((len + 15) >> 4) << 1) + 1;
2088 if (ctx->aes.ccm.blocks > (1ULL << 61))
2089 return -2; /* too much data */
2090 }
2091
2092 num = 0;
2093 rem = len & 0xf;
25868993 2094 len &= ~(size_t)0xf;
39f5b069
PS		 2095
2096 if (enc) {
2097 /* mac-then-encrypt */
2098 if (len)
2099 s390x_kmac(in, len, ctx->aes.ccm.fc, &ctx->aes.ccm.kmac_param);
2100 if (rem) {
2101 for (i = 0; i < rem; i++)
2102 ctx->aes.ccm.kmac_param.icv.b[i] ^= in[len + i];
2103
2104 s390x_km(ctx->aes.ccm.kmac_param.icv.b, 16,
2105 ctx->aes.ccm.kmac_param.icv.b, ctx->aes.ccm.fc,
2106 ctx->aes.ccm.kmac_param.k);
2107 }
2108
2109 CRYPTO_ctr128_encrypt_ctr32(in, out, len + rem, &ctx->aes.key.k,
2110 ctx->aes.ccm.nonce.b, ctx->aes.ccm.buf.b,
2111 &num, (ctr128_f)AES_ctr32_encrypt);
2112 } else {
2113 /* decrypt-then-mac */
2114 CRYPTO_ctr128_encrypt_ctr32(in, out, len + rem, &ctx->aes.key.k,
2115 ctx->aes.ccm.nonce.b, ctx->aes.ccm.buf.b,
2116 &num, (ctr128_f)AES_ctr32_encrypt);
2117
2118 if (len)
2119 s390x_kmac(out, len, ctx->aes.ccm.fc, &ctx->aes.ccm.kmac_param);
2120 if (rem) {
2121 for (i = 0; i < rem; i++)
2122 ctx->aes.ccm.kmac_param.icv.b[i] ^= out[len + i];
2123
2124 s390x_km(ctx->aes.ccm.kmac_param.icv.b, 16,
2125 ctx->aes.ccm.kmac_param.icv.b, ctx->aes.ccm.fc,
2126 ctx->aes.ccm.kmac_param.k);
2127 }
2128 }
2129 /* encrypt tag */
2130 for (i = 15 - l; i < 16; i++)
2131 ctx->aes.ccm.nonce.b[i] = 0;
2132
2133 s390x_km(ctx->aes.ccm.nonce.b, 16, ctx->aes.ccm.buf.b, ctx->aes.ccm.fc,
2134 ctx->aes.ccm.kmac_param.k);
2135 ctx->aes.ccm.kmac_param.icv.g[0] ^= ctx->aes.ccm.buf.g[0];
2136 ctx->aes.ccm.kmac_param.icv.g[1] ^= ctx->aes.ccm.buf.g[1];
2137
2138 ctx->aes.ccm.nonce.b[0] = flags; /* restore flags field */
2139 return 0;
2140}
2141
2142/*-
2143 * En/de-crypt and authenticate TLS packet. Returns the number of bytes written
2144 * if successful. Otherwise -1 is returned.
2145 */
2146static int s390x_aes_ccm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2147 const unsigned char *in, size_t len)
2148{
2149 S390X_AES_CCM_CTX *cctx = EVP_C_DATA(S390X_AES_CCM_CTX, ctx);
2150 unsigned char *ivec = EVP_CIPHER_CTX_iv_noconst(ctx);
2151 unsigned char *buf = EVP_CIPHER_CTX_buf_noconst(ctx);
2152 const int enc = EVP_CIPHER_CTX_encrypting(ctx);
2153
2154 if (out != in
2155 || len < (EVP_CCM_TLS_EXPLICIT_IV_LEN + (size_t)cctx->aes.ccm.m))
2156 return -1;
2157
2158 if (enc) {
2159 /* Set explicit iv (sequence number). */
2160 memcpy(out, buf, EVP_CCM_TLS_EXPLICIT_IV_LEN);
2161 }
2162
2163 len -= EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->aes.ccm.m;
2164 /*-
2165 * Get explicit iv (sequence number). We already have fixed iv
2166 * (server/client_write_iv) here.
2167 */
2168 memcpy(ivec + EVP_CCM_TLS_FIXED_IV_LEN, in, EVP_CCM_TLS_EXPLICIT_IV_LEN);
2169 s390x_aes_ccm_setiv(cctx, ivec, len);
2170
2171 /* Process aad (sequence number|type|version|length) */
2172 s390x_aes_ccm_aad(cctx, buf, cctx->aes.ccm.tls_aad_len);
2173
2174 in += EVP_CCM_TLS_EXPLICIT_IV_LEN;
2175 out += EVP_CCM_TLS_EXPLICIT_IV_LEN;
96530eea 2176
39f5b069
PS		 2177 if (enc) {
2178 if (s390x_aes_ccm(cctx, in, out, len, enc))
2179 return -1;
2180
2181 memcpy(out + len, cctx->aes.ccm.kmac_param.icv.b, cctx->aes.ccm.m);
2182 return len + EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->aes.ccm.m;
2183 } else {
2184 if (!s390x_aes_ccm(cctx, in, out, len, enc)) {
2185 if (!CRYPTO_memcmp(cctx->aes.ccm.kmac_param.icv.b, in + len,
2186 cctx->aes.ccm.m))
2187 return len;
2188 }
2189
2190 OPENSSL_cleanse(out, len);
2191 return -1;
2192 }
2193}
2194
2195/*-
2196 * Set key and flag field and/or iv. Returns 1 if successful. Otherwise 0 is
2197 * returned.
2198 */
96530eea
PS		 2199static int s390x_aes_ccm_init_key(EVP_CIPHER_CTX *ctx,
2200 const unsigned char *key,
39f5b069
PS		 2201 const unsigned char *iv, int enc)
2202{
2203 S390X_AES_CCM_CTX *cctx = EVP_C_DATA(S390X_AES_CCM_CTX, ctx);
2204 unsigned char *ivec;
2205 int keylen;
2206
2207 if (iv == NULL && key == NULL)
2208 return 1;
2209
2210 if (key != NULL) {
2211 keylen = EVP_CIPHER_CTX_key_length(ctx);
8eb399fb 2212 cctx->aes.ccm.fc = S390X_AES_FC(keylen);
39f5b069
PS		 2213 memcpy(cctx->aes.ccm.kmac_param.k, key, keylen);
2214
2215 /* Store encoded m and l. */
2216 cctx->aes.ccm.nonce.b[0] = ((cctx->aes.ccm.l - 1) & 0x7)
2217 | (((cctx->aes.ccm.m - 2) >> 1) & 0x7) << 3;
2218 memset(cctx->aes.ccm.nonce.b + 1, 0,
2219 sizeof(cctx->aes.ccm.nonce.b));
2220 cctx->aes.ccm.blocks = 0;
2221
2222 cctx->aes.ccm.key_set = 1;
2223 }
2224
2225 if (iv != NULL) {
2226 ivec = EVP_CIPHER_CTX_iv_noconst(ctx);
2227 memcpy(ivec, iv, 15 - cctx->aes.ccm.l);
2228
2229 cctx->aes.ccm.iv_set = 1;
2230 }
2231
2232 return 1;
2233}
2234
2235/*-
2236 * Called from EVP layer to initialize context, process additional
2237 * authenticated data, en/de-crypt plain/cipher-text and authenticate
2238 * plaintext or process a TLS packet, depending on context. Returns bytes
2239 * written on success. Otherwise -1 is returned.
2240 */
96530eea 2241static int s390x_aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
39f5b069
PS		 2242 const unsigned char *in, size_t len)
2243{
2244 S390X_AES_CCM_CTX *cctx = EVP_C_DATA(S390X_AES_CCM_CTX, ctx);
2245 const int enc = EVP_CIPHER_CTX_encrypting(ctx);
2246 int rv;
2247 unsigned char *buf, *ivec;
2248
2249 if (!cctx->aes.ccm.key_set)
2250 return -1;
2251
2252 if (cctx->aes.ccm.tls_aad_len >= 0)
2253 return s390x_aes_ccm_tls_cipher(ctx, out, in, len);
2254
2255 /*-
2256 * Final(): Does not return any data. Recall that ccm is mac-then-encrypt
2257 * so integrity must be checked already at Update() i.e., before
2258 * potentially corrupted data is output.
2259 */
2260 if (in == NULL && out != NULL)
2261 return 0;
2262
2263 if (!cctx->aes.ccm.iv_set)
2264 return -1;
2265
2266 if (!enc && !cctx->aes.ccm.tag_set)
2267 return -1;
2268
2269 if (out == NULL) {
2270 /* Update(): Pass message length. */
2271 if (in == NULL) {
2272 ivec = EVP_CIPHER_CTX_iv_noconst(ctx);
2273 s390x_aes_ccm_setiv(cctx, ivec, len);
2274
2275 cctx->aes.ccm.len_set = 1;
2276 return len;
2277 }
2278
2279 /* Update(): Process aad. */
2280 if (!cctx->aes.ccm.len_set && len)
2281 return -1;
2282
2283 s390x_aes_ccm_aad(cctx, in, len);
2284 return len;
2285 }
2286
2287 /* Update(): Process message. */
2288
2289 if (!cctx->aes.ccm.len_set) {
2290 /*-
46d08509 2291 * In case message length was not previously set explicitly via
39f5b069
PS		 2292 * Update(), set it now.
2293 */
2294 ivec = EVP_CIPHER_CTX_iv_noconst(ctx);
2295 s390x_aes_ccm_setiv(cctx, ivec, len);
2296
2297 cctx->aes.ccm.len_set = 1;
2298 }
2299
2300 if (enc) {
2301 if (s390x_aes_ccm(cctx, in, out, len, enc))
2302 return -1;
2303
2304 cctx->aes.ccm.tag_set = 1;
2305 return len;
2306 } else {
2307 rv = -1;
2308
2309 if (!s390x_aes_ccm(cctx, in, out, len, enc)) {
2310 buf = EVP_CIPHER_CTX_buf_noconst(ctx);
2311 if (!CRYPTO_memcmp(cctx->aes.ccm.kmac_param.icv.b, buf,
2312 cctx->aes.ccm.m))
2313 rv = len;
2314 }
2315
2316 if (rv == -1)
2317 OPENSSL_cleanse(out, len);
2318
2319 cctx->aes.ccm.iv_set = 0;
2320 cctx->aes.ccm.tag_set = 0;
2321 cctx->aes.ccm.len_set = 0;
2322 return rv;
2323 }
2324}
2325
2326/*-
2327 * Performs various operations on the context structure depending on control
2328 * type. Returns 1 for success, 0 for failure and -1 for unknown control type.
2329 * Code is big-endian.
2330 */
2331static int s390x_aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
2332{
2333 S390X_AES_CCM_CTX *cctx = EVP_C_DATA(S390X_AES_CCM_CTX, c);
2334 unsigned char *buf, *iv;
2335 int enc, len;
2336
2337 switch (type) {
2338 case EVP_CTRL_INIT:
2339 cctx->aes.ccm.key_set = 0;
2340 cctx->aes.ccm.iv_set = 0;
2341 cctx->aes.ccm.l = 8;
2342 cctx->aes.ccm.m = 12;
2343 cctx->aes.ccm.tag_set = 0;
2344 cctx->aes.ccm.len_set = 0;
2345 cctx->aes.ccm.tls_aad_len = -1;
2346 return 1;
2347
2348 case EVP_CTRL_AEAD_TLS1_AAD:
2349 if (arg != EVP_AEAD_TLS1_AAD_LEN)
2350 return 0;
2351
2352 /* Save the aad for later use. */
2353 buf = EVP_CIPHER_CTX_buf_noconst(c);
2354 memcpy(buf, ptr, arg);
2355 cctx->aes.ccm.tls_aad_len = arg;
2356
03a5e5ae 2357 len = buf[arg - 2] << 8 | buf[arg - 1];
39f5b069
PS		 2358 if (len < EVP_CCM_TLS_EXPLICIT_IV_LEN)
2359 return 0;
2360
2361 /* Correct length for explicit iv. */
2362 len -= EVP_CCM_TLS_EXPLICIT_IV_LEN;
2363
2364 enc = EVP_CIPHER_CTX_encrypting(c);
2365 if (!enc) {
2366 if (len < cctx->aes.ccm.m)
2367 return 0;
2368
2369 /* Correct length for tag. */
2370 len -= cctx->aes.ccm.m;
2371 }
2372
03a5e5ae
PS		 2373 buf[arg - 2] = len >> 8;
2374 buf[arg - 1] = len & 0xff;
2375
39f5b069
PS		 2376 /* Extra padding: tag appended to record. */
2377 return cctx->aes.ccm.m;
2378
2379 case EVP_CTRL_CCM_SET_IV_FIXED:
2380 if (arg != EVP_CCM_TLS_FIXED_IV_LEN)
2381 return 0;
2382
2383 /* Copy to first part of the iv. */
2384 iv = EVP_CIPHER_CTX_iv_noconst(c);
2385 memcpy(iv, ptr, arg);
2386 return 1;
2387
2388 case EVP_CTRL_AEAD_SET_IVLEN:
2389 arg = 15 - arg;
2390 /* fall-through */
2391
2392 case EVP_CTRL_CCM_SET_L:
2393 if (arg < 2 || arg > 8)
2394 return 0;
2395
2396 cctx->aes.ccm.l = arg;
2397 return 1;
2398
2399 case EVP_CTRL_AEAD_SET_TAG:
2400 if ((arg & 1) || arg < 4 || arg > 16)
2401 return 0;
2402
2403 enc = EVP_CIPHER_CTX_encrypting(c);
2404 if (enc && ptr)
2405 return 0;
2406
2407 if (ptr) {
2408 cctx->aes.ccm.tag_set = 1;
2409 buf = EVP_CIPHER_CTX_buf_noconst(c);
2410 memcpy(buf, ptr, arg);
2411 }
2412
2413 cctx->aes.ccm.m = arg;
2414 return 1;
2415
2416 case EVP_CTRL_AEAD_GET_TAG:
2417 enc = EVP_CIPHER_CTX_encrypting(c);
2418 if (!enc || !cctx->aes.ccm.tag_set)
2419 return 0;
2420
2421 if(arg < cctx->aes.ccm.m)
2422 return 0;
2423
2424 memcpy(ptr, cctx->aes.ccm.kmac_param.icv.b, cctx->aes.ccm.m);
2425 cctx->aes.ccm.tag_set = 0;
2426 cctx->aes.ccm.iv_set = 0;
2427 cctx->aes.ccm.len_set = 0;
2428 return 1;
2429
2430 case EVP_CTRL_COPY:
2431 return 1;
2432
2433 default:
2434 return -1;
2435 }
2436}
2437
96530eea
PS		 2438# define s390x_aes_ccm_cleanup aes_ccm_cleanup
2439
2440# ifndef OPENSSL_NO_OCB
2441# define S390X_AES_OCB_CTX EVP_AES_OCB_CTX
2442# define S390X_aes_128_ocb_CAPABLE 0
2443# define S390X_aes_192_ocb_CAPABLE 0
2444# define S390X_aes_256_ocb_CAPABLE 0
2445
2446# define s390x_aes_ocb_init_key aes_ocb_init_key
2447static int s390x_aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
2448 const unsigned char *iv, int enc);
2449# define s390x_aes_ocb_cipher aes_ocb_cipher
2450static int s390x_aes_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2451 const unsigned char *in, size_t len);
2452# define s390x_aes_ocb_cleanup aes_ocb_cleanup
2453static int s390x_aes_ocb_cleanup(EVP_CIPHER_CTX *);
2454# define s390x_aes_ocb_ctrl aes_ocb_ctrl
2455static int s390x_aes_ocb_ctrl(EVP_CIPHER_CTX *, int type, int arg, void *ptr);
2456# endif
2457
e74be3d4
RL		 2458# ifndef OPENSSL_NO_SIV
2459# define S390X_AES_SIV_CTX EVP_AES_SIV_CTX
2460# define S390X_aes_128_siv_CAPABLE 0
2461# define S390X_aes_192_siv_CAPABLE 0
2462# define S390X_aes_256_siv_CAPABLE 0
2463
2464# define s390x_aes_siv_init_key aes_siv_init_key
2465# define s390x_aes_siv_cipher aes_siv_cipher
2466# define s390x_aes_siv_cleanup aes_siv_cleanup
2467# define s390x_aes_siv_ctrl aes_siv_ctrl
2468# endif
2469
96530eea
PS		 2470# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode, \
2471 MODE,flags) \
2472static const EVP_CIPHER s390x_aes_##keylen##_##mode = { \
2473 nid##_##keylen##_##nmode,blocksize, \
2474 keylen / 8, \
2475 ivlen, \
2476 flags | EVP_CIPH_##MODE##_MODE, \
55bd169f 2477 s390x_aes_##mode##_init_key, \
96530eea
PS		 2478 s390x_aes_##mode##_cipher, \
2479 NULL, \
55bd169f 2480 sizeof(S390X_AES_##MODE##_CTX), \
96530eea
PS		 2481 NULL, \
2482 NULL, \
2483 NULL, \
2484 NULL \
2485}; \
2486static const EVP_CIPHER aes_##keylen##_##mode = { \
2487 nid##_##keylen##_##nmode, \
2488 blocksize, \
2489 keylen / 8, \
2490 ivlen, \
2491 flags | EVP_CIPH_##MODE##_MODE, \
2492 aes_init_key, \
2493 aes_##mode##_cipher, \
2494 NULL, \
2495 sizeof(EVP_AES_KEY), \
55bd169f
PS		 2496 NULL, \
2497 NULL, \
2498 NULL, \
2499 NULL \
96530eea
PS		 2500}; \
2501const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
2502{ \
2503 return S390X_aes_##keylen##_##mode##_CAPABLE ? \
2504 &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode; \
2505}
2506
2507# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags)\
2508static const EVP_CIPHER s390x_aes_##keylen##_##mode = { \
2509 nid##_##keylen##_##mode, \
2510 blocksize, \
e74be3d4 2511 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE ? 2 : 1) * keylen / 8, \
96530eea
PS		 2512 ivlen, \
2513 flags | EVP_CIPH_##MODE##_MODE, \
2514 s390x_aes_##mode##_init_key, \
2515 s390x_aes_##mode##_cipher, \
2516 s390x_aes_##mode##_cleanup, \
2517 sizeof(S390X_AES_##MODE##_CTX), \
2518 NULL, \
2519 NULL, \
2520 s390x_aes_##mode##_ctrl, \
2521 NULL \
2522}; \
2523static const EVP_CIPHER aes_##keylen##_##mode = { \
2524 nid##_##keylen##_##mode,blocksize, \
e74be3d4 2525 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE ? 2 : 1) * keylen / 8, \
96530eea
PS		 2526 ivlen, \
2527 flags | EVP_CIPH_##MODE##_MODE, \
2528 aes_##mode##_init_key, \
2529 aes_##mode##_cipher, \
2530 aes_##mode##_cleanup, \
2531 sizeof(EVP_AES_##MODE##_CTX), \
2532 NULL, \
2533 NULL, \
2534 aes_##mode##_ctrl, \
2535 NULL \
2536}; \
2537const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
2538{ \
2539 return S390X_aes_##keylen##_##mode##_CAPABLE ? \
2540 &s390x_aes_##keylen##_##mode : &aes_##keylen##_##mode; \
2541}
2542
5158c763 2543#else
17f121de 2544
5158c763 2545# define BLOCK_CIPHER_generic(nid,keylen,blocksize,ivlen,nmode,mode,MODE,flags) \
17f121de 2546static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e
MC		 2547 nid##_##keylen##_##nmode,blocksize,keylen/8,ivlen, \
2548 flags|EVP_CIPH_##MODE##_MODE, \
2549 aes_init_key, \
2550 aes_##mode##_cipher, \
2551 NULL, \
2552 sizeof(EVP_AES_KEY), \
2553 NULL,NULL,NULL,NULL }; \
17f121de
AP		 2554const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
2555{ return &aes_##keylen##_##mode; }
d1fff483 2556
5158c763 2557# define BLOCK_CIPHER_custom(nid,keylen,blocksize,ivlen,mode,MODE,flags) \
17f121de 2558static const EVP_CIPHER aes_##keylen##_##mode = { \
0f113f3e 2559 nid##_##keylen##_##mode,blocksize, \
b1ceb439
TS		 2560 (EVP_CIPH_##MODE##_MODE==EVP_CIPH_XTS_MODE||EVP_CIPH_##MODE##_MODE==EVP_CIPH_SIV_MODE?2:1)*keylen/8, \
2561 ivlen, \
0f113f3e
MC		 2562 flags|EVP_CIPH_##MODE##_MODE, \
2563 aes_##mode##_init_key, \
2564 aes_##mode##_cipher, \
2565 aes_##mode##_cleanup, \
2566 sizeof(EVP_AES_##MODE##_CTX), \
2567 NULL,NULL,aes_##mode##_ctrl,NULL }; \
17f121de
AP		 2568const EVP_CIPHER *EVP_aes_##keylen##_##mode(void) \
2569{ return &aes_##keylen##_##mode; }
9575d1a9 2570
5158c763 2571#endif
9575d1a9 2572
5158c763
MC		 2573#if defined(OPENSSL_CPUID_OBJ) && (defined(__arm__) || defined(__arm) || defined(__aarch64__))
2574# include "arm_arch.h"
2575# if __ARM_MAX_ARCH__>=7
2576# if defined(BSAES_ASM)
2577# define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
2578# endif
2579# if defined(VPAES_ASM)
2580# define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
0f113f3e 2581# endif
5158c763
MC		 2582# define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
2583# define HWAES_set_encrypt_key aes_v8_set_encrypt_key
2584# define HWAES_set_decrypt_key aes_v8_set_decrypt_key
2585# define HWAES_encrypt aes_v8_encrypt
2586# define HWAES_decrypt aes_v8_decrypt
2587# define HWAES_cbc_encrypt aes_v8_cbc_encrypt
2588# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
ddacb8f2 2589# endif
5158c763 2590#endif
d1fff483 2591
5158c763 2592#if defined(HWAES_CAPABLE)
ddacb8f2 2593int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
0f113f3e 2594 AES_KEY *key);
ddacb8f2 2595int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
0f113f3e 2596 AES_KEY *key);
ddacb8f2 2597void HWAES_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e 2598 const AES_KEY *key);
ddacb8f2 2599void HWAES_decrypt(const unsigned char *in, unsigned char *out,
0f113f3e 2600 const AES_KEY *key);
ddacb8f2 2601void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
0f113f3e
MC		 2602 size_t length, const AES_KEY *key,
2603 unsigned char *ivec, const int enc);
ddacb8f2 2604void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
0f113f3e
MC		 2605 size_t len, const AES_KEY *key,
2606 const unsigned char ivec[16]);
46f047d7
AP		 2607void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
2608 size_t len, const AES_KEY *key1,
2609 const AES_KEY *key2, const unsigned char iv[16]);
2610void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
2611 size_t len, const AES_KEY *key1,
2612 const AES_KEY *key2, const unsigned char iv[16]);
5158c763 2613#endif
ddacb8f2 2614
5158c763 2615#define BLOCK_CIPHER_generic_pack(nid,keylen,flags) \
0f113f3e
MC		 2616 BLOCK_CIPHER_generic(nid,keylen,16,16,cbc,cbc,CBC,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \
2617 BLOCK_CIPHER_generic(nid,keylen,16,0,ecb,ecb,ECB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \
2618 BLOCK_CIPHER_generic(nid,keylen,1,16,ofb128,ofb,OFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \
2619 BLOCK_CIPHER_generic(nid,keylen,1,16,cfb128,cfb,CFB,flags|EVP_CIPH_FLAG_DEFAULT_ASN1) \
2620 BLOCK_CIPHER_generic(nid,keylen,1,16,cfb1,cfb1,CFB,flags) \
2621 BLOCK_CIPHER_generic(nid,keylen,1,16,cfb8,cfb8,CFB,flags) \
2622 BLOCK_CIPHER_generic(nid,keylen,1,16,ctr,ctr,CTR,flags)
d1fff483
AP		 2623
2624static int aes_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 2625 const unsigned char *iv, int enc)
2626{
2627 int ret, mode;
6435f0f6 2628 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
0f113f3e 2629
6435f0f6 2630 mode = EVP_CIPHER_CTX_mode(ctx);
0f113f3e 2631 if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE)
c01a3c6d 2632 && !enc) {
5158c763 2633#ifdef HWAES_CAPABLE
0f113f3e 2634 if (HWAES_CAPABLE) {
6435f0f6
RL		 2635 ret = HWAES_set_decrypt_key(key,
2636 EVP_CIPHER_CTX_key_length(ctx) * 8,
2637 &dat->ks.ks);
0f113f3e
MC		 2638 dat->block = (block128_f) HWAES_decrypt;
2639 dat->stream.cbc = NULL;
5158c763 2640# ifdef HWAES_cbc_encrypt
0f113f3e
MC		 2641 if (mode == EVP_CIPH_CBC_MODE)
2642 dat->stream.cbc = (cbc128_f) HWAES_cbc_encrypt;
0f113f3e 2643# endif
5158c763
MC		 2644 } else
2645#endif
2646#ifdef BSAES_CAPABLE
0f113f3e 2647 if (BSAES_CAPABLE && mode == EVP_CIPH_CBC_MODE) {
6435f0f6
RL		 2648 ret = AES_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
2649 &dat->ks.ks);
0f113f3e
MC		 2650 dat->block = (block128_f) AES_decrypt;
2651 dat->stream.cbc = (cbc128_f) bsaes_cbc_encrypt;
2652 } else
5158c763
MC		 2653#endif
2654#ifdef VPAES_CAPABLE
0f113f3e 2655 if (VPAES_CAPABLE) {
6435f0f6
RL		 2656 ret = vpaes_set_decrypt_key(key,
2657 EVP_CIPHER_CTX_key_length(ctx) * 8,
2658 &dat->ks.ks);
0f113f3e
MC		 2659 dat->block = (block128_f) vpaes_decrypt;
2660 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
2661 (cbc128_f) vpaes_cbc_encrypt : NULL;
2662 } else
5158c763 2663#endif
0f113f3e 2664 {
6435f0f6
RL		 2665 ret = AES_set_decrypt_key(key,
2666 EVP_CIPHER_CTX_key_length(ctx) * 8,
2667 &dat->ks.ks);
0f113f3e
MC		 2668 dat->block = (block128_f) AES_decrypt;
2669 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
2670 (cbc128_f) AES_cbc_encrypt : NULL;
c01a3c6d 2671 }
0f113f3e 2672 } else
5158c763 2673#ifdef HWAES_CAPABLE
0f113f3e 2674 if (HWAES_CAPABLE) {
6435f0f6
RL		 2675 ret = HWAES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
2676 &dat->ks.ks);
0f113f3e
MC		 2677 dat->block = (block128_f) HWAES_encrypt;
2678 dat->stream.cbc = NULL;
5158c763 2679# ifdef HWAES_cbc_encrypt
0f113f3e
MC		 2680 if (mode == EVP_CIPH_CBC_MODE)
2681 dat->stream.cbc = (cbc128_f) HWAES_cbc_encrypt;
2682 else
5158c763
MC		 2683# endif
2684# ifdef HWAES_ctr32_encrypt_blocks
0f113f3e
MC		 2685 if (mode == EVP_CIPH_CTR_MODE)
2686 dat->stream.ctr = (ctr128_f) HWAES_ctr32_encrypt_blocks;
2687 else
5158c763 2688# endif
0f113f3e
MC		 2689 (void)0; /* terminate potentially open 'else' */
2690 } else
5158c763
MC		 2691#endif
2692#ifdef BSAES_CAPABLE
0f113f3e 2693 if (BSAES_CAPABLE && mode == EVP_CIPH_CTR_MODE) {
6435f0f6
RL		 2694 ret = AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
2695 &dat->ks.ks);
0f113f3e
MC		 2696 dat->block = (block128_f) AES_encrypt;
2697 dat->stream.ctr = (ctr128_f) bsaes_ctr32_encrypt_blocks;
2698 } else
5158c763
MC		 2699#endif
2700#ifdef VPAES_CAPABLE
0f113f3e 2701 if (VPAES_CAPABLE) {
6435f0f6
RL		 2702 ret = vpaes_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
2703 &dat->ks.ks);
0f113f3e
MC		 2704 dat->block = (block128_f) vpaes_encrypt;
2705 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
2706 (cbc128_f) vpaes_cbc_encrypt : NULL;
2707 } else
5158c763 2708#endif
0f113f3e 2709 {
6435f0f6
RL		 2710 ret = AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
2711 &dat->ks.ks);
0f113f3e
MC		 2712 dat->block = (block128_f) AES_encrypt;
2713 dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ?
2714 (cbc128_f) AES_cbc_encrypt : NULL;
5158c763 2715#ifdef AES_CTR_ASM
0f113f3e
MC		 2716 if (mode == EVP_CIPH_CTR_MODE)
2717 dat->stream.ctr = (ctr128_f) AES_ctr32_encrypt;
5158c763 2718#endif
0f113f3e 2719 }
d1fff483 2720
0f113f3e
MC		 2721 if (ret < 0) {
2722 EVPerr(EVP_F_AES_INIT_KEY, EVP_R_AES_KEY_SETUP_FAILED);
2723 return 0;
2724 }
d1fff483 2725
0f113f3e
MC		 2726 return 1;
2727}
d1fff483 2728
0f113f3e
MC		 2729static int aes_cbc_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2730 const unsigned char *in, size_t len)
17f121de 2731{
6435f0f6 2732 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
8ca28da0 2733
0f113f3e 2734 if (dat->stream.cbc)
6435f0f6
RL		 2735 (*dat->stream.cbc) (in, out, len, &dat->ks,
2736 EVP_CIPHER_CTX_iv_noconst(ctx),
2737 EVP_CIPHER_CTX_encrypting(ctx));
2738 else if (EVP_CIPHER_CTX_encrypting(ctx))
2739 CRYPTO_cbc128_encrypt(in, out, len, &dat->ks,
2740 EVP_CIPHER_CTX_iv_noconst(ctx), dat->block);
0f113f3e 2741 else
6435f0f6
RL		 2742 CRYPTO_cbc128_decrypt(in, out, len, &dat->ks,
2743 EVP_CIPHER_CTX_iv_noconst(ctx), dat->block);
17f121de 2744
0f113f3e 2745 return 1;
17f121de
AP		 2746}
2747
0f113f3e
MC		 2748static int aes_ecb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2749 const unsigned char *in, size_t len)
17f121de 2750{
6435f0f6 2751 size_t bl = EVP_CIPHER_CTX_block_size(ctx);
0f113f3e 2752 size_t i;
6435f0f6 2753 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
17f121de 2754
0f113f3e
MC		 2755 if (len < bl)
2756 return 1;
17f121de 2757
0f113f3e
MC		 2758 for (i = 0, len -= bl; i <= len; i += bl)
2759 (*dat->block) (in + i, out + i, &dat->ks);
17f121de 2760
0f113f3e 2761 return 1;
17f121de 2762}
deb2c1a1 2763
0f113f3e
MC		 2764static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2765 const unsigned char *in, size_t len)
17f121de 2766{
6435f0f6 2767 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
8ca28da0 2768
6435f0f6 2769 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2770 CRYPTO_ofb128_encrypt(in, out, len, &dat->ks,
6435f0f6
RL		 2771 EVP_CIPHER_CTX_iv_noconst(ctx), &num, dat->block);
2772 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e 2773 return 1;
17f121de 2774}
deb2c1a1 2775
0f113f3e
MC		 2776static int aes_cfb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2777 const unsigned char *in, size_t len)
17f121de 2778{
6435f0f6 2779 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
8ca28da0 2780
6435f0f6 2781 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2782 CRYPTO_cfb128_encrypt(in, out, len, &dat->ks,
6435f0f6
RL		 2783 EVP_CIPHER_CTX_iv_noconst(ctx), &num,
2784 EVP_CIPHER_CTX_encrypting(ctx), dat->block);
2785 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e 2786 return 1;
17f121de
AP		 2787}
2788
0f113f3e
MC		 2789static int aes_cfb8_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2790 const unsigned char *in, size_t len)
17f121de 2791{
6435f0f6 2792 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
8ca28da0 2793
6435f0f6 2794 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2795 CRYPTO_cfb128_8_encrypt(in, out, len, &dat->ks,
6435f0f6
RL		 2796 EVP_CIPHER_CTX_iv_noconst(ctx), &num,
2797 EVP_CIPHER_CTX_encrypting(ctx), dat->block);
2798 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e 2799 return 1;
17f121de 2800}
8d1ebe0b 2801
0f113f3e
MC		 2802static int aes_cfb1_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2803 const unsigned char *in, size_t len)
17f121de 2804{
6435f0f6 2805 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
0f113f3e 2806
6435f0f6
RL		 2807 if (EVP_CIPHER_CTX_test_flags(ctx, EVP_CIPH_FLAG_LENGTH_BITS)) {
2808 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2809 CRYPTO_cfb128_1_encrypt(in, out, len, &dat->ks,
6435f0f6
RL		 2810 EVP_CIPHER_CTX_iv_noconst(ctx), &num,
2811 EVP_CIPHER_CTX_encrypting(ctx), dat->block);
2812 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e
MC		 2813 return 1;
2814 }
2815
2816 while (len >= MAXBITCHUNK) {
6435f0f6 2817 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2818 CRYPTO_cfb128_1_encrypt(in, out, MAXBITCHUNK * 8, &dat->ks,
6435f0f6
RL		 2819 EVP_CIPHER_CTX_iv_noconst(ctx), &num,
2820 EVP_CIPHER_CTX_encrypting(ctx), dat->block);
2821 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e 2822 len -= MAXBITCHUNK;
604e591e
BE		 2823 out += MAXBITCHUNK;
2824 in += MAXBITCHUNK;
0f113f3e 2825 }
6435f0f6
RL		 2826 if (len) {
2827 int num = EVP_CIPHER_CTX_num(ctx);
0f113f3e 2828 CRYPTO_cfb128_1_encrypt(in, out, len * 8, &dat->ks,
6435f0f6
RL		 2829 EVP_CIPHER_CTX_iv_noconst(ctx), &num,
2830 EVP_CIPHER_CTX_encrypting(ctx), dat->block);
2831 EVP_CIPHER_CTX_set_num(ctx, num);
2832 }
0f113f3e
MC		 2833
2834 return 1;
17f121de 2835}
8d1ebe0b 2836
0f113f3e
MC		 2837static int aes_ctr_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
2838 const unsigned char *in, size_t len)
d976f992 2839{
6435f0f6
RL		 2840 unsigned int num = EVP_CIPHER_CTX_num(ctx);
2841 EVP_AES_KEY *dat = EVP_C_DATA(EVP_AES_KEY,ctx);
0f113f3e
MC		 2842
2843 if (dat->stream.ctr)
2844 CRYPTO_ctr128_encrypt_ctr32(in, out, len, &dat->ks,
6435f0f6
RL		 2845 EVP_CIPHER_CTX_iv_noconst(ctx),
2846 EVP_CIPHER_CTX_buf_noconst(ctx),
2847 &num, dat->stream.ctr);
0f113f3e
MC		 2848 else
2849 CRYPTO_ctr128_encrypt(in, out, len, &dat->ks,
6435f0f6
RL		 2850 EVP_CIPHER_CTX_iv_noconst(ctx),
2851 EVP_CIPHER_CTX_buf_noconst(ctx), &num,
2852 dat->block);
2853 EVP_CIPHER_CTX_set_num(ctx, num);
0f113f3e 2854 return 1;
d976f992
AP		 2855}
2856
0f113f3e
MC		 2857BLOCK_CIPHER_generic_pack(NID_aes, 128, 0)
2858 BLOCK_CIPHER_generic_pack(NID_aes, 192, 0)
2859 BLOCK_CIPHER_generic_pack(NID_aes, 256, 0)
bdaa5415
DSH		 2860
2861static int aes_gcm_cleanup(EVP_CIPHER_CTX *c)
0f113f3e 2862{
6435f0f6 2863 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,c);
273a0218
BE		 2864 if (gctx == NULL)
2865 return 0;
0f113f3e 2866 OPENSSL_cleanse(&gctx->gcm, sizeof(gctx->gcm));
6435f0f6 2867 if (gctx->iv != EVP_CIPHER_CTX_iv_noconst(c))
0f113f3e
MC		 2868 OPENSSL_free(gctx->iv);
2869 return 1;
2870}
bdaa5415
DSH		 2871
2872static int aes_gcm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
0f113f3e 2873{
6435f0f6 2874 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,c);
0f113f3e
MC		 2875 switch (type) {
2876 case EVP_CTRL_INIT:
2877 gctx->key_set = 0;
2878 gctx->iv_set = 0;
c5307d9c
AP		 2879 gctx->ivlen = c->cipher->iv_len;
2880 gctx->iv = c->iv;
0f113f3e
MC		 2881 gctx->taglen = -1;
2882 gctx->iv_gen = 0;
2883 gctx->tls_aad_len = -1;
2884 return 1;
2885
e640fa02 2886 case EVP_CTRL_AEAD_SET_IVLEN:
0f113f3e
MC		 2887 if (arg <= 0)
2888 return 0;
2889 /* Allocate memory for IV if needed */
2890 if ((arg > EVP_MAX_IV_LENGTH) && (arg > gctx->ivlen)) {
c5307d9c 2891 if (gctx->iv != c->iv)
0f113f3e 2892 OPENSSL_free(gctx->iv);
cdb10bae
RS		 2893 if ((gctx->iv = OPENSSL_malloc(arg)) == NULL) {
2894 EVPerr(EVP_F_AES_GCM_CTRL, ERR_R_MALLOC_FAILURE);
0f113f3e 2895 return 0;
cdb10bae 2896 }
0f113f3e
MC		 2897 }
2898 gctx->ivlen = arg;
2899 return 1;
2900
e640fa02 2901 case EVP_CTRL_AEAD_SET_TAG:
c5307d9c 2902 if (arg <= 0 || arg > 16 || c->encrypt)
0f113f3e 2903 return 0;
c5307d9c 2904 memcpy(c->buf, ptr, arg);
0f113f3e
MC		 2905 gctx->taglen = arg;
2906 return 1;
2907
e640fa02 2908 case EVP_CTRL_AEAD_GET_TAG:
c5307d9c 2909 if (arg <= 0 || arg > 16 || !c->encrypt
6435f0f6 2910 || gctx->taglen < 0)
0f113f3e 2911 return 0;
c5307d9c 2912 memcpy(ptr, c->buf, arg);
0f113f3e
MC		 2913 return 1;
2914
ecd1557f 2915 case EVP_CTRL_GET_IV:
bcf082d1 2916 if (gctx->iv_gen != 1 && gctx->iv_gen_rand != 1)
ecd1557f
BP		 2917 return 0;
2918 if (gctx->ivlen != arg)
2919 return 0;
2920 memcpy(ptr, gctx->iv, arg);
2921 return 1;
2922
0f113f3e
MC		 2923 case EVP_CTRL_GCM_SET_IV_FIXED:
2924 /* Special case: -1 length restores whole IV */
2925 if (arg == -1) {
2926 memcpy(gctx->iv, ptr, gctx->ivlen);
2927 gctx->iv_gen = 1;
2928 return 1;
2929 }
2930 /*
2931 * Fixed field must be at least 4 bytes and invocation field at least
2932 * 8.
2933 */
2934 if ((arg < 4) || (gctx->ivlen - arg) < 8)
2935 return 0;
2936 if (arg)
2937 memcpy(gctx->iv, ptr, arg);
c5307d9c 2938 if (c->encrypt && RAND_bytes(gctx->iv + arg, gctx->ivlen - arg) <= 0)
16cfc2c9 2939 return 0;
0f113f3e
MC		 2940 gctx->iv_gen = 1;
2941 return 1;
2942
2943 case EVP_CTRL_GCM_IV_GEN:
2944 if (gctx->iv_gen == 0 || gctx->key_set == 0)
2945 return 0;
2946 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
2947 if (arg <= 0 || arg > gctx->ivlen)
2948 arg = gctx->ivlen;
2949 memcpy(ptr, gctx->iv + gctx->ivlen - arg, arg);
2950 /*
2951 * Invocation field will be at least 8 bytes in size and so no need
2952 * to check wrap around or increment more than last 8 bytes.
2953 */
2954 ctr64_inc(gctx->iv + gctx->ivlen - 8);
2955 gctx->iv_set = 1;
2956 return 1;
2957
2958 case EVP_CTRL_GCM_SET_IV_INV:
c5307d9c 2959 if (gctx->iv_gen == 0 || gctx->key_set == 0 || c->encrypt)
0f113f3e
MC		 2960 return 0;
2961 memcpy(gctx->iv + gctx->ivlen - arg, ptr, arg);
2962 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
2963 gctx->iv_set = 1;
2964 return 1;
2965
2966 case EVP_CTRL_AEAD_TLS1_AAD:
2967 /* Save the AAD for later use */
c8269881 2968 if (arg != EVP_AEAD_TLS1_AAD_LEN)
0f113f3e 2969 return 0;
c5307d9c 2970 memcpy(c->buf, ptr, arg);
0f113f3e 2971 gctx->tls_aad_len = arg;
d6b34570 2972 gctx->tls_enc_records = 0;
0f113f3e 2973 {
c5307d9c 2974 unsigned int len = c->buf[arg - 2] << 8 | c->buf[arg - 1];
0f113f3e 2975 /* Correct length for explicit IV */
2198b3a5
AP		 2976 if (len < EVP_GCM_TLS_EXPLICIT_IV_LEN)
2977 return 0;
0f113f3e
MC		 2978 len -= EVP_GCM_TLS_EXPLICIT_IV_LEN;
2979 /* If decrypting correct for tag too */
c5307d9c 2980 if (!c->encrypt) {
2198b3a5
AP		 2981 if (len < EVP_GCM_TLS_TAG_LEN)
2982 return 0;
0f113f3e 2983 len -= EVP_GCM_TLS_TAG_LEN;
2198b3a5 2984 }
c5307d9c
AP		 2985 c->buf[arg - 2] = len >> 8;
2986 c->buf[arg - 1] = len & 0xff;
0f113f3e
MC		 2987 }
2988 /* Extra padding: tag appended to record */
2989 return EVP_GCM_TLS_TAG_LEN;
2990
2991 case EVP_CTRL_COPY:
2992 {
2993 EVP_CIPHER_CTX *out = ptr;
6435f0f6 2994 EVP_AES_GCM_CTX *gctx_out = EVP_C_DATA(EVP_AES_GCM_CTX,out);
0f113f3e
MC		 2995 if (gctx->gcm.key) {
2996 if (gctx->gcm.key != &gctx->ks)
2997 return 0;
2998 gctx_out->gcm.key = &gctx_out->ks;
2999 }
c5307d9c
AP		 3000 if (gctx->iv == c->iv)
3001 gctx_out->iv = out->iv;
0f113f3e 3002 else {
cdb10bae
RS		 3003 if ((gctx_out->iv = OPENSSL_malloc(gctx->ivlen)) == NULL) {
3004 EVPerr(EVP_F_AES_GCM_CTRL, ERR_R_MALLOC_FAILURE);
0f113f3e 3005 return 0;
cdb10bae 3006 }
0f113f3e
MC		 3007 memcpy(gctx_out->iv, gctx->iv, gctx->ivlen);
3008 }
3009 return 1;
3010 }
3011
3012 default:
3013 return -1;
3014
3015 }
3016}
bdaa5415
DSH		 3017
3018static int aes_gcm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 3019 const unsigned char *iv, int enc)
3020{
6435f0f6 3021 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
0f113f3e
MC		 3022 if (!iv && !key)
3023 return 1;
3024 if (key) {
3025 do {
5158c763 3026#ifdef HWAES_CAPABLE
0f113f3e 3027 if (HWAES_CAPABLE) {
c5307d9c 3028 HWAES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
0f113f3e
MC		 3029 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
3030 (block128_f) HWAES_encrypt);
5158c763 3031# ifdef HWAES_ctr32_encrypt_blocks
0f113f3e 3032 gctx->ctr = (ctr128_f) HWAES_ctr32_encrypt_blocks;
5158c763 3033# else
0f113f3e 3034 gctx->ctr = NULL;
5158c763 3035# endif
0f113f3e
MC		 3036 break;
3037 } else
5158c763
MC		 3038#endif
3039#ifdef BSAES_CAPABLE
0f113f3e 3040 if (BSAES_CAPABLE) {
c5307d9c 3041 AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
0f113f3e
MC		 3042 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
3043 (block128_f) AES_encrypt);
3044 gctx->ctr = (ctr128_f) bsaes_ctr32_encrypt_blocks;
3045 break;
3046 } else
5158c763
MC		 3047#endif
3048#ifdef VPAES_CAPABLE
0f113f3e 3049 if (VPAES_CAPABLE) {
c5307d9c 3050 vpaes_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
0f113f3e
MC		 3051 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
3052 (block128_f) vpaes_encrypt);
3053 gctx->ctr = NULL;
3054 break;
3055 } else
5158c763 3056#endif
0f113f3e
MC		 3057 (void)0; /* terminate potentially open 'else' */
3058
c5307d9c 3059 AES_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
0f113f3e
MC		 3060 CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks,
3061 (block128_f) AES_encrypt);
5158c763 3062#ifdef AES_CTR_ASM
0f113f3e 3063 gctx->ctr = (ctr128_f) AES_ctr32_encrypt;
5158c763 3064#else
0f113f3e 3065 gctx->ctr = NULL;
5158c763 3066#endif
0f113f3e
MC		 3067 } while (0);
3068
3069 /*
3070 * If we have an iv can set it directly, otherwise use saved IV.
3071 */
3072 if (iv == NULL && gctx->iv_set)
3073 iv = gctx->iv;
3074 if (iv) {
3075 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
3076 gctx->iv_set = 1;
3077 }
3078 gctx->key_set = 1;
3079 } else {
3080 /* If key set use IV, otherwise copy */
3081 if (gctx->key_set)
3082 CRYPTO_gcm128_setiv(&gctx->gcm, iv, gctx->ivlen);
3083 else
3084 memcpy(gctx->iv, iv, gctx->ivlen);
3085 gctx->iv_set = 1;
3086 gctx->iv_gen = 0;
3087 }
3088 return 1;
3089}
3090
3091/*
3092 * Handle TLS GCM packet format. This consists of the last portion of the IV
28dd49fa
DSH		 3093 * followed by the payload and finally the tag. On encrypt generate IV,
3094 * encrypt payload and write the tag. On verify retrieve IV, decrypt payload
3095 * and verify tag.
3096 */
3097
3098static int aes_gcm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 3099 const unsigned char *in, size_t len)
3100{
6435f0f6 3101 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
0f113f3e
MC		 3102 int rv = -1;
3103 /* Encrypt/decrypt must be performed in place */
3104 if (out != in
3105 || len < (EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN))
3106 return -1;
df443918 3107
d6b34570
P		 3108 /*
3109 * Check for too many keys as per FIPS 140-2 IG A.5 "Key/IV Pair Uniqueness
3110 * Requirements from SP 800-38D". The requirements is for one party to the
3111 * communication to fail after 2^64 - 1 keys. We do this on the encrypting
3112 * side only.
3113 */
3114 if (ctx->encrypt && ++gctx->tls_enc_records == 0) {
3115 EVPerr(EVP_F_AES_GCM_TLS_CIPHER, EVP_R_TOO_MANY_RECORDS);
3116 goto err;
3117 }
3118
0f113f3e
MC		 3119 /*
3120 * Set IV from start of buffer or generate IV and write to start of
3121 * buffer.
3122 */
c5307d9c
AP		 3123 if (EVP_CIPHER_CTX_ctrl(ctx, ctx->encrypt ? EVP_CTRL_GCM_IV_GEN
3124 : EVP_CTRL_GCM_SET_IV_INV,
0f113f3e
MC		 3125 EVP_GCM_TLS_EXPLICIT_IV_LEN, out) <= 0)
3126 goto err;
3127 /* Use saved AAD */
c5307d9c 3128 if (CRYPTO_gcm128_aad(&gctx->gcm, ctx->buf, gctx->tls_aad_len))
0f113f3e
MC		 3129 goto err;
3130 /* Fix buffer and length to point to payload */
3131 in += EVP_GCM_TLS_EXPLICIT_IV_LEN;
3132 out += EVP_GCM_TLS_EXPLICIT_IV_LEN;
3133 len -= EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
c5307d9c 3134 if (ctx->encrypt) {
0f113f3e
MC		 3135 /* Encrypt payload */
3136 if (gctx->ctr) {
3137 size_t bulk = 0;
5158c763 3138#if defined(AES_GCM_ASM)
0f113f3e
MC		 3139 if (len >= 32 && AES_GCM_ASM(gctx)) {
3140 if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0))
3141 return -1;
3142
3143 bulk = AES_gcm_encrypt(in, out, len,
3144 gctx->gcm.key,
3145 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3146 gctx->gcm.len.u[1] += bulk;
3147 }
5158c763 3148#endif
0f113f3e
MC		 3149 if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
3150 in + bulk,
3151 out + bulk,
3152 len - bulk, gctx->ctr))
3153 goto err;
3154 } else {
3155 size_t bulk = 0;
5158c763 3156#if defined(AES_GCM_ASM2)
0f113f3e
MC		 3157 if (len >= 32 && AES_GCM_ASM2(gctx)) {
3158 if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0))
3159 return -1;
3160
3161 bulk = AES_gcm_encrypt(in, out, len,
3162 gctx->gcm.key,
3163 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3164 gctx->gcm.len.u[1] += bulk;
3165 }
5158c763 3166#endif
0f113f3e
MC		 3167 if (CRYPTO_gcm128_encrypt(&gctx->gcm,
3168 in + bulk, out + bulk, len - bulk))
3169 goto err;
3170 }
3171 out += len;
3172 /* Finally write tag */
3173 CRYPTO_gcm128_tag(&gctx->gcm, out, EVP_GCM_TLS_TAG_LEN);
3174 rv = len + EVP_GCM_TLS_EXPLICIT_IV_LEN + EVP_GCM_TLS_TAG_LEN;
3175 } else {
3176 /* Decrypt */
3177 if (gctx->ctr) {
3178 size_t bulk = 0;
5158c763 3179#if defined(AES_GCM_ASM)
0f113f3e
MC		 3180 if (len >= 16 && AES_GCM_ASM(gctx)) {
3181 if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0))
3182 return -1;
3183
3184 bulk = AES_gcm_decrypt(in, out, len,
3185 gctx->gcm.key,
3186 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3187 gctx->gcm.len.u[1] += bulk;
3188 }
5158c763 3189#endif
0f113f3e
MC		 3190 if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
3191 in + bulk,
3192 out + bulk,
3193 len - bulk, gctx->ctr))
3194 goto err;
3195 } else {
3196 size_t bulk = 0;
5158c763 3197#if defined(AES_GCM_ASM2)
0f113f3e
MC		 3198 if (len >= 16 && AES_GCM_ASM2(gctx)) {
3199 if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0))
3200 return -1;
3201
3202 bulk = AES_gcm_decrypt(in, out, len,
3203 gctx->gcm.key,
3204 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3205 gctx->gcm.len.u[1] += bulk;
3206 }
5158c763 3207#endif
0f113f3e
MC		 3208 if (CRYPTO_gcm128_decrypt(&gctx->gcm,
3209 in + bulk, out + bulk, len - bulk))
3210 goto err;
3211 }
3212 /* Retrieve tag */
c5307d9c 3213 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, EVP_GCM_TLS_TAG_LEN);
0f113f3e 3214 /* If tag mismatch wipe buffer */
c5307d9c 3215 if (CRYPTO_memcmp(ctx->buf, in + len, EVP_GCM_TLS_TAG_LEN)) {
0f113f3e
MC		 3216 OPENSSL_cleanse(out, len);
3217 goto err;
3218 }
3219 rv = len;
3220 }
3221
3222 err:
3223 gctx->iv_set = 0;
3224 gctx->tls_aad_len = -1;
3225 return rv;
3226}
28dd49fa 3227
bcf082d1
SL		 3228#ifdef FIPS_MODE
3229/*
3230 * See SP800-38D (GCM) Section 8 "Uniqueness requirement on IVS and keys"
3231 *
3232 * See also 8.2.2 RBG-based construction.
3233 * Random construction consists of a free field (which can be NULL) and a
3234 * random field which will use a DRBG that can return at least 96 bits of
3235 * entropy strength. (The DRBG must be seeded by the FIPS module).
3236 */
3237static int aes_gcm_iv_generate(EVP_AES_GCM_CTX *gctx, int offset)
3238{
3239 int sz = gctx->ivlen - offset;
3240
3241 /* Must be at least 96 bits */
3242 if (sz <= 0 || gctx->ivlen < 12)
3243 return 0;
3244
3245 /* Use DRBG to generate random iv */
3246 if (RAND_bytes(gctx->iv + offset, sz) <= 0)
3247 return 0;
3248 return 1;
3249}
3250#endif /* FIPS_MODE */
3251
17f121de 3252static int aes_gcm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 3253 const unsigned char *in, size_t len)
3254{
6435f0f6 3255 EVP_AES_GCM_CTX *gctx = EVP_C_DATA(EVP_AES_GCM_CTX,ctx);
bcf082d1 3256
0f113f3e
MC		 3257 /* If not set up, return error */
3258 if (!gctx->key_set)
3259 return -1;
3260
3261 if (gctx->tls_aad_len >= 0)
3262 return aes_gcm_tls_cipher(ctx, out, in, len);
3263
bcf082d1
SL		 3264#ifdef FIPS_MODE
3265 /*
3266 * FIPS requires generation of AES-GCM IV's inside the FIPS module.
3267 * The IV can still be set externally (the security policy will state that
3268 * this is not FIPS compliant). There are some applications
3269 * where setting the IV externally is the only option available.
3270 */
3271 if (!gctx->iv_set) {
3272 if (!ctx->encrypt || !aes_gcm_iv_generate(gctx, 0))
3273 return -1;
3274 CRYPTO_gcm128_setiv(&gctx->gcm, gctx->iv, gctx->ivlen);
3275 gctx->iv_set = 1;
3276 gctx->iv_gen_rand = 1;
3277 }
3278#else
0f113f3e
MC		 3279 if (!gctx->iv_set)
3280 return -1;
bcf082d1
SL		 3281#endif /* FIPS_MODE */
3282
0f113f3e
MC		 3283 if (in) {
3284 if (out == NULL) {
3285 if (CRYPTO_gcm128_aad(&gctx->gcm, in, len))
3286 return -1;
c5307d9c 3287 } else if (ctx->encrypt) {
0f113f3e
MC		 3288 if (gctx->ctr) {
3289 size_t bulk = 0;
5158c763 3290#if defined(AES_GCM_ASM)
0f113f3e
MC		 3291 if (len >= 32 && AES_GCM_ASM(gctx)) {
3292 size_t res = (16 - gctx->gcm.mres) % 16;
3293
3294 if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res))
3295 return -1;
3296
3297 bulk = AES_gcm_encrypt(in + res,
3298 out + res, len - res,
3299 gctx->gcm.key, gctx->gcm.Yi.c,
3300 gctx->gcm.Xi.u);
3301 gctx->gcm.len.u[1] += bulk;
3302 bulk += res;
3303 }
5158c763 3304#endif
0f113f3e
MC		 3305 if (CRYPTO_gcm128_encrypt_ctr32(&gctx->gcm,
3306 in + bulk,
3307 out + bulk,
3308 len - bulk, gctx->ctr))
3309 return -1;
3310 } else {
3311 size_t bulk = 0;
5158c763 3312#if defined(AES_GCM_ASM2)
0f113f3e
MC		 3313 if (len >= 32 && AES_GCM_ASM2(gctx)) {
3314 size_t res = (16 - gctx->gcm.mres) % 16;
3315
3316 if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res))
3317 return -1;
3318
3319 bulk = AES_gcm_encrypt(in + res,
3320 out + res, len - res,
3321 gctx->gcm.key, gctx->gcm.Yi.c,
3322 gctx->gcm.Xi.u);
3323 gctx->gcm.len.u[1] += bulk;
3324 bulk += res;
3325 }
5158c763 3326#endif
0f113f3e
MC		 3327 if (CRYPTO_gcm128_encrypt(&gctx->gcm,
3328 in + bulk, out + bulk, len - bulk))
3329 return -1;
3330 }
3331 } else {
3332 if (gctx->ctr) {
3333 size_t bulk = 0;
5158c763 3334#if defined(AES_GCM_ASM)
0f113f3e
MC		 3335 if (len >= 16 && AES_GCM_ASM(gctx)) {
3336 size_t res = (16 - gctx->gcm.mres) % 16;
3337
3338 if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res))
3339 return -1;
3340
3341 bulk = AES_gcm_decrypt(in + res,
3342 out + res, len - res,
3343 gctx->gcm.key,
3344 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3345 gctx->gcm.len.u[1] += bulk;
3346 bulk += res;
3347 }
5158c763 3348#endif
0f113f3e
MC		 3349 if (CRYPTO_gcm128_decrypt_ctr32(&gctx->gcm,
3350 in + bulk,
3351 out + bulk,
3352 len - bulk, gctx->ctr))
3353 return -1;
3354 } else {
3355 size_t bulk = 0;
5158c763 3356#if defined(AES_GCM_ASM2)
0f113f3e
MC		 3357 if (len >= 16 && AES_GCM_ASM2(gctx)) {
3358 size_t res = (16 - gctx->gcm.mres) % 16;
3359
3360 if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res))
3361 return -1;
3362
3363 bulk = AES_gcm_decrypt(in + res,
3364 out + res, len - res,
3365 gctx->gcm.key,
3366 gctx->gcm.Yi.c, gctx->gcm.Xi.u);
3367 gctx->gcm.len.u[1] += bulk;
3368 bulk += res;
3369 }
5158c763 3370#endif
0f113f3e
MC		 3371 if (CRYPTO_gcm128_decrypt(&gctx->gcm,
3372 in + bulk, out + bulk, len - bulk))
3373 return -1;
3374 }
3375 }
3376 return len;
3377 } else {
c5307d9c 3378 if (!ctx->encrypt) {
0f113f3e
MC		 3379 if (gctx->taglen < 0)
3380 return -1;
c5307d9c 3381 if (CRYPTO_gcm128_finish(&gctx->gcm, ctx->buf, gctx->taglen) != 0)
0f113f3e
MC		 3382 return -1;
3383 gctx->iv_set = 0;
3384 return 0;
3385 }
c5307d9c 3386 CRYPTO_gcm128_tag(&gctx->gcm, ctx->buf, 16);
0f113f3e
MC		 3387 gctx->taglen = 16;
3388 /* Don't reuse the IV */
3389 gctx->iv_set = 0;
3390 return 0;
3391 }
3392
3393}
3394
5158c763 3395#define CUSTOM_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 \
0f113f3e
MC		 3396 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
3397 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
3398 | EVP_CIPH_CUSTOM_COPY)
3399
3400BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, gcm, GCM,
3401 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
3402 BLOCK_CIPHER_custom(NID_aes, 192, 1, 12, gcm, GCM,
3403 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
3404 BLOCK_CIPHER_custom(NID_aes, 256, 1, 12, gcm, GCM,
3405 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
32a2d8dd
DSH		 3406
3407static int aes_xts_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
0f113f3e 3408{
6435f0f6 3409 EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,c);
0f113f3e
MC		 3410 if (type == EVP_CTRL_COPY) {
3411 EVP_CIPHER_CTX *out = ptr;
6435f0f6 3412 EVP_AES_XTS_CTX *xctx_out = EVP_C_DATA(EVP_AES_XTS_CTX,out);
0f113f3e
MC		 3413 if (xctx->xts.key1) {
3414 if (xctx->xts.key1 != &xctx->ks1)
3415 return 0;
3416 xctx_out->xts.key1 = &xctx_out->ks1;
3417 }
3418 if (xctx->xts.key2) {
3419 if (xctx->xts.key2 != &xctx->ks2)
3420 return 0;
3421 xctx_out->xts.key2 = &xctx_out->ks2;
3422 }
3423 return 1;
3424 } else if (type != EVP_CTRL_INIT)
3425 return -1;
3426 /* key1 and key2 are used as an indicator both key and IV are set */
3427 xctx->xts.key1 = NULL;
3428 xctx->xts.key2 = NULL;
3429 return 1;
3430}
32a2d8dd
DSH		 3431
3432static int aes_xts_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 3433 const unsigned char *iv, int enc)
3434{
6435f0f6 3435 EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
0f113f3e
MC		 3436 if (!iv && !key)
3437 return 1;
3438
3538b0f7 3439 if (key) {
0f113f3e 3440 do {
3538b0f7
P		 3441 /* The key is two half length keys in reality */
3442 const int bytes = EVP_CIPHER_CTX_key_length(ctx) / 2;
3443 const int bits = bytes * 8;
3444
3445 /*
3446 * Verify that the two keys are different.
3447 *
3448 * This addresses the vulnerability described in Rogaway's
3449 * September 2004 paper:
3450 *
3451 * "Efficient Instantiations of Tweakable Blockciphers and
3452 * Refinements to Modes OCB and PMAC".
3453 * (http://web.cs.ucdavis.edu/~rogaway/papers/offsets.pdf)
3454 *
3455 * FIPS 140-2 IG A.9 XTS-AES Key Generation Requirements states
3456 * that:
3457 * "The check for Key_1 != Key_2 shall be done at any place
3458 * BEFORE using the keys in the XTS-AES algorithm to process
3459 * data with them."
3460 */
3461 if (memcmp(key, key + bytes, bytes) == 0) {
3462 EVPerr(EVP_F_AES_XTS_INIT_KEY, EVP_R_XTS_DUPLICATED_KEYS);
3463 return 0;
3464 }
3465
5158c763 3466#ifdef AES_XTS_ASM
0f113f3e 3467 xctx->stream = enc ? AES_xts_encrypt : AES_xts_decrypt;
5158c763 3468#else
0f113f3e 3469 xctx->stream = NULL;
5158c763 3470#endif
0f113f3e 3471 /* key_len is two AES keys */
5158c763 3472#ifdef HWAES_CAPABLE
0f113f3e
MC		 3473 if (HWAES_CAPABLE) {
3474 if (enc) {
3538b0f7 3475 HWAES_set_encrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e 3476 xctx->xts.block1 = (block128_f) HWAES_encrypt;
46f047d7
AP		 3477# ifdef HWAES_xts_encrypt
3478 xctx->stream = HWAES_xts_encrypt;
3479# endif
0f113f3e 3480 } else {
3538b0f7 3481 HWAES_set_decrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e 3482 xctx->xts.block1 = (block128_f) HWAES_decrypt;
46f047d7
AP		 3483# ifdef HWAES_xts_decrypt
3484 xctx->stream = HWAES_xts_decrypt;
3485#endif
0f113f3e
MC		 3486 }
3487
3538b0f7 3488 HWAES_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
0f113f3e
MC		 3489 xctx->xts.block2 = (block128_f) HWAES_encrypt;
3490
3491 xctx->xts.key1 = &xctx->ks1;
3492 break;
3493 } else
5158c763
MC		 3494#endif
3495#ifdef BSAES_CAPABLE
0f113f3e
MC		 3496 if (BSAES_CAPABLE)
3497 xctx->stream = enc ? bsaes_xts_encrypt : bsaes_xts_decrypt;
3498 else
5158c763
MC		 3499#endif
3500#ifdef VPAES_CAPABLE
0f113f3e
MC		 3501 if (VPAES_CAPABLE) {
3502 if (enc) {
3538b0f7 3503 vpaes_set_encrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 3504 xctx->xts.block1 = (block128_f) vpaes_encrypt;
3505 } else {
3538b0f7 3506 vpaes_set_decrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 3507 xctx->xts.block1 = (block128_f) vpaes_decrypt;
3508 }
3509
3538b0f7 3510 vpaes_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
0f113f3e
MC		 3511 xctx->xts.block2 = (block128_f) vpaes_encrypt;
3512
3513 xctx->xts.key1 = &xctx->ks1;
3514 break;
3515 } else
5158c763 3516#endif
0f113f3e
MC		 3517 (void)0; /* terminate potentially open 'else' */
3518
3519 if (enc) {
3538b0f7 3520 AES_set_encrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 3521 xctx->xts.block1 = (block128_f) AES_encrypt;
3522 } else {
3538b0f7 3523 AES_set_decrypt_key(key, bits, &xctx->ks1.ks);
0f113f3e
MC		 3524 xctx->xts.block1 = (block128_f) AES_decrypt;
3525 }
3526
3538b0f7 3527 AES_set_encrypt_key(key + bytes, bits, &xctx->ks2.ks);
0f113f3e
MC		 3528 xctx->xts.block2 = (block128_f) AES_encrypt;
3529
3530 xctx->xts.key1 = &xctx->ks1;
3531 } while (0);
3538b0f7 3532 }
0f113f3e
MC		 3533
3534 if (iv) {
3535 xctx->xts.key2 = &xctx->ks2;
6435f0f6 3536 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 16);
0f113f3e
MC		 3537 }
3538
3539 return 1;
3540}
32a2d8dd 3541
17f121de 3542static int aes_xts_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 3543 const unsigned char *in, size_t len)
3544{
6435f0f6 3545 EVP_AES_XTS_CTX *xctx = EVP_C_DATA(EVP_AES_XTS_CTX,ctx);
95eda4f0
P		 3546
3547 if (xctx->xts.key1 == NULL
3548 || xctx->xts.key2 == NULL
3549 || out == NULL
3550 || in == NULL
3551 || len < AES_BLOCK_SIZE)
0f113f3e 3552 return 0;
95eda4f0 3553
5516c19b
P		 3554 /*
3555 * Impose a limit of 2^20 blocks per data unit as specifed by
3556 * IEEE Std 1619-2018. The earlier and obsolete IEEE Std 1619-2007
3557 * indicated that this was a SHOULD NOT rather than a MUST NOT.
3558 * NIST SP 800-38E mandates the same limit.
3559 */
3560 if (len > XTS_MAX_BLOCKS_PER_DATA_UNIT * AES_BLOCK_SIZE) {
3561 EVPerr(EVP_F_AES_XTS_CIPHER, EVP_R_XTS_DATA_UNIT_IS_TOO_LARGE);
3562 return 0;
3563 }
3564
0f113f3e
MC		 3565 if (xctx->stream)
3566 (*xctx->stream) (in, out, len,
6435f0f6
RL		 3567 xctx->xts.key1, xctx->xts.key2,
3568 EVP_CIPHER_CTX_iv_noconst(ctx));
3569 else if (CRYPTO_xts128_encrypt(&xctx->xts, EVP_CIPHER_CTX_iv_noconst(ctx),
3570 in, out, len,
3571 EVP_CIPHER_CTX_encrypting(ctx)))
0f113f3e
MC		 3572 return 0;
3573 return 1;
3574}
3575
5158c763 3576#define aes_xts_cleanup NULL
0f113f3e 3577
5158c763 3578#define XTS_FLAGS (EVP_CIPH_FLAG_DEFAULT_ASN1 | EVP_CIPH_CUSTOM_IV \
0f113f3e
MC		 3579 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CTRL_INIT \
3580 | EVP_CIPH_CUSTOM_COPY)
3581
3582BLOCK_CIPHER_custom(NID_aes, 128, 1, 16, xts, XTS, XTS_FLAGS)
3583 BLOCK_CIPHER_custom(NID_aes, 256, 1, 16, xts, XTS, XTS_FLAGS)
23916810
DSH		 3584
3585static int aes_ccm_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
0f113f3e 3586{
6435f0f6 3587 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,c);
0f113f3e
MC		 3588 switch (type) {
3589 case EVP_CTRL_INIT:
3590 cctx->key_set = 0;
3591 cctx->iv_set = 0;
3592 cctx->L = 8;
3593 cctx->M = 12;
3594 cctx->tag_set = 0;
3595 cctx->len_set = 0;
e75c5a79
DSH		 3596 cctx->tls_aad_len = -1;
3597 return 1;
3598
3599 case EVP_CTRL_AEAD_TLS1_AAD:
3600 /* Save the AAD for later use */
3601 if (arg != EVP_AEAD_TLS1_AAD_LEN)
3602 return 0;
6435f0f6 3603 memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg);
e75c5a79
DSH		 3604 cctx->tls_aad_len = arg;
3605 {
6435f0f6
RL		 3606 uint16_t len =
3607 EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] << 8
3608 | EVP_CIPHER_CTX_buf_noconst(c)[arg - 1];
e75c5a79 3609 /* Correct length for explicit IV */
2198b3a5
AP		 3610 if (len < EVP_CCM_TLS_EXPLICIT_IV_LEN)
3611 return 0;
e75c5a79
DSH		 3612 len -= EVP_CCM_TLS_EXPLICIT_IV_LEN;
3613 /* If decrypting correct for tag too */
2198b3a5
AP		 3614 if (!EVP_CIPHER_CTX_encrypting(c)) {
3615 if (len < cctx->M)
3616 return 0;
e75c5a79 3617 len -= cctx->M;
2198b3a5 3618 }
6435f0f6
RL		 3619 EVP_CIPHER_CTX_buf_noconst(c)[arg - 2] = len >> 8;
3620 EVP_CIPHER_CTX_buf_noconst(c)[arg - 1] = len & 0xff;
e75c5a79
DSH		 3621 }
3622 /* Extra padding: tag appended to record */
3623 return cctx->M;
3624
3625 case EVP_CTRL_CCM_SET_IV_FIXED:
3626 /* Sanity check length */
3627 if (arg != EVP_CCM_TLS_FIXED_IV_LEN)
3628 return 0;
3629 /* Just copy to first part of IV */
6435f0f6 3630 memcpy(EVP_CIPHER_CTX_iv_noconst(c), ptr, arg);
0f113f3e
MC		 3631 return 1;
3632
e640fa02 3633 case EVP_CTRL_AEAD_SET_IVLEN:
0f113f3e 3634 arg = 15 - arg;
018fcbec 3635 /* fall thru */
0f113f3e
MC		 3636 case EVP_CTRL_CCM_SET_L:
3637 if (arg < 2 || arg > 8)
3638 return 0;
3639 cctx->L = arg;
3640 return 1;
3641
e640fa02 3642 case EVP_CTRL_AEAD_SET_TAG:
0f113f3e
MC		 3643 if ((arg & 1) || arg < 4 || arg > 16)
3644 return 0;
6435f0f6 3645 if (EVP_CIPHER_CTX_encrypting(c) && ptr)
0f113f3e
MC		 3646 return 0;
3647 if (ptr) {
3648 cctx->tag_set = 1;
6435f0f6 3649 memcpy(EVP_CIPHER_CTX_buf_noconst(c), ptr, arg);
0f113f3e
MC		 3650 }
3651 cctx->M = arg;
3652 return 1;
3653
e640fa02 3654 case EVP_CTRL_AEAD_GET_TAG:
6435f0f6 3655 if (!EVP_CIPHER_CTX_encrypting(c) || !cctx->tag_set)
0f113f3e
MC		 3656 return 0;
3657 if (!CRYPTO_ccm128_tag(&cctx->ccm, ptr, (size_t)arg))
3658 return 0;
3659 cctx->tag_set = 0;
3660 cctx->iv_set = 0;
3661 cctx->len_set = 0;
3662 return 1;
3663
3664 case EVP_CTRL_COPY:
3665 {
3666 EVP_CIPHER_CTX *out = ptr;
6435f0f6 3667 EVP_AES_CCM_CTX *cctx_out = EVP_C_DATA(EVP_AES_CCM_CTX,out);
0f113f3e
MC		 3668 if (cctx->ccm.key) {
3669 if (cctx->ccm.key != &cctx->ks)
3670 return 0;
3671 cctx_out->ccm.key = &cctx_out->ks;
3672 }
3673 return 1;
3674 }
3675
3676 default:
3677 return -1;
3678
3679 }
3680}
23916810
DSH		 3681
3682static int aes_ccm_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 3683 const unsigned char *iv, int enc)
3684{
6435f0f6 3685 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx);
0f113f3e
MC		 3686 if (!iv && !key)
3687 return 1;
3688 if (key)
3689 do {
5158c763 3690#ifdef HWAES_CAPABLE
0f113f3e 3691 if (HWAES_CAPABLE) {
6435f0f6
RL		 3692 HWAES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
3693 &cctx->ks.ks);
0f113f3e
MC		 3694
3695 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
3696 &cctx->ks, (block128_f) HWAES_encrypt);
3697 cctx->str = NULL;
3698 cctx->key_set = 1;
3699 break;
3700 } else
5158c763
MC		 3701#endif
3702#ifdef VPAES_CAPABLE
0f113f3e 3703 if (VPAES_CAPABLE) {
6435f0f6
RL		 3704 vpaes_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
3705 &cctx->ks.ks);
0f113f3e
MC		 3706 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
3707 &cctx->ks, (block128_f) vpaes_encrypt);
3708 cctx->str = NULL;
3709 cctx->key_set = 1;
3710 break;
3711 }
5158c763 3712#endif
6435f0f6
RL		 3713 AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
3714 &cctx->ks.ks);
0f113f3e
MC		 3715 CRYPTO_ccm128_init(&cctx->ccm, cctx->M, cctx->L,
3716 &cctx->ks, (block128_f) AES_encrypt);
3717 cctx->str = NULL;
3718 cctx->key_set = 1;
3719 } while (0);
3720 if (iv) {
6435f0f6 3721 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, 15 - cctx->L);
0f113f3e
MC		 3722 cctx->iv_set = 1;
3723 }
3724 return 1;
3725}
23916810 3726
e75c5a79
DSH		 3727static int aes_ccm_tls_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
3728 const unsigned char *in, size_t len)
3729{
6435f0f6 3730 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx);
e75c5a79
DSH		 3731 CCM128_CONTEXT *ccm = &cctx->ccm;
3732 /* Encrypt/decrypt must be performed in place */
3733 if (out != in || len < (EVP_CCM_TLS_EXPLICIT_IV_LEN + (size_t)cctx->M))
3734 return -1;
3735 /* If encrypting set explicit IV from sequence number (start of AAD) */
6435f0f6
RL		 3736 if (EVP_CIPHER_CTX_encrypting(ctx))
3737 memcpy(out, EVP_CIPHER_CTX_buf_noconst(ctx),
3738 EVP_CCM_TLS_EXPLICIT_IV_LEN);
e75c5a79 3739 /* Get rest of IV from explicit IV */
6435f0f6
RL		 3740 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx) + EVP_CCM_TLS_FIXED_IV_LEN, in,
3741 EVP_CCM_TLS_EXPLICIT_IV_LEN);
e75c5a79
DSH		 3742 /* Correct length value */
3743 len -= EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->M;
6435f0f6
RL		 3744 if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx), 15 - cctx->L,
3745 len))
e75c5a79
DSH		 3746 return -1;
3747 /* Use saved AAD */
6435f0f6 3748 CRYPTO_ccm128_aad(ccm, EVP_CIPHER_CTX_buf_noconst(ctx), cctx->tls_aad_len);
e75c5a79
DSH		 3749 /* Fix buffer to point to payload */
3750 in += EVP_CCM_TLS_EXPLICIT_IV_LEN;
3751 out += EVP_CCM_TLS_EXPLICIT_IV_LEN;
6435f0f6 3752 if (EVP_CIPHER_CTX_encrypting(ctx)) {
e75c5a79
DSH		 3753 if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len,
3754 cctx->str) :
3755 CRYPTO_ccm128_encrypt(ccm, in, out, len))
3756 return -1;
3757 if (!CRYPTO_ccm128_tag(ccm, out + len, cctx->M))
3758 return -1;
3759 return len + EVP_CCM_TLS_EXPLICIT_IV_LEN + cctx->M;
3760 } else {
3761 if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len,
3762 cctx->str) :
3763 !CRYPTO_ccm128_decrypt(ccm, in, out, len)) {
3764 unsigned char tag[16];
3765 if (CRYPTO_ccm128_tag(ccm, tag, cctx->M)) {
3766 if (!CRYPTO_memcmp(tag, in + len, cctx->M))
3767 return len;
3768 }
3769 }
3770 OPENSSL_cleanse(out, len);
3771 return -1;
3772 }
3773}
3774
17f121de 3775static int aes_ccm_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 3776 const unsigned char *in, size_t len)
3777{
6435f0f6 3778 EVP_AES_CCM_CTX *cctx = EVP_C_DATA(EVP_AES_CCM_CTX,ctx);
0f113f3e
MC		 3779 CCM128_CONTEXT *ccm = &cctx->ccm;
3780 /* If not set up, return error */
e75c5a79
DSH		 3781 if (!cctx->key_set)
3782 return -1;
3783
3784 if (cctx->tls_aad_len >= 0)
3785 return aes_ccm_tls_cipher(ctx, out, in, len);
3786
197421b1
DSH		 3787 /* EVP_*Final() doesn't return any data */
3788 if (in == NULL && out != NULL)
3789 return 0;
3790
e75c5a79 3791 if (!cctx->iv_set)
0f113f3e 3792 return -1;
e75c5a79 3793
6435f0f6 3794 if (!EVP_CIPHER_CTX_encrypting(ctx) && !cctx->tag_set)
0f113f3e
MC		 3795 return -1;
3796 if (!out) {
3797 if (!in) {
6435f0f6
RL		 3798 if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx),
3799 15 - cctx->L, len))
0f113f3e
MC		 3800 return -1;
3801 cctx->len_set = 1;
3802 return len;
3803 }
3804 /* If have AAD need message length */
3805 if (!cctx->len_set && len)
3806 return -1;
3807 CRYPTO_ccm128_aad(ccm, in, len);
3808 return len;
3809 }
0f113f3e
MC		 3810 /* If not set length yet do it */
3811 if (!cctx->len_set) {
6435f0f6
RL		 3812 if (CRYPTO_ccm128_setiv(ccm, EVP_CIPHER_CTX_iv_noconst(ctx),
3813 15 - cctx->L, len))
0f113f3e
MC		 3814 return -1;
3815 cctx->len_set = 1;
3816 }
6435f0f6 3817 if (EVP_CIPHER_CTX_encrypting(ctx)) {
0f113f3e
MC		 3818 if (cctx->str ? CRYPTO_ccm128_encrypt_ccm64(ccm, in, out, len,
3819 cctx->str) :
3820 CRYPTO_ccm128_encrypt(ccm, in, out, len))
3821 return -1;
3822 cctx->tag_set = 1;
3823 return len;
3824 } else {
3825 int rv = -1;
3826 if (cctx->str ? !CRYPTO_ccm128_decrypt_ccm64(ccm, in, out, len,
3827 cctx->str) :
3828 !CRYPTO_ccm128_decrypt(ccm, in, out, len)) {
3829 unsigned char tag[16];
3830 if (CRYPTO_ccm128_tag(ccm, tag, cctx->M)) {
6435f0f6
RL		 3831 if (!CRYPTO_memcmp(tag, EVP_CIPHER_CTX_buf_noconst(ctx),
3832 cctx->M))
0f113f3e
MC		 3833 rv = len;
3834 }
3835 }
3836 if (rv == -1)
3837 OPENSSL_cleanse(out, len);
3838 cctx->iv_set = 0;
3839 cctx->tag_set = 0;
3840 cctx->len_set = 0;
3841 return rv;
3842 }
0f113f3e
MC		 3843}
3844
5158c763 3845#define aes_ccm_cleanup NULL
0f113f3e 3846
e75c5a79
DSH		 3847BLOCK_CIPHER_custom(NID_aes, 128, 1, 12, ccm, CCM,
3848 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
3849 BLOCK_CIPHER_custom(NID_aes, 192, 1, 12, ccm, CCM,
3850 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
3851 BLOCK_CIPHER_custom(NID_aes, 256, 1, 12, ccm, CCM,
3852 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
0f113f3e
MC		 3853
3854typedef struct {
3855 union {
3856 double align;
3857 AES_KEY ks;
3858 } ks;
3859 /* Indicates if IV has been set */
3860 unsigned char *iv;
3861} EVP_AES_WRAP_CTX;
97cf1f6c
DSH		 3862
3863static int aes_wrap_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 3864 const unsigned char *iv, int enc)
3865{
6435f0f6 3866 EVP_AES_WRAP_CTX *wctx = EVP_C_DATA(EVP_AES_WRAP_CTX,ctx);
0f113f3e
MC		 3867 if (!iv && !key)
3868 return 1;
3869 if (key) {
6435f0f6
RL		 3870 if (EVP_CIPHER_CTX_encrypting(ctx))
3871 AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
3872 &wctx->ks.ks);
0f113f3e 3873 else
6435f0f6
RL		 3874 AES_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
3875 &wctx->ks.ks);
0f113f3e
MC		 3876 if (!iv)
3877 wctx->iv = NULL;
3878 }
3879 if (iv) {
6435f0f6
RL		 3880 memcpy(EVP_CIPHER_CTX_iv_noconst(ctx), iv, EVP_CIPHER_CTX_iv_length(ctx));
3881 wctx->iv = EVP_CIPHER_CTX_iv_noconst(ctx);
0f113f3e
MC		 3882 }
3883 return 1;
3884}
97cf1f6c
DSH		 3885
3886static int aes_wrap_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 3887 const unsigned char *in, size_t inlen)
3888{
6435f0f6 3889 EVP_AES_WRAP_CTX *wctx = EVP_C_DATA(EVP_AES_WRAP_CTX,ctx);
0f113f3e
MC		 3890 size_t rv;
3891 /* AES wrap with padding has IV length of 4, without padding 8 */
3892 int pad = EVP_CIPHER_CTX_iv_length(ctx) == 4;
3893 /* No final operation so always return zero length */
3894 if (!in)
3895 return 0;
3896 /* Input length must always be non-zero */
3897 if (!inlen)
3898 return -1;
3899 /* If decrypting need at least 16 bytes and multiple of 8 */
6435f0f6 3900 if (!EVP_CIPHER_CTX_encrypting(ctx) && (inlen < 16 || inlen & 0x7))
0f113f3e
MC		 3901 return -1;
3902 /* If not padding input must be multiple of 8 */
3903 if (!pad && inlen & 0x7)
3904 return -1;
7141ba31
MC		 3905 if (is_partially_overlapping(out, in, inlen)) {
3906 EVPerr(EVP_F_AES_WRAP_CIPHER, EVP_R_PARTIALLY_OVERLAPPING);
3907 return 0;
3908 }
0f113f3e 3909 if (!out) {
6435f0f6 3910 if (EVP_CIPHER_CTX_encrypting(ctx)) {
0f113f3e
MC		 3911 /* If padding round up to multiple of 8 */
3912 if (pad)
3913 inlen = (inlen + 7) / 8 * 8;
3914 /* 8 byte prefix */
3915 return inlen + 8;
3916 } else {
3917 /*
3918 * If not padding output will be exactly 8 bytes smaller than
3919 * input. If padding it will be at least 8 bytes smaller but we
3920 * don't know how much.
3921 */
3922 return inlen - 8;
3923 }
3924 }
3925 if (pad) {
6435f0f6 3926 if (EVP_CIPHER_CTX_encrypting(ctx))
0f113f3e
MC		 3927 rv = CRYPTO_128_wrap_pad(&wctx->ks.ks, wctx->iv,
3928 out, in, inlen,
3929 (block128_f) AES_encrypt);
3930 else
3931 rv = CRYPTO_128_unwrap_pad(&wctx->ks.ks, wctx->iv,
3932 out, in, inlen,
3933 (block128_f) AES_decrypt);
3934 } else {
6435f0f6 3935 if (EVP_CIPHER_CTX_encrypting(ctx))
0f113f3e
MC		 3936 rv = CRYPTO_128_wrap(&wctx->ks.ks, wctx->iv,
3937 out, in, inlen, (block128_f) AES_encrypt);
3938 else
3939 rv = CRYPTO_128_unwrap(&wctx->ks.ks, wctx->iv,
3940 out, in, inlen, (block128_f) AES_decrypt);
3941 }
3942 return rv ? (int)rv : -1;
3943}
3944
5158c763 3945#define WRAP_FLAGS (EVP_CIPH_WRAP_MODE \
0f113f3e
MC		 3946 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
3947 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_FLAG_DEFAULT_ASN1)
97cf1f6c
DSH		 3948
3949static const EVP_CIPHER aes_128_wrap = {
0f113f3e
MC		 3950 NID_id_aes128_wrap,
3951 8, 16, 8, WRAP_FLAGS,
3952 aes_wrap_init_key, aes_wrap_cipher,
3953 NULL,
3954 sizeof(EVP_AES_WRAP_CTX),
3955 NULL, NULL, NULL, NULL
3956};
97cf1f6c
DSH		 3957
3958const EVP_CIPHER *EVP_aes_128_wrap(void)
0f113f3e
MC		 3959{
3960 return &aes_128_wrap;
3961}
97cf1f6c
DSH		 3962
3963static const EVP_CIPHER aes_192_wrap = {
0f113f3e
MC		 3964 NID_id_aes192_wrap,
3965 8, 24, 8, WRAP_FLAGS,
3966 aes_wrap_init_key, aes_wrap_cipher,
3967 NULL,
3968 sizeof(EVP_AES_WRAP_CTX),
3969 NULL, NULL, NULL, NULL
3970};
97cf1f6c
DSH		 3971
3972const EVP_CIPHER *EVP_aes_192_wrap(void)
0f113f3e
MC		 3973{
3974 return &aes_192_wrap;
3975}
97cf1f6c
DSH		 3976
3977static const EVP_CIPHER aes_256_wrap = {
0f113f3e
MC		 3978 NID_id_aes256_wrap,
3979 8, 32, 8, WRAP_FLAGS,
3980 aes_wrap_init_key, aes_wrap_cipher,
3981 NULL,
3982 sizeof(EVP_AES_WRAP_CTX),
3983 NULL, NULL, NULL, NULL
3984};
97cf1f6c
DSH		 3985
3986const EVP_CIPHER *EVP_aes_256_wrap(void)
0f113f3e
MC		 3987{
3988 return &aes_256_wrap;
3989}
97cf1f6c 3990
d31fed73 3991static const EVP_CIPHER aes_128_wrap_pad = {
0f113f3e
MC		 3992 NID_id_aes128_wrap_pad,
3993 8, 16, 4, WRAP_FLAGS,
3994 aes_wrap_init_key, aes_wrap_cipher,
3995 NULL,
3996 sizeof(EVP_AES_WRAP_CTX),
3997 NULL, NULL, NULL, NULL
3998};
d31fed73
DSH		 3999
4000const EVP_CIPHER *EVP_aes_128_wrap_pad(void)
0f113f3e
MC		 4001{
4002 return &aes_128_wrap_pad;
4003}
d31fed73
DSH		 4004
4005static const EVP_CIPHER aes_192_wrap_pad = {
0f113f3e
MC		 4006 NID_id_aes192_wrap_pad,
4007 8, 24, 4, WRAP_FLAGS,
4008 aes_wrap_init_key, aes_wrap_cipher,
4009 NULL,
4010 sizeof(EVP_AES_WRAP_CTX),
4011 NULL, NULL, NULL, NULL
4012};
d31fed73
DSH		 4013
4014const EVP_CIPHER *EVP_aes_192_wrap_pad(void)
0f113f3e
MC		 4015{
4016 return &aes_192_wrap_pad;
4017}
d31fed73
DSH		 4018
4019static const EVP_CIPHER aes_256_wrap_pad = {
0f113f3e
MC		 4020 NID_id_aes256_wrap_pad,
4021 8, 32, 4, WRAP_FLAGS,
4022 aes_wrap_init_key, aes_wrap_cipher,
4023 NULL,
4024 sizeof(EVP_AES_WRAP_CTX),
4025 NULL, NULL, NULL, NULL
4026};
d31fed73
DSH		 4027
4028const EVP_CIPHER *EVP_aes_256_wrap_pad(void)
0f113f3e
MC		 4029{
4030 return &aes_256_wrap_pad;
4031}
d31fed73 4032
5158c763 4033#ifndef OPENSSL_NO_OCB
e6b336ef 4034static int aes_ocb_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
0f113f3e 4035{
6435f0f6 4036 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,c);
0f113f3e
MC		 4037 EVP_CIPHER_CTX *newc;
4038 EVP_AES_OCB_CTX *new_octx;
4039
4040 switch (type) {
4041 case EVP_CTRL_INIT:
4042 octx->key_set = 0;
4043 octx->iv_set = 0;
6435f0f6
RL		 4044 octx->ivlen = EVP_CIPHER_CTX_iv_length(c);
4045 octx->iv = EVP_CIPHER_CTX_iv_noconst(c);
0f113f3e
MC		 4046 octx->taglen = 16;
4047 octx->data_buf_len = 0;
4048 octx->aad_buf_len = 0;
4049 return 1;
4050
e640fa02 4051 case EVP_CTRL_AEAD_SET_IVLEN:
0f113f3e
MC		 4052 /* IV len must be 1 to 15 */
4053 if (arg <= 0 || arg > 15)
4054 return 0;
4055
4056 octx->ivlen = arg;
4057 return 1;
4058
e640fa02 4059 case EVP_CTRL_AEAD_SET_TAG:
d57d135c
MC		 4060 if (!ptr) {
4061 /* Tag len must be 0 to 16 */
4062 if (arg < 0 || arg > 16)
4063 return 0;
4064
4065 octx->taglen = arg;
4066 return 1;
4067 }
6435f0f6 4068 if (arg != octx->taglen || EVP_CIPHER_CTX_encrypting(c))
0f113f3e
MC		 4069 return 0;
4070 memcpy(octx->tag, ptr, arg);
4071 return 1;
4072
e640fa02 4073 case EVP_CTRL_AEAD_GET_TAG:
6435f0f6 4074 if (arg != octx->taglen || !EVP_CIPHER_CTX_encrypting(c))
0f113f3e
MC		 4075 return 0;
4076
4077 memcpy(ptr, octx->tag, arg);
4078 return 1;
4079
4080 case EVP_CTRL_COPY:
4081 newc = (EVP_CIPHER_CTX *)ptr;
6435f0f6 4082 new_octx = EVP_C_DATA(EVP_AES_OCB_CTX,newc);
0f113f3e 4083 return CRYPTO_ocb128_copy_ctx(&new_octx->ocb, &octx->ocb,
bdc985b1
AP		 4084 &new_octx->ksenc.ks,
4085 &new_octx->ksdec.ks);
0f113f3e
MC		 4086
4087 default:
4088 return -1;
4089
4090 }
4091}
e6b336ef 4092
5158c763
MC		 4093# ifdef HWAES_CAPABLE
4094# ifdef HWAES_ocb_encrypt
02dc0b82
AP		 4095void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
4096 size_t blocks, const void *key,
4097 size_t start_block_num,
4098 unsigned char offset_i[16],
4099 const unsigned char L_[][16],
4100 unsigned char checksum[16]);
5158c763 4101# else
365f95ad 4102# define HWAES_ocb_encrypt ((ocb128_f)NULL)
5158c763
MC		 4103# endif
4104# ifdef HWAES_ocb_decrypt
02dc0b82
AP		 4105void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
4106 size_t blocks, const void *key,
4107 size_t start_block_num,
4108 unsigned char offset_i[16],
4109 const unsigned char L_[][16],
4110 unsigned char checksum[16]);
5158c763 4111# else
365f95ad 4112# define HWAES_ocb_decrypt ((ocb128_f)NULL)
02dc0b82 4113# endif
5158c763 4114# endif
02dc0b82 4115
e6b336ef 4116static int aes_ocb_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
0f113f3e
MC		 4117 const unsigned char *iv, int enc)
4118{
6435f0f6 4119 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx);
0f113f3e
MC		 4120 if (!iv && !key)
4121 return 1;
4122 if (key) {
4123 do {
4124 /*
4125 * We set both the encrypt and decrypt key here because decrypt
4126 * needs both. We could possibly optimise to remove setting the
4127 * decrypt for an encryption operation.
4128 */
5158c763 4129# ifdef HWAES_CAPABLE
02dc0b82 4130 if (HWAES_CAPABLE) {
6435f0f6
RL		 4131 HWAES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4132 &octx->ksenc.ks);
4133 HWAES_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4134 &octx->ksdec.ks);
02dc0b82
AP		 4135 if (!CRYPTO_ocb128_init(&octx->ocb,
4136 &octx->ksenc.ks, &octx->ksdec.ks,
4137 (block128_f) HWAES_encrypt,
4138 (block128_f) HWAES_decrypt,
4139 enc ? HWAES_ocb_encrypt
4140 : HWAES_ocb_decrypt))
4141 return 0;
4142 break;
4143 }
5158c763
MC		 4144# endif
4145# ifdef VPAES_CAPABLE
0f113f3e 4146 if (VPAES_CAPABLE) {
6435f0f6
RL		 4147 vpaes_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4148 &octx->ksenc.ks);
4149 vpaes_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4150 &octx->ksdec.ks);
bdc985b1
AP		 4151 if (!CRYPTO_ocb128_init(&octx->ocb,
4152 &octx->ksenc.ks, &octx->ksdec.ks,
4153 (block128_f) vpaes_encrypt,
bd30091c
AP		 4154 (block128_f) vpaes_decrypt,
4155 NULL))
0f113f3e
MC		 4156 return 0;
4157 break;
4158 }
5158c763 4159# endif
6435f0f6
RL		 4160 AES_set_encrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4161 &octx->ksenc.ks);
4162 AES_set_decrypt_key(key, EVP_CIPHER_CTX_key_length(ctx) * 8,
4163 &octx->ksdec.ks);
bdc985b1
AP		 4164 if (!CRYPTO_ocb128_init(&octx->ocb,
4165 &octx->ksenc.ks, &octx->ksdec.ks,
0f113f3e 4166 (block128_f) AES_encrypt,
bd30091c
AP		 4167 (block128_f) AES_decrypt,
4168 NULL))
0f113f3e
MC		 4169 return 0;
4170 }
4171 while (0);
4172
4173 /*
4174 * If we have an iv we can set it directly, otherwise use saved IV.
4175 */
4176 if (iv == NULL && octx->iv_set)
4177 iv = octx->iv;
4178 if (iv) {
4179 if (CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen)
4180 != 1)
4181 return 0;
4182 octx->iv_set = 1;
4183 }
4184 octx->key_set = 1;
4185 } else {
4186 /* If key set use IV, otherwise copy */
4187 if (octx->key_set)
4188 CRYPTO_ocb128_setiv(&octx->ocb, iv, octx->ivlen, octx->taglen);
4189 else
4190 memcpy(octx->iv, iv, octx->ivlen);
4191 octx->iv_set = 1;
4192 }
4193 return 1;
4194}
e6b336ef
MC		 4195
4196static int aes_ocb_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
0f113f3e
MC		 4197 const unsigned char *in, size_t len)
4198{
4199 unsigned char *buf;
4200 int *buf_len;
4201 int written_len = 0;
4202 size_t trailing_len;
6435f0f6 4203 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,ctx);
0f113f3e
MC		 4204
4205 /* If IV or Key not set then return error */
4206 if (!octx->iv_set)
4207 return -1;
4208
4209 if (!octx->key_set)
4210 return -1;
4211
0ba5a9ea 4212 if (in != NULL) {
0f113f3e
MC		 4213 /*
4214 * Need to ensure we are only passing full blocks to low level OCB
4215 * routines. We do it here rather than in EVP_EncryptUpdate/
4216 * EVP_DecryptUpdate because we need to pass full blocks of AAD too
4217 * and those routines don't support that
4218 */
4219
4220 /* Are we dealing with AAD or normal data here? */
4221 if (out == NULL) {
4222 buf = octx->aad_buf;
4223 buf_len = &(octx->aad_buf_len);
4224 } else {
4225 buf = octx->data_buf;
4226 buf_len = &(octx->data_buf_len);
7141ba31
MC		 4227
4228 if (is_partially_overlapping(out + *buf_len, in, len)) {
4229 EVPerr(EVP_F_AES_OCB_CIPHER, EVP_R_PARTIALLY_OVERLAPPING);
4230 return 0;
4231 }
0f113f3e
MC		 4232 }
4233
4234 /*
4235 * If we've got a partially filled buffer from a previous call then
4236 * use that data first
4237 */
0ba5a9ea 4238 if (*buf_len > 0) {
0f113f3e
MC		 4239 unsigned int remaining;
4240
0ba5a9ea 4241 remaining = AES_BLOCK_SIZE - (*buf_len);
0f113f3e
MC		 4242 if (remaining > len) {
4243 memcpy(buf + (*buf_len), in, len);
4244 *(buf_len) += len;
4245 return 0;
4246 }
4247 memcpy(buf + (*buf_len), in, remaining);
4248
4249 /*
4250 * If we get here we've filled the buffer, so process it
4251 */
4252 len -= remaining;
4253 in += remaining;
4254 if (out == NULL) {
0ba5a9ea 4255 if (!CRYPTO_ocb128_aad(&octx->ocb, buf, AES_BLOCK_SIZE))
0f113f3e 4256 return -1;
6435f0f6 4257 } else if (EVP_CIPHER_CTX_encrypting(ctx)) {
0ba5a9ea
MC		 4258 if (!CRYPTO_ocb128_encrypt(&octx->ocb, buf, out,
4259 AES_BLOCK_SIZE))
0f113f3e
MC		 4260 return -1;
4261 } else {
0ba5a9ea
MC		 4262 if (!CRYPTO_ocb128_decrypt(&octx->ocb, buf, out,
4263 AES_BLOCK_SIZE))
0f113f3e
MC		 4264 return -1;
4265 }
0ba5a9ea 4266 written_len = AES_BLOCK_SIZE;
0f113f3e 4267 *buf_len = 0;
7c12c7b6
MC		 4268 if (out != NULL)
4269 out += AES_BLOCK_SIZE;
0f113f3e
MC		 4270 }
4271
4272 /* Do we have a partial block to handle at the end? */
0ba5a9ea 4273 trailing_len = len % AES_BLOCK_SIZE;
0f113f3e
MC		 4274
4275 /*
4276 * If we've got some full blocks to handle, then process these first
4277 */
4278 if (len != trailing_len) {
4279 if (out == NULL) {
4280 if (!CRYPTO_ocb128_aad(&octx->ocb, in, len - trailing_len))
4281 return -1;
6435f0f6 4282 } else if (EVP_CIPHER_CTX_encrypting(ctx)) {
0f113f3e
MC		 4283 if (!CRYPTO_ocb128_encrypt
4284 (&octx->ocb, in, out, len - trailing_len))
4285 return -1;
4286 } else {
4287 if (!CRYPTO_ocb128_decrypt
4288 (&octx->ocb, in, out, len - trailing_len))
4289 return -1;
4290 }
4291 written_len += len - trailing_len;
4292 in += len - trailing_len;
4293 }
4294
4295 /* Handle any trailing partial block */
0ba5a9ea 4296 if (trailing_len > 0) {
0f113f3e
MC		 4297 memcpy(buf, in, trailing_len);
4298 *buf_len = trailing_len;
4299 }
4300
4301 return written_len;
4302 } else {
4303 /*
4304 * First of all empty the buffer of any partial block that we might
4305 * have been provided - both for data and AAD
4306 */
0ba5a9ea 4307 if (octx->data_buf_len > 0) {
6435f0f6 4308 if (EVP_CIPHER_CTX_encrypting(ctx)) {
0f113f3e
MC		 4309 if (!CRYPTO_ocb128_encrypt(&octx->ocb, octx->data_buf, out,
4310 octx->data_buf_len))
4311 return -1;
4312 } else {
4313 if (!CRYPTO_ocb128_decrypt(&octx->ocb, octx->data_buf, out,
4314 octx->data_buf_len))
4315 return -1;
4316 }
4317 written_len = octx->data_buf_len;
4318 octx->data_buf_len = 0;
4319 }
0ba5a9ea 4320 if (octx->aad_buf_len > 0) {
0f113f3e
MC		 4321 if (!CRYPTO_ocb128_aad
4322 (&octx->ocb, octx->aad_buf, octx->aad_buf_len))
4323 return -1;
4324 octx->aad_buf_len = 0;
4325 }
4326 /* If decrypting then verify */
6435f0f6 4327 if (!EVP_CIPHER_CTX_encrypting(ctx)) {
0f113f3e
MC		 4328 if (octx->taglen < 0)
4329 return -1;
4330 if (CRYPTO_ocb128_finish(&octx->ocb,
4331 octx->tag, octx->taglen) != 0)
4332 return -1;
4333 octx->iv_set = 0;
4334 return written_len;
4335 }
4336 /* If encrypting then just get the tag */
4337 if (CRYPTO_ocb128_tag(&octx->ocb, octx->tag, 16) != 1)
4338 return -1;
4339 /* Don't reuse the IV */
4340 octx->iv_set = 0;
4341 return written_len;
4342 }
4343}
e6b336ef
MC		 4344
4345static int aes_ocb_cleanup(EVP_CIPHER_CTX *c)
0f113f3e 4346{
6435f0f6 4347 EVP_AES_OCB_CTX *octx = EVP_C_DATA(EVP_AES_OCB_CTX,c);
0f113f3e
MC		 4348 CRYPTO_ocb128_cleanup(&octx->ocb);
4349 return 1;
4350}
e6b336ef 4351
c4aede20
MC		 4352BLOCK_CIPHER_custom(NID_aes, 128, 16, 12, ocb, OCB,
4353 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
4354BLOCK_CIPHER_custom(NID_aes, 192, 16, 12, ocb, OCB,
4355 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
4356BLOCK_CIPHER_custom(NID_aes, 256, 16, 12, ocb, OCB,
4357 EVP_CIPH_FLAG_AEAD_CIPHER | CUSTOM_FLAGS)
5158c763 4358#endif /* OPENSSL_NO_OCB */
b1ceb439
TS		 4359
4360/* AES-SIV mode */
4361#ifndef OPENSSL_NO_SIV
4362
4363typedef SIV128_CONTEXT EVP_AES_SIV_CTX;
4364
4365#define aesni_siv_init_key aes_siv_init_key
4366static int aes_siv_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
4367 const unsigned char *iv, int enc)
4368{
4369 const EVP_CIPHER *ctr;
4370 const EVP_CIPHER *cbc;
4371 SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, ctx);
4372 int klen = EVP_CIPHER_CTX_key_length(ctx) / 2;
4373
4374 if (key == NULL)
4375 return 1;
4376
4377 switch (klen) {
4378 case 16:
4379 cbc = EVP_aes_128_cbc();
4380 ctr = EVP_aes_128_ctr();
4381 break;
4382 case 24:
4383 cbc = EVP_aes_192_cbc();
4384 ctr = EVP_aes_192_ctr();
4385 break;
4386 case 32:
4387 cbc = EVP_aes_256_cbc();
4388 ctr = EVP_aes_256_ctr();
4389 break;
4390 default:
4391 return 0;
4392 }
4393
4394 /* klen is the length of the underlying cipher, not the input key,
4395 which should be twice as long */
4396 return CRYPTO_siv128_init(sctx, key, klen, cbc, ctr);
4397}
4398
4399#define aesni_siv_cipher aes_siv_cipher
4400static int aes_siv_cipher(EVP_CIPHER_CTX *ctx, unsigned char *out,
4401 const unsigned char *in, size_t len)
4402{
4403 SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, ctx);
4404
4405 /* EncryptFinal or DecryptFinal */
4406 if (in == NULL)
4407 return CRYPTO_siv128_finish(sctx);
4408
4409 /* Deal with associated data */
4410 if (out == NULL)
4411 return CRYPTO_siv128_aad(sctx, in, len);
4412
4413 if (EVP_CIPHER_CTX_encrypting(ctx))
4414 return CRYPTO_siv128_encrypt(sctx, in, out, len);
4415
4416 return CRYPTO_siv128_decrypt(sctx, in, out, len);
4417}
4418
4419#define aesni_siv_cleanup aes_siv_cleanup
4420static int aes_siv_cleanup(EVP_CIPHER_CTX *c)
4421{
4422 SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, c);
4423
4424 return CRYPTO_siv128_cleanup(sctx);
4425}
4426
4427
4428#define aesni_siv_ctrl aes_siv_ctrl
4429static int aes_siv_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
4430{
4431 SIV128_CONTEXT *sctx = EVP_C_DATA(SIV128_CONTEXT, c);
4432 SIV128_CONTEXT *sctx_out;
4433
4434 switch (type) {
4435 case EVP_CTRL_INIT:
4436 return CRYPTO_siv128_cleanup(sctx);
4437
4438 case EVP_CTRL_SET_SPEED:
4439 return CRYPTO_siv128_speed(sctx, arg);
4440
4441 case EVP_CTRL_AEAD_SET_TAG:
4442 if (!EVP_CIPHER_CTX_encrypting(c))
4443 return CRYPTO_siv128_set_tag(sctx, ptr, arg);
4444 return 1;
4445
4446 case EVP_CTRL_AEAD_GET_TAG:
4447 if (!EVP_CIPHER_CTX_encrypting(c))
4448 return 0;
4449 return CRYPTO_siv128_get_tag(sctx, ptr, arg);
4450
4451 case EVP_CTRL_COPY:
4452 sctx_out = EVP_C_DATA(SIV128_CONTEXT, (EVP_CIPHER_CTX*)ptr);
4453 return CRYPTO_siv128_copy_ctx(sctx_out, sctx);
4454
4455 default:
4456 return -1;
4457
4458 }
4459}
4460
4461#define SIV_FLAGS (EVP_CIPH_FLAG_AEAD_CIPHER | EVP_CIPH_FLAG_DEFAULT_ASN1 \
4462 | EVP_CIPH_CUSTOM_IV | EVP_CIPH_FLAG_CUSTOM_CIPHER \
4463 | EVP_CIPH_ALWAYS_CALL_INIT | EVP_CIPH_CUSTOM_COPY \
4464 | EVP_CIPH_CTRL_INIT)
4465
4466BLOCK_CIPHER_custom(NID_aes, 128, 1, 0, siv, SIV, SIV_FLAGS)
4467BLOCK_CIPHER_custom(NID_aes, 192, 1, 0, siv, SIV, SIV_FLAGS)
4468BLOCK_CIPHER_custom(NID_aes, 256, 1, 0, siv, SIV, SIV_FLAGS)
4469#endif
A mirror of the official openssl repository