From: Marcel Cornu Date: Fri, 10 Apr 2026 09:36:25 +0000 (+0000) Subject: ML-DSA: Add AVX512VL SHAKE x4 multi-buffer integration X-Git-Url: http://git.ipfire.org/gitweb/?a=commitdiff_plain;h=a248ec771e8d111db6f69a29cab50cff7610f70b;p=thirdparty%2Fopenssl.git ML-DSA: Add AVX512VL SHAKE x4 multi-buffer integration Changes: - Adds new SHAKE x4 API to perform 4 SHAKE operations in parallel when AVX512VL is supported. - Adds AVX512VL Keccak x4 assembly module (keccak1600x4-avx512vl). - Adds internal SHA3 x4 APIs/context in sha3.h and wrappers in sha3_x4.c modules. - Adds runtime dispatch for ML-DSA sample operations with an OSSL_ML_DSA_SAMPLE_OPS vtable. Callers obtain the correct implementation via ossl_ml_dsa_sample_ops(), which returns either the generic scalar ops functions, or the AVX512VL multi-buffer ops depending on the build and CPU capabilities. - Adds x86-64 multi-buffer function implementation into ml_dsa_sample_hw_x86_64.inc, included in ml_dsa_sample.c when KECCAK1600_ASM and x86_64 are defined. Co-authored-by: Tomasz Kantecki Signed-off-by: Marcel Cornu Reviewed-by: Viktor Dukhovni Reviewed-by: Neil Horman Reviewed-by: Shane Lontis MergeDate: Mon Jul 13 15:55:54 2026 (Merged from https://github.com/openssl/openssl/pull/31090) --- diff --git a/CHANGES.md b/CHANGES.md index 6d4af0b8572..da1bfd15ec9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -31,6 +31,10 @@ OpenSSL Releases ### Changes between 4.0 and 4.1 [xx XXX xxxx] + * Added AVX512 optimized SHAKE x4 operations for ML-DSA on x86_64. + + *Marcel Cornu and Tomasz Kantecki* + * EC key point format simplification. The point conversion form (compressed, uncompressed, or hybrid) diff --git a/crypto/ml_dsa/ml_dsa_key.c b/crypto/ml_dsa/ml_dsa_key.c index 6b5b8b092fb..ea5f4ee4dab 100644 --- a/crypto/ml_dsa/ml_dsa_key.c +++ b/crypto/ml_dsa/ml_dsa_key.c @@ -332,7 +332,7 @@ int ossl_ml_dsa_key_has(const ML_DSA_KEY *key, int selection) * @returns 1 on success, or 0 on failure. */ static int public_from_private(const ML_DSA_KEY *key, EVP_MD_CTX *md_ctx, - VECTOR *t1, VECTOR *t0) + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops, VECTOR *t1, VECTOR *t0) { int ret = 0; const ML_DSA_PARAMS *params = key->params; @@ -351,7 +351,7 @@ static int public_from_private(const ML_DSA_KEY *key, EVP_MD_CTX *md_ctx, matrix_init(&a_ntt, s1_ntt.poly + l, k, l); /* Using rho generate A' = A in NTT form */ - if (!matrix_expand_A(md_ctx, key->shake128_md, key->rho, &a_ntt)) + if (!sample_ops->matrix_expand_A(md_ctx, key->shake128_md, key->rho, &a_ntt)) goto err; /* t = NTT_inv(A' * NTT(s1)) + s2 */ @@ -376,6 +376,7 @@ err: int ossl_ml_dsa_key_public_from_private(ML_DSA_KEY *key) { int ret = 0; + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops(); VECTOR t0; EVP_MD_CTX *md_ctx = NULL; @@ -383,7 +384,7 @@ int ossl_ml_dsa_key_public_from_private(ML_DSA_KEY *key) return 0; ret = ((md_ctx = EVP_MD_CTX_new()) != NULL) && ossl_ml_dsa_key_pub_alloc(key) /* allocate space for t1 */ - && public_from_private(key, md_ctx, &key->t1, &t0) + && public_from_private(key, md_ctx, sample_ops, &key->t1, &t0) && vector_equal(&t0, &key->t0) /* compare the generated t0 to the expected */ && ossl_ml_dsa_pk_encode(key) && shake_xof(md_ctx, key->shake256_md, @@ -397,6 +398,7 @@ int ossl_ml_dsa_key_public_from_private(ML_DSA_KEY *key) int ossl_ml_dsa_key_pairwise_check(const ML_DSA_KEY *key) { int ret = 0; + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops(); VECTOR t1, t0; POLY *polys = NULL; uint32_t k = (uint32_t)key->params->k; @@ -414,7 +416,7 @@ int ossl_ml_dsa_key_pairwise_check(const ML_DSA_KEY *key) vector_init(&t1, polys, k); vector_init(&t0, polys + k, k); - if (!public_from_private(key, md_ctx, &t1, &t0)) + if (!public_from_private(key, md_ctx, sample_ops, &t1, &t0)) goto err; ret = vector_equal(&t1, &key->t1) && vector_equal(&t0, &key->t0); @@ -435,6 +437,7 @@ err: static int keygen_internal(ML_DSA_KEY *out) { int ret = 0; + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops(); uint8_t augmented_seed[ML_DSA_SEED_BYTES + 2]; uint8_t expanded_seed[ML_DSA_RHO_BYTES + ML_DSA_PRIV_SEED_BYTES + ML_DSA_K_BYTES]; const uint8_t *const rho = expanded_seed; /* p = Public Random Seed */ @@ -461,8 +464,9 @@ static int keygen_internal(ML_DSA_KEY *out) memcpy(out->rho, rho, sizeof(out->rho)); memcpy(out->K, K, sizeof(out->K)); - ret = vector_expand_S(md_ctx, out->shake256_md, params->eta, priv_seed, &out->s1, &out->s2) - && public_from_private(out, md_ctx, &out->t1, &out->t0) + ret = sample_ops->vector_expand_S(md_ctx, out->shake256_md, params->eta, + priv_seed, &out->s1, &out->s2) + && public_from_private(out, md_ctx, sample_ops, &out->t1, &out->t0) && ossl_ml_dsa_pk_encode(out) && shake_xof(md_ctx, out->shake256_md, out->pub_encoding, out->params->pk_len, out->tr, sizeof(out->tr)) diff --git a/crypto/ml_dsa/ml_dsa_local.h b/crypto/ml_dsa/ml_dsa_local.h index 9d01856ce32..23e2db247c1 100644 --- a/crypto/ml_dsa/ml_dsa_local.h +++ b/crypto/ml_dsa/ml_dsa_local.h @@ -59,10 +59,21 @@ typedef struct vector_st VECTOR; typedef struct matrix_st MATRIX; typedef struct ml_dsa_sig_st ML_DSA_SIG; -int ossl_ml_dsa_matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md, +typedef int(ML_DSA_MATRIX_EXPAND_A_FN)(EVP_MD_CTX *g_ctx, const EVP_MD *md, const uint8_t *rho, MATRIX *out); -int ossl_ml_dsa_vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta, - const uint8_t *seed, VECTOR *s1, VECTOR *s2); +typedef int(ML_DSA_VECTOR_EXPAND_S_FN)(EVP_MD_CTX *h_ctx, const EVP_MD *md, + int eta, const uint8_t *seed, VECTOR *s1, VECTOR *s2); +typedef void(ML_DSA_VECTOR_EXPAND_MASK_FN)(VECTOR *out, + const uint8_t rho_prime[ML_DSA_RHO_PRIME_BYTES], uint32_t kappa, uint32_t gamma1, + EVP_MD_CTX *h_ctx, const EVP_MD *md); + +typedef struct ossl_ml_dsa_sample_ops_st { + ML_DSA_MATRIX_EXPAND_A_FN *matrix_expand_A; + ML_DSA_VECTOR_EXPAND_S_FN *vector_expand_S; + ML_DSA_VECTOR_EXPAND_MASK_FN *vector_expand_mask; +} OSSL_ML_DSA_SAMPLE_OPS; + +const OSSL_ML_DSA_SAMPLE_OPS *ossl_ml_dsa_sample_ops(void); void ossl_ml_dsa_matrix_mult_vector(const MATRIX *matrix_kl, const VECTOR *vl, VECTOR *vk); int ossl_ml_dsa_poly_expand_mask(POLY *out, const uint8_t *seed, size_t seed_len, diff --git a/crypto/ml_dsa/ml_dsa_matrix.h b/crypto/ml_dsa/ml_dsa_matrix.h index 3bc053720bb..e5f4ebf6d93 100644 --- a/crypto/ml_dsa/ml_dsa_matrix.h +++ b/crypto/ml_dsa/ml_dsa_matrix.h @@ -41,11 +41,4 @@ matrix_mult_vector(const MATRIX *a, const VECTOR *s, VECTOR *t) ossl_ml_dsa_matrix_mult_vector(a, s, t); } -static ossl_inline ossl_unused int -matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md, const uint8_t *rho, - MATRIX *out) -{ - return ossl_ml_dsa_matrix_expand_A(g_ctx, md, rho, out); -} - #endif /* !defined(OSSL_LIBCRYPTO_ML_DSA_ML_DSA_MATRIX_H) */ diff --git a/crypto/ml_dsa/ml_dsa_sample.c b/crypto/ml_dsa/ml_dsa_sample.c index 5d9dc84a54f..afa09b7971c 100644 --- a/crypto/ml_dsa/ml_dsa_sample.c +++ b/crypto/ml_dsa/ml_dsa_sample.c @@ -8,6 +8,7 @@ */ #include +#include #include "ml_dsa_local.h" #include "ml_dsa_vector.h" #include "ml_dsa_matrix.h" @@ -35,6 +36,10 @@ typedef int(COEFF_FROM_NIBBLE_FUNC)(uint32_t nibble, uint32_t *out); static COEFF_FROM_NIBBLE_FUNC coeff_from_nibble_4; static COEFF_FROM_NIBBLE_FUNC coeff_from_nibble_2; +static ML_DSA_MATRIX_EXPAND_A_FN matrix_expand_A_scalar; +static ML_DSA_VECTOR_EXPAND_S_FN vector_expand_S_scalar; +static ML_DSA_VECTOR_EXPAND_MASK_FN vector_expand_mask_scalar; + /** * @brief Combine 3 bytes to form an coefficient. * See FIPS 204, Algorithm 14, CoeffFromThreeBytes() @@ -160,13 +165,14 @@ static int rej_bounded_poly(EVP_MD_CTX *h_ctx, const EVP_MD *md, COEFF_FROM_NIBBLE_FUNC *coef_from_nibble, const uint8_t *seed, size_t seed_len, POLY *out) { + int ret = 0; int j = 0; uint32_t z0, z1; uint8_t blocks[SHAKE256_BLOCKSIZE], *b, *end = blocks + sizeof(blocks); /* Instead of just squeezing 1 byte at a time, we grab a whole block */ if (!shake_xof(h_ctx, md, seed, seed_len, blocks, sizeof(blocks))) - return 0; + goto err; while (1) { for (b = blocks; b < end; b++) { @@ -174,15 +180,22 @@ static int rej_bounded_poly(EVP_MD_CTX *h_ctx, const EVP_MD *md, z1 = *b >> 4; /* high nibble of byte */ if (coef_from_nibble(z0, &out->coeff[j]) - && ++j >= ML_DSA_NUM_POLY_COEFFICIENTS) - return 1; + && ++j >= ML_DSA_NUM_POLY_COEFFICIENTS) { + ret = 1; + goto err; + } if (coef_from_nibble(z1, &out->coeff[j]) - && ++j >= ML_DSA_NUM_POLY_COEFFICIENTS) - return 1; + && ++j >= ML_DSA_NUM_POLY_COEFFICIENTS) { + ret = 1; + goto err; + } } if (!EVP_DigestSqueeze(h_ctx, blocks, sizeof(blocks))) - return 0; + goto err; } +err: + OPENSSL_cleanse(blocks, sizeof(blocks)); + return ret; } /** @@ -198,7 +211,7 @@ static int rej_bounded_poly(EVP_MD_CTX *h_ctx, const EVP_MD *md, * in the range of 0..q-1. * @returns 1 if the matrix was generated, or 0 on error. */ -int ossl_ml_dsa_matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md, +static int matrix_expand_A_scalar(EVP_MD_CTX *g_ctx, const EVP_MD *md, const uint8_t *rho, MATRIX *out) { int ret = 0; @@ -208,7 +221,6 @@ int ossl_ml_dsa_matrix_expand_A(EVP_MD_CTX *g_ctx, const EVP_MD *md, /* The seed used for each matrix element is rho + column_index + row_index */ memcpy(derived_seed, rho, ML_DSA_RHO_BYTES); - for (i = 0; i < out->k; i++) { for (j = 0; j < out->l; j++) { derived_seed[ML_DSA_RHO_BYTES + 1] = (uint8_t)i; @@ -241,7 +253,7 @@ err: * the range (q-eta)..0..eta * @returns 1 if s1 and s2 were successfully generated, or 0 otherwise. */ -int ossl_ml_dsa_vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta, +static int vector_expand_S_scalar(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta, const uint8_t *seed, VECTOR *s1, VECTOR *s2) { int ret = 0; @@ -275,6 +287,7 @@ int ossl_ml_dsa_vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta, } ret = 1; err: + OPENSSL_cleanse(derived_seed, sizeof(derived_seed)); return ret; } @@ -285,9 +298,11 @@ int ossl_ml_dsa_poly_expand_mask(POLY *out, const uint8_t *seed, size_t seed_len { uint8_t buf[32 * 20]; size_t buf_len = 32 * (gamma1 == ML_DSA_GAMMA1_TWO_POWER_19 ? 20 : 18); - - return shake_xof(h_ctx, md, seed, seed_len, buf, buf_len) + int ret = shake_xof(h_ctx, md, seed, seed_len, buf, buf_len) && ossl_ml_dsa_poly_decode_expand_mask(out, buf, buf_len, gamma1); + + OPENSSL_cleanse(buf, sizeof(buf)); + return ret; } /* @@ -376,3 +391,46 @@ int ossl_ml_dsa_poly_sample_in_ball(POLY *out_c, const uint8_t *seed, int seed_l } return 1; } + +static void vector_expand_mask_scalar(VECTOR *out, + const uint8_t rho_prime[ML_DSA_RHO_PRIME_BYTES], uint32_t kappa, uint32_t gamma1, + EVP_MD_CTX *h_ctx, const EVP_MD *md) +{ + size_t i; + uint8_t derived_seed[ML_DSA_RHO_PRIME_BYTES + 2]; + + memcpy(derived_seed, rho_prime, ML_DSA_RHO_PRIME_BYTES); + + for (i = 0; i < out->num_poly; i++) { + size_t index = kappa + i; + + derived_seed[ML_DSA_RHO_PRIME_BYTES] = index & 0xFF; + derived_seed[ML_DSA_RHO_PRIME_BYTES + 1] = (index >> 8) & 0xFF; + poly_expand_mask(out->poly + i, derived_seed, sizeof(derived_seed), + gamma1, h_ctx, md); + } + OPENSSL_cleanse(derived_seed, sizeof(derived_seed)); +} + +static const OSSL_ML_DSA_SAMPLE_OPS ml_dsa_sample_generic_meth = { + matrix_expand_A_scalar, + vector_expand_S_scalar, + vector_expand_mask_scalar +}; + +#if defined(KECCAK1600_ASM) \ + && (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) \ + && !defined(OPENSSL_NO_ASM) +#include "ml_dsa_sample_hw_x86_64.inc" +const OSSL_ML_DSA_SAMPLE_OPS *ossl_ml_dsa_sample_ops(void) +{ + if (SHA3_avx512vl_capable()) + return &ml_dsa_sample_x86_64; + return &ml_dsa_sample_generic_meth; +} +#else +const OSSL_ML_DSA_SAMPLE_OPS *ossl_ml_dsa_sample_ops(void) +{ + return &ml_dsa_sample_generic_meth; +} +#endif diff --git a/crypto/ml_dsa/ml_dsa_sample_hw_x86_64.inc b/crypto/ml_dsa/ml_dsa_sample_hw_x86_64.inc new file mode 100644 index 00000000000..fcf5f033237 --- /dev/null +++ b/crypto/ml_dsa/ml_dsa_sample_hw_x86_64.inc @@ -0,0 +1,307 @@ +/* + * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (c) 2026 Intel Corporation. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#define ML_DSA_SHAKE_X4_BATCH_SIZE 4 +#define ML_DSA_SHAKE_X4_DONE_MASK ((1 << ML_DSA_SHAKE_X4_BATCH_SIZE) - 1) +#define ML_DSA_EXPAND_MASK_BYTES_PER_COEFF 32 +#define ML_DSA_EXPAND_MASK_COEFFS_GAMMA1_19 20 +#define ML_DSA_EXPAND_MASK_COEFFS_GAMMA1_17 18 +#define ML_DSA_EXPAND_MASK_BUF_SIZE_GAMMA1_19 \ + (ML_DSA_EXPAND_MASK_BYTES_PER_COEFF * ML_DSA_EXPAND_MASK_COEFFS_GAMMA1_19) +#define ML_DSA_EXPAND_MASK_BUF_SIZE_GAMMA1_17 \ + (ML_DSA_EXPAND_MASK_BYTES_PER_COEFF * ML_DSA_EXPAND_MASK_COEFFS_GAMMA1_17) +#define ML_DSA_EXPAND_MASK_BUF_SIZE(gamma1) \ + ((gamma1) == ML_DSA_GAMMA1_TWO_POWER_19 \ + ? ML_DSA_EXPAND_MASK_BUF_SIZE_GAMMA1_19 \ + : ML_DSA_EXPAND_MASK_BUF_SIZE_GAMMA1_17) + +static ossl_unused int rej_ntt_poly_mb(const uint8_t *seeds[ML_DSA_SHAKE_X4_BATCH_SIZE], + const size_t seed_len, POLY *outs[ML_DSA_SHAKE_X4_BATCH_SIZE], const size_t count) +{ + KECCAK1600_X4_AVX512VL_CTX ctx; + uint8_t blocks[ML_DSA_SHAKE_X4_BATCH_SIZE][SHAKE128_BLOCKSIZE]; + int coeff_idx[ML_DSA_SHAKE_X4_BATCH_SIZE] = { 0, 0, 0, 0 }; + size_t done_mask = 0; + size_t lane; + + for (lane = count; lane < ML_DSA_SHAKE_X4_BATCH_SIZE; lane++) + done_mask |= ((size_t)1 << lane); + + ossl_sha3_shake128_x4_inc_init_avx512vl(&ctx); + ossl_sha3_shake128_x4_inc_absorb_avx512vl(&ctx, seeds[0], seeds[1], + seeds[2], seeds[3], seed_len); + + while (done_mask != ML_DSA_SHAKE_X4_DONE_MASK) { + ossl_sha3_shake128_x4_inc_squeeze_avx512vl(blocks[0], blocks[1], + blocks[2], blocks[3], SHAKE128_BLOCKSIZE, &ctx); + + for (lane = 0; lane < ML_DSA_SHAKE_X4_BATCH_SIZE; lane++) { + if (done_mask & ((size_t)1 << lane)) + continue; + + const uint8_t *b = blocks[lane]; + const uint8_t *end = b + SHAKE128_BLOCKSIZE; + + for (; b < end && coeff_idx[lane] < ML_DSA_NUM_POLY_COEFFICIENTS; b += 3) { + uint32_t *coeff_ptr = &(outs[lane]->coeff[coeff_idx[lane]]); + + if (coeff_from_three_bytes(b, coeff_ptr)) + coeff_idx[lane]++; + } + + if (coeff_idx[lane] >= ML_DSA_NUM_POLY_COEFFICIENTS) + done_mask |= ((size_t)1 << lane); + } + } + + return 1; +} + +static void vector_expand_mask_mb(VECTOR *out, + const uint8_t rho_prime[ML_DSA_RHO_PRIME_BYTES], const uint32_t kappa, const uint32_t gamma1, + EVP_MD_CTX *h_ctx, const EVP_MD *md) +{ + size_t i; + const size_t num_polys = out->num_poly; + uint8_t derived_seeds[ML_DSA_SHAKE_X4_BATCH_SIZE][ML_DSA_RHO_PRIME_BYTES + 2]; + const size_t seed_len = sizeof(derived_seeds[0]); + const size_t buf_size = ML_DSA_EXPAND_MASK_BUF_SIZE(gamma1); + uint8_t buffers[ML_DSA_SHAKE_X4_BATCH_SIZE][ML_DSA_EXPAND_MASK_BUF_SIZE_GAMMA1_19]; + + (void)h_ctx; + (void)md; + + for (i = 0; i < ML_DSA_SHAKE_X4_BATCH_SIZE; i++) + memcpy(derived_seeds[i], rho_prime, ML_DSA_RHO_PRIME_BYTES); + + for (i = 0; i + (ML_DSA_SHAKE_X4_BATCH_SIZE - 1) < num_polys; i += ML_DSA_SHAKE_X4_BATCH_SIZE) { + size_t b; + + for (b = 0; b < ML_DSA_SHAKE_X4_BATCH_SIZE; b++) { + const size_t index = kappa + i + b; + + derived_seeds[b][ML_DSA_RHO_PRIME_BYTES] = index & 0xFF; + derived_seeds[b][ML_DSA_RHO_PRIME_BYTES + 1] = (index >> 8) & 0xFF; + } + + ossl_sha3_shake256_x4_avx512vl(buffers[0], buffers[1], buffers[2], buffers[3], buf_size, + derived_seeds[0], derived_seeds[1], derived_seeds[2], derived_seeds[3], seed_len); + + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 0], buffers[0], buf_size, gamma1); + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 1], buffers[1], buf_size, gamma1); + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 2], buffers[2], buf_size, gamma1); + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 3], buffers[3], buf_size, gamma1); + } + + /* + * num_polys is always 4 (ML-DSA-44), 5 (ML-DSA-65), or 7 (ML-DSA-87), so the + * above loops will always runs at least once, initializing derived_seeds. + * As a result, 'left' below will be 0, 1, or 3, meaning the 4 way shake will + * recalculate values that are not used. + */ + if (i < num_polys) { + const size_t left = num_polys - i; + size_t b; + + for (b = 0; b < left; b++) { + const size_t index = kappa + i + b; + + derived_seeds[b][ML_DSA_RHO_PRIME_BYTES] = (uint8_t)index; + derived_seeds[b][ML_DSA_RHO_PRIME_BYTES + 1] = (uint8_t)(index >> 8); + } + + ossl_sha3_shake256_x4_avx512vl(buffers[0], buffers[1], buffers[2], buffers[3], buf_size, + derived_seeds[0], derived_seeds[1], derived_seeds[2], derived_seeds[3], seed_len); + + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 0], buffers[0], buf_size, gamma1); + + if ((i + 1) < num_polys) + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 1], buffers[1], buf_size, gamma1); + + if ((i + 2) < num_polys) + ossl_ml_dsa_poly_decode_expand_mask(&out->poly[i + 2], buffers[2], buf_size, gamma1); + } + + OPENSSL_cleanse(buffers, sizeof(buffers)); + OPENSSL_cleanse(derived_seeds, sizeof(derived_seeds)); +} + +static ossl_unused int rej_bounded_poly_mb(COEFF_FROM_NIBBLE_FUNC *coef_from_nibble, + const uint8_t *seeds[ML_DSA_SHAKE_X4_BATCH_SIZE], const size_t seed_len, + POLY *outs[ML_DSA_SHAKE_X4_BATCH_SIZE], const size_t count) +{ + KECCAK1600_X4_AVX512VL_CTX ctx; + uint8_t blocks[ML_DSA_SHAKE_X4_BATCH_SIZE][SHAKE256_BLOCKSIZE]; + int coeff_idx[ML_DSA_SHAKE_X4_BATCH_SIZE] = { 0, 0, 0, 0 }; + size_t done_mask = 0; + size_t lane; + + for (lane = count; lane < ML_DSA_SHAKE_X4_BATCH_SIZE; lane++) + done_mask |= ((size_t)1 << lane); + + ossl_sha3_shake256_x4_inc_init_avx512vl(&ctx); + ossl_sha3_shake256_x4_inc_absorb_avx512vl(&ctx, seeds[0], seeds[1], + seeds[2], seeds[3], seed_len); + + while (done_mask != ML_DSA_SHAKE_X4_DONE_MASK) { + ossl_sha3_shake256_x4_inc_squeeze_avx512vl(blocks[0], blocks[1], + blocks[2], blocks[3], SHAKE256_BLOCKSIZE, &ctx); + + for (lane = 0; lane < ML_DSA_SHAKE_X4_BATCH_SIZE; lane++) { + if (done_mask & ((size_t)1 << lane)) + continue; + + const uint8_t *b = blocks[lane]; + const uint8_t *end = b + SHAKE256_BLOCKSIZE; + + for (; b < end && coeff_idx[lane] < ML_DSA_NUM_POLY_COEFFICIENTS; b++) { + uint32_t z0 = *b & 0x0F; + uint32_t z1 = *b >> 4; + + if (coef_from_nibble(z0, &outs[lane]->coeff[coeff_idx[lane]])) + coeff_idx[lane]++; + + if (coeff_idx[lane] >= ML_DSA_NUM_POLY_COEFFICIENTS) { + done_mask |= ((size_t)1 << lane); + break; + } + + if (coef_from_nibble(z1, &outs[lane]->coeff[coeff_idx[lane]])) + coeff_idx[lane]++; + + if (coeff_idx[lane] >= ML_DSA_NUM_POLY_COEFFICIENTS) { + done_mask |= ((size_t)1 << lane); + break; + } + } + } + } + + OPENSSL_cleanse(blocks, sizeof(blocks)); + ossl_sha3_shake256_x4_inc_cleanup_avx512vl(&ctx); + return 1; +} + +static int matrix_expand_A_mb(EVP_MD_CTX *g_ctx, const EVP_MD *md, + const uint8_t *rho, MATRIX *out) +{ + size_t b, idx; + uint8_t derived_seeds[ML_DSA_SHAKE_X4_BATCH_SIZE][ML_DSA_RHO_BYTES + 2]; + const size_t seed_len = sizeof(derived_seeds[0]); + const uint8_t *seeds[ML_DSA_SHAKE_X4_BATCH_SIZE]; + POLY *polys[ML_DSA_SHAKE_X4_BATCH_SIZE]; + POLY *poly = out->m_poly; + + for (b = 0; b < ML_DSA_SHAKE_X4_BATCH_SIZE; b++) { + memcpy(derived_seeds[b], rho, ML_DSA_RHO_BYTES); + seeds[b] = derived_seeds[b]; + } + + for (idx = 0; (idx + ML_DSA_SHAKE_X4_BATCH_SIZE - 1) < (out->k * out->l); + idx += ML_DSA_SHAKE_X4_BATCH_SIZE) { + for (b = 0; b < ML_DSA_SHAKE_X4_BATCH_SIZE; b++) { + const size_t row = (idx + b) / out->l; + const size_t col = (idx + b) % out->l; + + derived_seeds[b][ML_DSA_RHO_BYTES] = (uint8_t)col; + derived_seeds[b][ML_DSA_RHO_BYTES + 1] = (uint8_t)row; + polys[b] = &poly[idx + b]; + } + + if (!rej_ntt_poly_mb(seeds, seed_len, polys, 4)) + return 0; + } + + if (idx < (out->k * out->l)) { + const size_t left = (out->k * out->l) - idx; + + for (b = 0; b < left; b++) { + const size_t row = (idx + b) / out->l; + const size_t col = (idx + b) % out->l; + + derived_seeds[b][ML_DSA_RHO_BYTES] = (uint8_t)col; + derived_seeds[b][ML_DSA_RHO_BYTES + 1] = (uint8_t)row; + polys[b] = &poly[idx + b]; + } + + if (!rej_ntt_poly_mb(seeds, seed_len, polys, left)) + return 0; + } + + return 1; +} + +static int vector_expand_S_mb(EVP_MD_CTX *h_ctx, const EVP_MD *md, const int eta, + const uint8_t *seed, VECTOR *s1, VECTOR *s2) +{ + int ret = 0; + size_t b, idx; + const size_t l = s1->num_poly; + const size_t total = l + s2->num_poly; + uint8_t derived_seeds[ML_DSA_SHAKE_X4_BATCH_SIZE][ML_DSA_PRIV_SEED_BYTES + 2]; + const uint8_t *seeds[ML_DSA_SHAKE_X4_BATCH_SIZE]; + const size_t seed_len = sizeof(derived_seeds[0]); + POLY *polys[ML_DSA_SHAKE_X4_BATCH_SIZE]; + COEFF_FROM_NIBBLE_FUNC *coef_from_nibble_fn = (eta == ML_DSA_ETA_4) ? coeff_from_nibble_4 : coeff_from_nibble_2; + + for (b = 0; b < ML_DSA_SHAKE_X4_BATCH_SIZE; b++) { + memcpy(derived_seeds[b], seed, ML_DSA_PRIV_SEED_BYTES); + seeds[b] = derived_seeds[b]; + } + + for (idx = 0; (idx + ML_DSA_SHAKE_X4_BATCH_SIZE - 1) < total; idx += ML_DSA_SHAKE_X4_BATCH_SIZE) { + for (b = 0; b < ML_DSA_SHAKE_X4_BATCH_SIZE; b++) { + const size_t poly_idx = idx + b; + + derived_seeds[b][ML_DSA_PRIV_SEED_BYTES] = (uint8_t)(poly_idx); + derived_seeds[b][ML_DSA_PRIV_SEED_BYTES + 1] = (uint8_t)(poly_idx >> 8); + + if (poly_idx < l) + polys[b] = &s1->poly[poly_idx]; + else + polys[b] = &s2->poly[poly_idx - l]; + } + + if (!rej_bounded_poly_mb(coef_from_nibble_fn, + seeds, seed_len, polys, ML_DSA_SHAKE_X4_BATCH_SIZE)) + goto err; + } + + if (idx < total) { + const size_t batch_count = total - idx; + + for (b = 0; b < batch_count; b++) { + const size_t poly_idx = idx + b; + + derived_seeds[b][ML_DSA_PRIV_SEED_BYTES] = (uint8_t)(poly_idx); + derived_seeds[b][ML_DSA_PRIV_SEED_BYTES + 1] = (uint8_t)(poly_idx >> 8); + + if (poly_idx < l) + polys[b] = &s1->poly[poly_idx]; + else + polys[b] = &s2->poly[poly_idx - l]; + } + + if (!rej_bounded_poly_mb(coef_from_nibble_fn, + seeds, seed_len, polys, batch_count)) + goto err; + } + + ret = 1; +err: + OPENSSL_cleanse(derived_seeds, sizeof(derived_seeds)); + return ret; +} + +static const OSSL_ML_DSA_SAMPLE_OPS ml_dsa_sample_x86_64 = { + matrix_expand_A_mb, + vector_expand_S_mb, + vector_expand_mask_mb +}; diff --git a/crypto/ml_dsa/ml_dsa_sign.c b/crypto/ml_dsa/ml_dsa_sign.c index 05251a6dd0e..62dfd08d53c 100644 --- a/crypto/ml_dsa/ml_dsa_sign.c +++ b/crypto/ml_dsa/ml_dsa_sign.c @@ -164,6 +164,7 @@ static int ml_dsa_sign_internal(const ML_DSA_KEY *priv, uint8_t *out_sig) { int ret = 0; + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops(); const ML_DSA_PARAMS *params = priv->params; EVP_MD_CTX *md_ctx = NULL; uint32_t k = (uint32_t)params->k, l = (uint32_t)params->l; @@ -236,7 +237,7 @@ static int ml_dsa_sign_internal(const ML_DSA_KEY *priv, CONSTTIME_SECRET_VECTOR(priv->s2); CONSTTIME_SECRET_VECTOR(priv->t0); - if (!matrix_expand_A(md_ctx, priv->shake128_md, priv->rho, &a_ntt)) + if (!sample_ops->matrix_expand_A(md_ctx, priv->shake128_md, priv->rho, &a_ntt)) goto err; /* @@ -267,8 +268,8 @@ static int ml_dsa_sign_internal(const ML_DSA_KEY *priv, VECTOR *ct0 = &w1; uint32_t z_max, r0_max, ct0_max, h_ones; - vector_expand_mask(&y, rho_prime, sizeof(rho_prime), (uint32_t)kappa, - gamma1, md_ctx, priv->shake256_md); + sample_ops->vector_expand_mask(&y, rho_prime, + (uint32_t)kappa, gamma1, md_ctx, priv->shake256_md); vector_copy(y_ntt, &y); vector_ntt(y_ntt); @@ -391,6 +392,7 @@ static int ml_dsa_verify_internal(const ML_DSA_KEY *pub, const uint8_t *sig_enc, size_t sig_enc_len) { int ret = 0; + const OSSL_ML_DSA_SAMPLE_OPS *sample_ops = ossl_ml_dsa_sample_ops(); uint8_t *alloc = NULL, *w1_encoded = NULL; void *alloc_freeptr = NULL; POLY *p, *c_ntt; @@ -448,7 +450,7 @@ static int ml_dsa_verify_internal(const ML_DSA_KEY *pub, vector_init(&ct1_ntt, p + k, k); if (!ossl_ml_dsa_sig_decode(&sig, sig_enc, sig_enc_len, pub->params) - || !matrix_expand_A(md_ctx, pub->shake128_md, pub->rho, &a_ntt)) + || !sample_ops->matrix_expand_A(md_ctx, pub->shake128_md, pub->rho, &a_ntt)) goto err; /* Compute verifiers challenge c_ntt = NTT(SampleInBall(c_tilde)) */ diff --git a/crypto/ml_dsa/ml_dsa_vector.h b/crypto/ml_dsa/ml_dsa_vector.h index 6b408acdc20..9b83c0420e9 100644 --- a/crypto/ml_dsa/ml_dsa_vector.h +++ b/crypto/ml_dsa/ml_dsa_vector.h @@ -152,33 +152,6 @@ vector_mult_scalar(const VECTOR *lhs, const POLY *rhs, VECTOR *out) ossl_ml_dsa_poly_ntt_mult(lhs->poly + i, rhs, out->poly + i); } -static ossl_inline ossl_unused int -vector_expand_S(EVP_MD_CTX *h_ctx, const EVP_MD *md, int eta, - const uint8_t *seed, VECTOR *s1, VECTOR *s2) -{ - return ossl_ml_dsa_vector_expand_S(h_ctx, md, eta, seed, s1, s2); -} - -static ossl_inline ossl_unused void -vector_expand_mask(VECTOR *out, const uint8_t *rho_prime, size_t rho_prime_len, - uint32_t kappa, uint32_t gamma1, - EVP_MD_CTX *h_ctx, const EVP_MD *md) -{ - size_t i; - uint8_t derived_seed[ML_DSA_RHO_PRIME_BYTES + 2]; - - memcpy(derived_seed, rho_prime, ML_DSA_RHO_PRIME_BYTES); - - for (i = 0; i < out->num_poly; i++) { - size_t index = kappa + i; - - derived_seed[ML_DSA_RHO_PRIME_BYTES] = index & 0xFF; - derived_seed[ML_DSA_RHO_PRIME_BYTES + 1] = (index >> 8) & 0xFF; - poly_expand_mask(out->poly + i, derived_seed, sizeof(derived_seed), - gamma1, h_ctx, md); - } -} - /* Scale back previously rounded value */ static ossl_inline ossl_unused void vector_scale_power2_round_ntt(const VECTOR *in, VECTOR *out) diff --git a/crypto/sha/asm/keccak1600x4-avx512vl.pl b/crypto/sha/asm/keccak1600x4-avx512vl.pl new file mode 100755 index 00000000000..b5a6ac73776 --- /dev/null +++ b/crypto/sha/asm/keccak1600x4-avx512vl.pl @@ -0,0 +1,2344 @@ +#!/usr/bin/env perl +# +# Copyright 2026 The OpenSSL Project Authors. All Rights Reserved. +# Copyright (c) 2026 Intel Corporation. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html + +############################################################################### +# Keccak x4 AVX512VL SHA3/SHAKE Assembly Routines +# +# Description: +# This file emits x86_64 assembly for AVX512VL accelerated Keccak-f[1600] +# processing of 4 independent states in parallel ("x4"). +# +# It provides the core 24-round Keccak permutation and x4 helper routines +# used by SHA3 and SHAKE absorb/finalize/squeeze paths. Data from four +# input/output lanes is packed across YMM registers so lane-local operations +# execute in SIMD. +# +############################################################################### + +# $output is the last argument if it looks like a file (it has an extension) +# $flavour is the first argument if it doesn't look like a file +$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef; +$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef; + +$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); + +$avx512vl = 0; + +$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; +( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or +( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or +die "can't locate x86_64-xlate.pl"; + +# Check for AVX512VL support in assembler +if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1` =~ /GNU assembler version (\d+)\.(\d+)/) { + my ($gas_major, $gas_minor) = ($1, $2); + $avx512vl = ($gas_major > 2 || ($gas_major == 2 && $gas_minor >= 26)); +} + +if (!$avx512vl + && $win64 + && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) + && `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)(?:\.([0-9]+))?/) +{ + $avx512vl = ($1 >= 2.12); +} + +if (!$avx512vl && `$ENV{CC} -v 2>&1` =~ /((?:clang|LLVM) version|.*based on LLVM) ([0-9]+\.[0-9]+)/) { + $avx512vl = ($2>=3.9); +} + +open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"" + or die "can't call $xlate: $!"; +*STDOUT=*OUT; + +$arg1="%rdi"; +$arg2="%rsi"; +$arg3="%rdx"; +$arg4="%rcx"; +$arg5="%r8"; +$arg6="%r9"; +$roundn="%r13d"; +$tblptr="%r14"; + +# Define SHAKE rates +$SHAKE128_RATE="\$168"; +$SHAKE256_RATE="\$136"; + +# Stack frame offsets for SHAKE x4 wrapper functions +$STATE_SIZE="808"; # (25 * 8 * 4) + 8 = 808 bytes +$sf_arg1="0"; +$sf_arg2="8"; +$sf_arg3="16"; +$sf_arg4="24"; +$sf_arg5="32"; +$sf_state_ptr="40"; +$sf_state_x4="48"; +$sf_size="856"; # 48 + 808 = 856 bytes + +# Emit an internal helper call used by one-shot wrappers. +# - Win64: call the provided *_internal shim and bracket it with 32-byte +# shadow space so shim entry can use xlate-compatible [rsp+8]/[rsp+16]. +# - non-Win64: call the public API symbol (same base name without _internal). +# The argument must be the shim/internal symbol name, e.g. +# SHA3_shake128_x4_inc_squeeze_avx512vl_internal +sub call_internal { + my ($shim_name) = @_; + my $external_name = $shim_name; + + $external_name =~ s/_internal$//; + + return <<___ if ($win64); + sub \$32, %rsp + call $shim_name + add \$32, %rsp +___ + + return <<___; + call $external_name +___ +} + +if ($avx512vl>0) {{{ + +my $avx512_mask = (1<<31)|(1<<30)|(1<<17)|(1<<16); # AVX512VL|BW|DQ|F + +$code .= <<___; +.text + +.extern OPENSSL_ia32cap_P + +.globl SHA3_avx512vl_capable +.type SHA3_avx512vl_capable,\@abi-omnipotent +.align 32 +SHA3_avx512vl_capable: + mov OPENSSL_ia32cap_P+8(%rip), %ecx + xor %eax, %eax + # 1<<31|1<<30|1<<17|1<<16: AVX512VL|AVX512BW|AVX512DQ|AVX512F + and \$$avx512_mask, %ecx + cmp \$$avx512_mask, %ecx + cmove %ecx, %eax + ret +.size SHA3_avx512vl_capable, .-SHA3_avx512vl_capable +___ + +$code.=<<___; +.text + +# Perform Keccak permutation +# +# YMM registers 0 to 24 are used as Keccak state registers. +# This function, as is, can work on 1 to 4 independent states at the same time. +# +# There is no clear boundary between Theta, Rho, Pi, Chi and Iota steps. +# Instructions corresponding to these steps overlap for better efficiency. +# +# Arguments: +# ymm0-ymm24 [in/out] Keccak state registers (one SIMD per one state register) +# ymm25-ymm31 [clobbered] temporary SIMD registers +# $roundn [clobbered] used for round tracking +# $tblptr [clobbered] used for access to SHA3 constant table +.type keccak_1600_permute,\@abi-omnipotent +.align 32 +keccak_1600_permute: +.cfi_startproc + mov \$24, $roundn # 24 rounds + lea iotas(%rip), $tblptr # Load the address of the SHA3 round constants + +.align 32 +.Lkeccak_rnd_loop: + # Theta step + + # Compute column parities + # C[5] = [0, 0, 0, 0, 0] + # for x in 0 to 4: + # C[x] = state[x][0] XOR state[x][1] XOR state[x][2] XOR state[x][3] XOR state[x][4] + + vmovdqa64 %ymm0, %ymm25 + vpternlogq \$0x96, %ymm5, %ymm10, %ymm25 + vmovdqa64 %ymm1, %ymm26 + vpternlogq \$0x96, %ymm11, %ymm6, %ymm26 + vmovdqa64 %ymm2, %ymm27 + vpternlogq \$0x96, %ymm12, %ymm7, %ymm27 + + vmovdqa64 %ymm3, %ymm28 + vpternlogq \$0x96, %ymm13, %ymm8, %ymm28 + vmovdqa64 %ymm4, %ymm29 + vpternlogq \$0x96, %ymm14, %ymm9, %ymm29 + vpternlogq \$0x96, %ymm20, %ymm15, %ymm25 + + vpternlogq \$0x96, %ymm21, %ymm16, %ymm26 + vpternlogq \$0x96, %ymm22, %ymm17, %ymm27 + vpternlogq \$0x96, %ymm23, %ymm18, %ymm28 + + # Start computing D values and keep computing column parity + # D[5] = [0, 0, 0, 0, 0] + # for x in 0 to 4: + # D[x] = C[(x+4) mod 5] XOR ROTATE_LEFT(C[(x+1) mod 5], 1) + + vprolq \$1, %ymm26, %ymm30 + vprolq \$1, %ymm27, %ymm31 + vpternlogq \$0x96, %ymm24, %ymm19, %ymm29 + + # Continue computing D values and apply Theta + # for x in 0 to 4: + # for y in 0 to 4: + # state[x][y] = state[x][y] XOR D[x] + + vpternlogq \$0x96, %ymm30, %ymm29, %ymm0 + vpternlogq \$0x96, %ymm30, %ymm29, %ymm10 + vpternlogq \$0x96, %ymm30, %ymm29, %ymm20 + + vpternlogq \$0x96, %ymm30, %ymm29, %ymm5 + vpternlogq \$0x96, %ymm30, %ymm29, %ymm15 + vprolq \$1, %ymm28, %ymm30 + + vpternlogq \$0x96, %ymm31, %ymm25, %ymm6 + vpternlogq \$0x96, %ymm31, %ymm25, %ymm16 + vpternlogq \$0x96, %ymm31, %ymm25, %ymm1 + + vpternlogq \$0x96, %ymm31, %ymm25, %ymm11 + vpternlogq \$0x96, %ymm31, %ymm25, %ymm21 + vprolq \$1, %ymm29, %ymm31 + + vpbroadcastq ($tblptr), %ymm29 # Load the round constant into ymm29 (Iota) + add \$8, $tblptr # Increment the pointer to the next round constant + + vpternlogq \$0x96, %ymm30, %ymm26, %ymm12 + vpternlogq \$0x96, %ymm30, %ymm26, %ymm7 + vpternlogq \$0x96, %ymm30, %ymm26, %ymm22 + + vpternlogq \$0x96, %ymm30, %ymm26, %ymm17 + vpternlogq \$0x96, %ymm30, %ymm26, %ymm2 + vprolq \$1, %ymm25, %ymm30 + + # Rho step + # Keep applying Theta and start Rho step + # + # ROTATION_OFFSETS[5][5] = [ + # [0, 1, 62, 28, 27], + # [36, 44, 6, 55, 20], + # [3, 10, 43, 25, 39], + # [41, 45, 15, 21, 8], + # [18, 2, 61, 56, 14] ] + # + # for x in 0 to 4: + # for y in 0 to 4: + # state[x][y] = ROTATE_LEFT(state[x][y], ROTATION_OFFSETS[x][y]) + + vpternlogq \$0x96, %ymm31, %ymm27, %ymm3 + vpternlogq \$0x96, %ymm31, %ymm27, %ymm13 + vpternlogq \$0x96, %ymm31, %ymm27, %ymm23 + + vprolq \$44, %ymm6, %ymm6 + vpternlogq \$0x96, %ymm31, %ymm27, %ymm18 + vpternlogq \$0x96, %ymm31, %ymm27, %ymm8 + + vprolq \$43, %ymm12, %ymm12 + vprolq \$21, %ymm18, %ymm18 + vpternlogq \$0x96, %ymm30, %ymm28, %ymm24 + + vprolq \$14, %ymm24, %ymm24 + vprolq \$28, %ymm3, %ymm3 + vpternlogq \$0x96, %ymm30, %ymm28, %ymm9 + + vprolq \$20, %ymm9, %ymm9 + vprolq \$3, %ymm10, %ymm10 + vpternlogq \$0x96, %ymm30, %ymm28, %ymm19 + + vprolq \$45, %ymm16, %ymm16 + vprolq \$61, %ymm22, %ymm22 + vpternlogq \$0x96, %ymm30, %ymm28, %ymm4 + + vprolq \$1, %ymm1, %ymm1 + vprolq \$6, %ymm7, %ymm7 + vpternlogq \$0x96, %ymm30, %ymm28, %ymm14 + + # Continue with Rho and start Pi and Chi steps at the same time + # Ternary logic 0xD2 is used for Chi step + # + # for x in 0 to 4: + # for y in 0 to 4: + # state[x][y] = state[x][y] XOR ((NOT state[(x+1) mod 5][y]) AND state[(x+2) mod 5][y]) + + vprolq \$25, %ymm13, %ymm13 + vprolq \$8, %ymm19, %ymm19 + vmovdqa64 %ymm0, %ymm30 + vpternlogq \$0xD2, %ymm12, %ymm6, %ymm30 + + vprolq \$18, %ymm20, %ymm20 + vprolq \$27, %ymm4, %ymm4 + vpxorq %ymm29, %ymm30, %ymm30 # Iota step + + vprolq \$36, %ymm5, %ymm5 + vprolq \$10, %ymm11, %ymm11 + vmovdqa64 %ymm6, %ymm31 + vpternlogq \$0xD2, %ymm18, %ymm12, %ymm31 + + vprolq \$15, %ymm17, %ymm17 + vprolq \$56, %ymm23, %ymm23 + vpternlogq \$0xD2, %ymm24, %ymm18, %ymm12 + + vprolq \$62, %ymm2, %ymm2 + vprolq \$55, %ymm8, %ymm8 + vpternlogq \$0xD2, %ymm0, %ymm24, %ymm18 + + vprolq \$39, %ymm14, %ymm14 + vprolq \$41, %ymm15, %ymm15 + vpternlogq \$0xD2, %ymm6, %ymm0, %ymm24 + vmovdqa64 %ymm30, %ymm0 + vmovdqa64 %ymm31, %ymm6 + + vprolq \$2, %ymm21, %ymm21 + vmovdqa64 %ymm3, %ymm30 + vpternlogq \$0xD2, %ymm10, %ymm9, %ymm30 + vmovdqa64 %ymm9, %ymm31 + vpternlogq \$0xD2, %ymm16, %ymm10, %ymm31 + + vpternlogq \$0xD2, %ymm22, %ymm16, %ymm10 + vpternlogq \$0xD2, %ymm3, %ymm22, %ymm16 + vpternlogq \$0xD2, %ymm9, %ymm3, %ymm22 + vmovdqa64 %ymm30, %ymm3 + vmovdqa64 %ymm31, %ymm9 + + vmovdqa64 %ymm1, %ymm30 + vpternlogq \$0xD2, %ymm13, %ymm7, %ymm30 + vmovdqa64 %ymm7, %ymm31 + vpternlogq \$0xD2, %ymm19, %ymm13, %ymm31 + vpternlogq \$0xD2, %ymm20, %ymm19, %ymm13 + + vpternlogq \$0xD2, %ymm1, %ymm20, %ymm19 + vpternlogq \$0xD2, %ymm7, %ymm1, %ymm20 + vmovdqa64 %ymm30, %ymm1 + vmovdqa64 %ymm31, %ymm7 + vmovdqa64 %ymm4, %ymm30 + vpternlogq \$0xD2, %ymm11, %ymm5, %ymm30 + + vmovdqa64 %ymm5, %ymm31 + vpternlogq \$0xD2, %ymm17, %ymm11, %ymm31 + vpternlogq \$0xD2, %ymm23, %ymm17, %ymm11 + vpternlogq \$0xD2, %ymm4, %ymm23, %ymm17 + + vpternlogq \$0xD2, %ymm5, %ymm4, %ymm23 + vmovdqa64 %ymm30, %ymm4 + vmovdqa64 %ymm31, %ymm5 + vmovdqa64 %ymm2, %ymm30 + vpternlogq \$0xD2, %ymm14, %ymm8, %ymm30 + vmovdqa64 %ymm8, %ymm31 + vpternlogq \$0xD2, %ymm15, %ymm14, %ymm31 + + vpternlogq \$0xD2, %ymm21, %ymm15, %ymm14 + vpternlogq \$0xD2, %ymm2, %ymm21, %ymm15 + vpternlogq \$0xD2, %ymm8, %ymm2, %ymm21 + vmovdqa64 %ymm30, %ymm2 + vmovdqa64 %ymm31, %ymm8 + + # Complete the steps and get updated state registers in ymm0 to ymm24 + vmovdqa64 %ymm3, %ymm30 + vmovdqa64 %ymm18, %ymm3 + vmovdqa64 %ymm17, %ymm18 + vmovdqa64 %ymm11, %ymm17 + vmovdqa64 %ymm7, %ymm11 + vmovdqa64 %ymm10, %ymm7 + vmovdqa64 %ymm1, %ymm10 + vmovdqa64 %ymm6, %ymm1 + vmovdqa64 %ymm9, %ymm6 + vmovdqa64 %ymm22, %ymm9 + vmovdqa64 %ymm14, %ymm22 + vmovdqa64 %ymm20, %ymm14 + vmovdqa64 %ymm2, %ymm20 + vmovdqa64 %ymm12, %ymm2 + vmovdqa64 %ymm13, %ymm12 + vmovdqa64 %ymm19, %ymm13 + vmovdqa64 %ymm23, %ymm19 + vmovdqa64 %ymm15, %ymm23 + vmovdqa64 %ymm4, %ymm15 + vmovdqa64 %ymm24, %ymm4 + vmovdqa64 %ymm21, %ymm24 + vmovdqa64 %ymm8, %ymm21 + vmovdqa64 %ymm16, %ymm8 + vmovdqa64 %ymm5, %ymm16 + vmovdqa64 %ymm30, %ymm5 + + dec $roundn # Decrement the round counter + jnz .Lkeccak_rnd_loop # Jump to the start of the loop if r13d is not zero + ret +.cfi_endproc +.size keccak_1600_permute,.-keccak_1600_permute + +# Initialize YMM registers 0-24 to zero +.globl keccak_1600_init_state +.type keccak_1600_init_state,\@abi-omnipotent +.align 32 +keccak_1600_init_state: +.cfi_startproc + vpxorq %ymm0, %ymm0, %ymm0 + vmovdqa64 %ymm0, %ymm1 + vmovdqa64 %ymm0, %ymm2 + vmovdqa64 %ymm0, %ymm3 + vmovdqa64 %ymm0, %ymm4 + vmovdqa64 %ymm0, %ymm5 + vmovdqa64 %ymm0, %ymm6 + vmovdqa64 %ymm0, %ymm7 + vmovdqa64 %ymm0, %ymm8 + vmovdqa64 %ymm0, %ymm9 + vmovdqa64 %ymm0, %ymm10 + vmovdqa64 %ymm0, %ymm11 + vmovdqa64 %ymm0, %ymm12 + vmovdqa64 %ymm0, %ymm13 + vmovdqa64 %ymm0, %ymm14 + vmovdqa64 %ymm0, %ymm15 + vmovdqa64 %ymm0, %ymm16 + vmovdqa64 %ymm0, %ymm17 + vmovdqa64 %ymm0, %ymm18 + vmovdqa64 %ymm0, %ymm19 + vmovdqa64 %ymm0, %ymm20 + vmovdqa64 %ymm0, %ymm21 + vmovdqa64 %ymm0, %ymm22 + vmovdqa64 %ymm0, %ymm23 + vmovdqa64 %ymm0, %ymm24 + ret +.cfi_endproc +.size keccak_1600_init_state,.-keccak_1600_init_state + +.globl keccak_1600_load_state_x4 +.type keccak_1600_load_state_x4,\@abi-omnipotent +.align 32 +keccak_1600_load_state_x4: +.cfi_startproc + vmovdqu64 32*0($arg1), %ymm0 + vmovdqu64 32*1($arg1), %ymm1 + vmovdqu64 32*2($arg1), %ymm2 + vmovdqu64 32*3($arg1), %ymm3 + vmovdqu64 32*4($arg1), %ymm4 + vmovdqu64 32*5($arg1), %ymm5 + vmovdqu64 32*6($arg1), %ymm6 + vmovdqu64 32*7($arg1), %ymm7 + vmovdqu64 32*8($arg1), %ymm8 + vmovdqu64 32*9($arg1), %ymm9 + vmovdqu64 32*10($arg1), %ymm10 + vmovdqu64 32*11($arg1), %ymm11 + vmovdqu64 32*12($arg1), %ymm12 + vmovdqu64 32*13($arg1), %ymm13 + vmovdqu64 32*14($arg1), %ymm14 + vmovdqu64 32*15($arg1), %ymm15 + vmovdqu64 32*16($arg1), %ymm16 + vmovdqu64 32*17($arg1), %ymm17 + vmovdqu64 32*18($arg1), %ymm18 + vmovdqu64 32*19($arg1), %ymm19 + vmovdqu64 32*20($arg1), %ymm20 + vmovdqu64 32*21($arg1), %ymm21 + vmovdqu64 32*22($arg1), %ymm22 + vmovdqu64 32*23($arg1), %ymm23 + vmovdqu64 32*24($arg1), %ymm24 + ret +.cfi_endproc +.size keccak_1600_load_state_x4,.-keccak_1600_load_state_x4 + + +.globl keccak_1600_save_state_x4 +.type keccak_1600_save_state_x4,\@abi-omnipotent +.align 32 +keccak_1600_save_state_x4: +.cfi_startproc + vmovdqu64 %ymm0, 32*0($arg1) + vmovdqu64 %ymm1, 32*1($arg1) + vmovdqu64 %ymm2, 32*2($arg1) + vmovdqu64 %ymm3, 32*3($arg1) + vmovdqu64 %ymm4, 32*4($arg1) + vmovdqu64 %ymm5, 32*5($arg1) + vmovdqu64 %ymm6, 32*6($arg1) + vmovdqu64 %ymm7, 32*7($arg1) + vmovdqu64 %ymm8, 32*8($arg1) + vmovdqu64 %ymm9, 32*9($arg1) + vmovdqu64 %ymm10, 32*10($arg1) + vmovdqu64 %ymm11, 32*11($arg1) + vmovdqu64 %ymm12, 32*12($arg1) + vmovdqu64 %ymm13, 32*13($arg1) + vmovdqu64 %ymm14, 32*14($arg1) + vmovdqu64 %ymm15, 32*15($arg1) + vmovdqu64 %ymm16, 32*16($arg1) + vmovdqu64 %ymm17, 32*17($arg1) + vmovdqu64 %ymm18, 32*18($arg1) + vmovdqu64 %ymm19, 32*19($arg1) + vmovdqu64 %ymm20, 32*20($arg1) + vmovdqu64 %ymm21, 32*21($arg1) + vmovdqu64 %ymm22, 32*22($arg1) + vmovdqu64 %ymm23, 32*23($arg1) + vmovdqu64 %ymm24, 32*24($arg1) + ret +.cfi_endproc +.size keccak_1600_save_state_x4,.-keccak_1600_save_state_x4 + + +# Add input data to state when message length is less than rate +# Arguments: +# r10: state pointer to absorb into (clobbered) +# arg2 (rsi): message pointer lane 0 (updated on output) +# arg3 (rdx): message pointer lane 1 (updated on output) +# arg4 (rcx): message pointer lane 2 (updated on output) +# arg5 (r8): message pointer lane 3 (updated on output) +# r12: length in bytes (clobbered on output) +# Clobbers: r9, rbx, r15, k1, ymm31-ymm29 +.globl keccak_1600_partial_add_x4 +.type keccak_1600_partial_add_x4,\@abi-omnipotent +.align 32 +keccak_1600_partial_add_x4: +.cfi_startproc + mov 8*100(%r10), %r9 + test \$7, %r9d + jz .Lstart_aligned_to_4x8 + + # Start offset is not aligned to register size + mov %r9, %r15 # %r15 = s[100] + + and \$7, %r9d + neg %r9d + add \$8, %r9d # register capacity = 8 - (offset % 8) + cmp %r9d, %r12d + cmovnae %r12d, %r9d # %r9d = min(register capacity, length) + + lea byte_kmask_0_to_7(%rip), %rbx + kmovb (%rbx,%r9), %k1 # message load mask + + mov %r15, %rbx + and \$~7, %ebx + lea (%r10,%rbx,4), %r10 # get to state starting register + + mov %r15, %rbx + and \$7, %ebx + + vmovdqu8 (%r10), %ymm31 # load & store / allocate SB for the register + vmovdqu8 %ymm31, (%r10) + + vmovdqu8 ($arg2), %xmm31{%k1}{z} # Read 1 to 7 bytes from lane 0 + vmovdqu8 8*0(%r10,%rbx), %xmm30{%k1}{z} # Read 1 to 7 bytes from state reg lane 0 + vpxorq %xmm30, %xmm31, %xmm31 + vmovdqu8 %xmm31, 8*0(%r10,%rbx){%k1} # Write 1 to 7 bytes to state reg lane 0 + + vmovdqu8 ($arg3), %xmm31{%k1}{z} # Read 1 to 7 bytes from lane 1 + vmovdqu8 8*1(%r10,%rbx), %xmm30{%k1}{z} # Read 1 to 7 bytes from state reg lane 1 + vpxorq %xmm30, %xmm31, %xmm31 + vmovdqu8 %xmm31, 8*1(%r10,%rbx){%k1} # Write 1 to 7 bytes to state reg lane 1 + + vmovdqu8 ($arg4), %xmm31{%k1}{z} # Read 1 to 7 bytes from lane 2 + vmovdqu8 8*2(%r10,%rbx), %xmm30{%k1}{z} # Read 1 to 7 bytes from state reg lane 2 + vpxorq %xmm30, %xmm31, %xmm31 + vmovdqu8 %xmm31, 8*2(%r10,%rbx){%k1} # Write 1 to 7 bytes to state reg lane 2 + + vmovdqu8 ($arg5), %xmm31{%k1}{z} # Read 1 to 7 bytes from lane 3 + vmovdqu8 8*3(%r10,%rbx), %xmm30{%k1}{z} # Read 1 to 7 bytes from state reg lane 3 + vpxorq %xmm30, %xmm31, %xmm31 + vmovdqu8 %xmm31, 8*3(%r10,%rbx){%k1} # Write 1 to 7 bytes to state reg lane 3 + + sub %r9, %r12 + jz .Lzero_bytes + + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 + add %r9, $arg5 + add \$32, %r10 + xor %r9, %r9 + jmp .Lymm_loop + +.Lstart_aligned_to_4x8: + lea (%r10,%r9,4), %r10 + xor %r9, %r9 + +.align 32 +.Lymm_loop: + cmp \$8, %r12d + jb .Llt_8_bytes + + vmovq ($arg2,%r9), %xmm31 # Read 8 bytes from lane 0 + vpinsrq \$1, ($arg3,%r9), %xmm31, %xmm31 # Read 8 bytes from lane 1 + vmovq ($arg4,%r9), %xmm30 # Read 8 bytes from lane 2 + vpinsrq \$1, ($arg5,%r9),%xmm30, %xmm30 # Read 8 bytes from lane 3 + vinserti32x4 \$1, %xmm30, %ymm31, %ymm31 + vpxorq (%r10,%r9,4), %ymm31, %ymm31 # Add data with the state + vmovdqu64 %ymm31, (%r10,%r9,4) + add \$8, %r9 + sub \$8, %r12 + jz .Lzero_bytes + + jmp .Lymm_loop + +.align 32 +.Lzero_bytes: + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 + add %r9, $arg5 + ret + +.align 32 +.Llt_8_bytes: + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 + add %r9, $arg5 + lea (%r10,%r9,4), %r10 + + lea byte_kmask_0_to_7(%rip), %rbx + kmovb (%rbx,%r12), %k1 # message load mask + + vmovdqu8 ($arg2), %xmm31{%k1}{z} # Read 1 to 7 bytes from lane 0 + vmovdqu8 ($arg3), %xmm30{%k1}{z} # Read 1 to 7 bytes from lane 1 + vpunpcklqdq %xmm30, %xmm31, %xmm31 # Interleave data from lane 0 and lane 1 + vmovdqu8 ($arg4), %xmm30{%k1}{z} # Read 1 to 7 bytes from lane 2 + vmovdqu8 ($arg5), %xmm29{%k1}{z} # Read 1 to 7 bytes from lane 3 + vpunpcklqdq %xmm29, %xmm30, %xmm30 # Interleave data from lane 2 and lane 3 + vinserti32x4 \$1, %xmm30, %ymm31, %ymm31 + + vpxorq (%r10), %ymm31, %ymm31 # Add data to the state + vmovdqu64 %ymm31, (%r10) # Update state in memory + + add %r12, $arg2 # increment message pointer lane 0 + add %r12, $arg3 # increment message pointer lane 1 + add %r12, $arg4 # increment message pointer lane 2 + add %r12, $arg5 # increment message pointer lane 3 + ret +.cfi_endproc +.size keccak_1600_partial_add_x4,.-keccak_1600_partial_add_x4 + + +# Extract bytes from state and write to outputs +# Arguments: +# r10: state pointer to start extracting from (clobbered) +# arg1 (rdi): output pointer lane 0 (updated on output) +# arg2 (rsi): output pointer lane 1 (updated on output) +# arg3 (rdx): output pointer lane 2 (updated on output) +# arg4 (rcx): output pointer lane 3 (updated on output) +# r12: length in bytes (clobbered on output) +# r11: state offset to start extract from +.globl keccak_1600_extract_bytes_x4 +.type keccak_1600_extract_bytes_x4,\@abi-omnipotent +.align 32 +keccak_1600_extract_bytes_x4: +.cfi_startproc + or %r12, %r12 + jz .Lextract_zero_bytes + + test \$7, %r11d + jz .Lextract_start_aligned_to_4x8 + + # Extract offset is not aligned to the register size (8 bytes) + mov %r11, %r9 + + and \$7, %r9d + neg %r9d + add \$8, %r9d # register capacity = 8 - (offset % 8) + cmp %r9d, %r12d + cmovnae %r12d, %r9d # %r9d = min(register capacity, length) + + lea byte_kmask_0_to_7(%rip), %rbx + kmovb (%rbx,%r9), %k1 # message store mask + + mov %r11, %rbx + and \$~7, %ebx + lea (%r10,%rbx,4), %r10 # get to state starting register + + mov %r11, %rbx + and \$7, %ebx + + vmovdqu8 8*0(%r10,%rbx), %xmm31{%k1}{z} # Read 1-7 bytes from state reg lane 0 + vmovdqu8 %xmm31, ($arg1){%k1} # Write 1-7 bytes to lane 0 output + + vmovdqu8 8*1(%r10,%rbx), %xmm31{%k1}{z} # Read 1-7 bytes from state reg lane 1 + vmovdqu8 %xmm31, ($arg2){%k1} # Write 1-7 bytes to lane 1 output + + vmovdqu8 8*2(%r10,%rbx), %xmm31{%k1}{z} # Read 1-7 bytes from state reg lane 2 + vmovdqu8 %xmm31, ($arg3){%k1} # Write 1-7 bytes to lane 2 output + + vmovdqu8 8*3(%r10,%rbx), %xmm31{%k1}{z} # Read 1-7 bytes from state reg lane 3 + vmovdqu8 %xmm31, ($arg4){%k1} # Write 1-7 bytes to lane 3 output + + # Increment output registers + add %r9, $arg1 + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 + + # Decrement length to extract + sub %r9, %r12 + jz .Lextract_zero_bytes + + # More data to extract, update state register pointer + add \$32, %r10 + xor %r9, %r9 + jmp .Lextract_ymm_loop + +.Lextract_start_aligned_to_4x8: + lea (%r10,%r11,4), %r10 + xor %r9, %r9 + +.align 32 +.Lextract_ymm_loop: + cmp \$8, %r12 + jb .Lextract_lt_8_bytes + + vmovdqu64 (%r10), %xmm31 + vmovdqu64 16(%r10), %xmm30 + vmovq %xmm31, ($arg1,%r9) + vpextrq \$1, %xmm31, ($arg2,%r9) + vmovq %xmm30, ($arg3,%r9) + vpextrq \$1, %xmm30, ($arg4,%r9) + add \$8, %r9 + sub \$8, %r12 + jz .Lzero_bytes_left + + add \$32, %r10 + jmp .Lextract_ymm_loop + +.align 32 +.Lzero_bytes_left: + # Increment output pointers + add %r9, $arg1 + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 +.Lextract_zero_bytes: + ret + +.align 32 +.Lextract_lt_8_bytes: + add %r9, $arg1 + add %r9, $arg2 + add %r9, $arg3 + add %r9, $arg4 + + lea byte_kmask_0_to_7(%rip), %r9 + kmovb (%r9,%r12), %k1 # k1 is the mask of message bytes to read + + vmovq 0*8(%r10), %xmm31 # Read 8 bytes from state lane 0 + vmovdqu8 %xmm31, ($arg1){%k1} # Extract 1-7 bytes into output 0 + vmovq 1*8(%r10), %xmm31 # Read 8 bytes from state lane 1 + vmovdqu8 %xmm31, ($arg2){%k1} # Extract 1-7 bytes into output 1 + vmovq 2*8(%r10), %xmm31 # Read 8 bytes from state lane 2 + vmovdqu8 %xmm31, ($arg3){%k1} # Extract 1-7 bytes into output 2 + vmovq 3*8(%r10), %xmm31 # Read 8 bytes from state lane 3 + vmovdqu8 %xmm31, ($arg4){%k1} # Extract 1-7 bytes into output 3 + + # Increment output pointers + add %r12, $arg1 + add %r12, $arg2 + add %r12, $arg3 + add %r12, $arg4 + ret +.cfi_endproc +.size keccak_1600_extract_bytes_x4,.-keccak_1600_extract_bytes_x4 + + +# SHAKE128 x4 multi-buffer functions +# These functions process 4 independent SHAKE128 streams in parallel using AVX-512VL +# State layout: 25 ymm registers (200 bytes each) + 1 qword = 808 bytes per context +# Rate: 168 bytes for SHAKE128 + +# SHA3_shake128_x4_avx512vl +# One-shot SHAKE-128 x4 function: init + absorb + finalize + squeeze +# Arguments: +# arg1 (rdi): pointer to output lane 0 +# arg2 (rsi): pointer to output lane 1 +# arg3 (rdx): pointer to output lane 2 +# arg4 (rcx): pointer to output lane 3 +# arg5 (r8): output length in bytes (must be same for all lanes) +# arg6 (r9): pointer to input lane 0 +# [stack+0]: pointer to input lane 1 +# [stack+8]: pointer to input lane 2 +# [stack+16]: pointer to input lane 3 +# [stack+24]: input length in bytes (must be same for all lanes) +# Returns: void +.globl SHA3_shake128_x4_avx512vl +.type SHA3_shake128_x4_avx512vl,\@function,10 +.align 32 +SHA3_shake128_x4_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp, %rbp + push %rbx +.cfi_push %rbx +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + + sub \$$sf_size, %rsp + mov %rsp, %rbx + +.Lshake128_x4_body: + mov $arg1, $sf_arg1(%rbx) + mov $arg2, $sf_arg2(%rbx) + mov $arg3, $sf_arg3(%rbx) + mov $arg4, $sf_arg4(%rbx) + mov $arg5, $sf_arg5(%rbx) + + lea $sf_state_x4(%rbx), $arg1 # start of x4 state on the stack frame + mov $arg1, $sf_state_ptr(%rbx) + + # Initialize the state array to zero + call keccak_1600_init_state + + call keccak_1600_save_state_x4 + + movq \$0, 8*100($arg1) # clear s[100] + + mov $sf_state_ptr(%rbx), $arg1 + mov $arg6, $arg2 +___ +$code .= <<___ if ($win64); + # xlate prologue handles up to six arguments. For one-shot x4 wrappers + # (10 args), the remaining four stay in Win64 stack slots. + mov 64(%rbp), $arg3 # arg7 from stack + mov 72(%rbp), $arg4 # arg8 from stack + mov 80(%rbp), $arg5 # arg9 from stack + mov 88(%rbp), $arg6 # arg10 from stack +___ +$code .= <<___ if (!$win64); + mov 16(%rbp), $arg3 # arg7 from stack + mov 24(%rbp), $arg4 # arg8 from stack + mov 32(%rbp), $arg5 # arg9 from stack + mov 40(%rbp), $arg6 # arg10 from stack +___ +$code.=<<___; + # Internal entry avoids Win64 xlate prologue argument remapping. +___ +$code .= call_internal("SHA3_shake128_x4_inc_absorb_avx512vl_internal"); +$code.=<<___; + + mov $sf_state_ptr(%rbx), $arg1 + call .L_SHA3_shake128_x4_inc_finalize_avx512vl + + # squeeze + mov $sf_arg1(%rbx), $arg1 + mov $sf_arg2(%rbx), $arg2 + mov $sf_arg3(%rbx), $arg3 + mov $sf_arg4(%rbx), $arg4 + mov $sf_arg5(%rbx), $arg5 + mov $sf_state_ptr(%rbx), $arg6 +___ +$code .= call_internal("SHA3_shake128_x4_inc_squeeze_avx512vl_internal"); +$code.=<<___; + + # Clear the temporary buffer + lea $sf_state_x4(%rbx), %r9 + vpxorq %ymm31, %ymm31, %ymm31 + vmovdqu64 %ymm31, 32*0(%r9) + vmovdqu64 %ymm31, 32*1(%r9) + vmovdqu64 %ymm31, 32*2(%r9) + vmovdqu64 %ymm31, 32*3(%r9) + vmovdqu64 %ymm31, 32*4(%r9) + vmovdqu64 %ymm31, 32*5(%r9) + vmovdqu64 %ymm31, 32*6(%r9) + vmovdqu64 %ymm31, 32*7(%r9) + vmovdqu64 %ymm31, 32*8(%r9) + vmovdqu64 %ymm31, 32*9(%r9) + vmovdqu64 %ymm31, 32*10(%r9) + vmovdqu64 %ymm31, 32*11(%r9) + vmovdqu64 %ymm31, 32*12(%r9) + vmovdqu64 %ymm31, 32*13(%r9) + vmovdqu64 %ymm31, 32*14(%r9) + vmovdqu64 %ymm31, 32*15(%r9) + vmovdqu64 %ymm31, 32*16(%r9) + vmovdqu64 %ymm31, 32*17(%r9) + vmovdqu64 %ymm31, 32*18(%r9) + vmovdqu64 %ymm31, 32*19(%r9) + vmovdqu64 %ymm31, 32*20(%r9) + vmovdqu64 %ymm31, 32*21(%r9) + vmovdqu64 %ymm31, 32*22(%r9) + vmovdqu64 %ymm31, 32*23(%r9) + vmovdqu64 %ymm31, 32*24(%r9) + vmovq %xmm31, 32*25(%r9) + +.Lshake128_x4_epilogue: +___ +$code .= <<___ if ($win64); + vmovups $sf_size+0(%rsp), %xmm6 + vmovups $sf_size+16(%rsp), %xmm7 + vmovups $sf_size+32(%rsp), %xmm8 + vmovups $sf_size+48(%rsp), %xmm9 + vmovups $sf_size+64(%rsp), %xmm10 + vmovups $sf_size+80(%rsp), %xmm11 + vmovups $sf_size+96(%rsp), %xmm12 + vmovups $sf_size+112(%rsp), %xmm13 + vmovups $sf_size+128(%rsp), %xmm14 + vmovups $sf_size+144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + add \$$sf_size, %rsp + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake128_x4_avx512vl,.-SHA3_shake128_x4_avx512vl + +___ + +$code .= <<___ if ($win64); +# Internal Win64 shim for absorb entry. It establishes xlate-compatible +# unwind state and then jumps to the function entry after the prologue. +# This is required for internal calls since the xlate ABI conversion +# is already done in the caller function. +.type SHA3_shake128_x4_inc_absorb_avx512vl_internal,\@abi-omnipotent +.align 32 +.LSEH_begin_SHA3_shake128_x4_inc_absorb_avx512vl_internal: +SHA3_shake128_x4_inc_absorb_avx512vl_internal: + mov %rsp, %rax + mov $arg1, 8(%rsp) + mov $arg2, 16(%rsp) + jmp .L_SHA3_shake128_x4_inc_absorb_avx512vl +.LSEH_end_SHA3_shake128_x4_inc_absorb_avx512vl_internal: +.size SHA3_shake128_x4_inc_absorb_avx512vl_internal,.-SHA3_shake128_x4_inc_absorb_avx512vl_internal +___ +$code.=<<___; + +# SHA3_shake128_x4_inc_absorb_avx512vl +# Absorb input data into 4 parallel SHAKE128 states +# Arguments: +# arg1 (rdi): pointer to state context (808 bytes) +# arg2 (rsi): pointer to lane 0 input data +# arg3 (rdx): pointer to lane 1 input data +# arg4 (rcx): pointer to lane 2 input data +# arg5 (r8): pointer to lane 3 input data +# arg6 (r9): input length in bytes (must be same for all lanes) +# Returns: void +# Note: Input is XORed into state and Keccak permutation is applied for each rate-sized block +.globl SHA3_shake128_x4_inc_absorb_avx512vl +.type SHA3_shake128_x4_inc_absorb_avx512vl,\@function,6 +.align 32 +SHA3_shake128_x4_inc_absorb_avx512vl: +.L_SHA3_shake128_x4_inc_absorb_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + +.Lshake128_absorb_body: + # check for partially processed block + mov 8*100($arg1), %r14 + or %r14, %r14 # s[100] == 0? + je .Lshake128_absorb_main_loop_start + + # process remaining bytes if message long enough + mov \$168, %r12 # SHAKE128_RATE = 168 + sub %r14, %r12 # %r12 = capacity + + cmp %r12, $arg6 # if mlen <= capacity then no permute + jbe .Lshake128_absorb_skip_permute + + sub %r12, $arg6 + mov $arg6, %r11 # preserve remaining length across helper calls + + # r10/state, arg2-arg5/inputs, r12/length + mov $arg1, %r10 # %r10 = state + call keccak_1600_partial_add_x4 # arg2-arg5 are updated + + call keccak_1600_load_state_x4 + + call keccak_1600_permute + + movq \$0, 8*100($arg1) # clear s[100] + jmp .Lshake128_absorb_partial_block_done + +.Lshake128_absorb_skip_permute: + # r10/state, arg2-arg5/inputs, r12/length + mov $arg1, %r10 + mov $arg6, %r12 + mov $arg6, %r11 # preserve input length across helper call + call keccak_1600_partial_add_x4 + + lea (%r11,%r14), %r15 + mov %r15, 8*100($arg1) # s[100] += inlen + + cmp \$168, %r15 # check s[100] below SHAKE128_RATE + jb .Lshake128_absorb_exit + + call keccak_1600_load_state_x4 + + call keccak_1600_permute + + call keccak_1600_save_state_x4 + + movq \$0, 8*100($arg1) # clear s[100] + jmp .Lshake128_absorb_exit + +.Lshake128_absorb_main_loop_start: + call keccak_1600_load_state_x4 + mov $arg6, %r11 # full input length when no prior partial block + +.Lshake128_absorb_partial_block_done: + xor %r12, %r12 # zero message offset + + # Process the input message in blocks +.align 32 +.Lshake128_absorb_while_loop: + cmp \$168, %r11 # compare mlen to SHAKE128_RATE + jb .Lshake128_absorb_while_loop_done + + # Inline absorb_bytes_x4 for SHAKE128_RATE (168 bytes = 21 ymm registers) +___ + +# Generate absorb code for SHAKE128 rate (168 bytes) +for (my $i = 0; $i < 21; $i++) { + my $offset = $i * 8; + $code.=<<___; + vmovq $offset($arg2,%r12), %xmm31 + vpinsrq \$1, $offset($arg3,%r12), %xmm31, %xmm31 + vmovq $offset($arg4,%r12), %xmm30 + vpinsrq \$1, $offset($arg5,%r12), %xmm30, %xmm30 + vinserti32x4 \$1, %xmm30, %ymm31, %ymm31 + vpxorq %ymm31, %ymm$i, %ymm$i +___ +} + +$code.=<<___; + sub \$168, %r11 # Subtract the rate from the remaining length + add \$168, %r12 # Adjust offset to next block + call keccak_1600_permute # Perform the Keccak permutation + + jmp .Lshake128_absorb_while_loop + +.align 32 +.Lshake128_absorb_while_loop_done: + call keccak_1600_save_state_x4 + + mov %r11, 8*100($arg1) # update s[100] + or %r11, %r11 + jz .Lshake128_absorb_exit + + movq \$0, 8*100($arg1) # clear s[100] + + # r10/state, arg2-arg5/input, r12/length + mov $arg1, %r10 + add %r12, $arg2 + add %r12, $arg3 + add %r12, $arg4 + add %r12, $arg5 + mov %r11, %r12 + call keccak_1600_partial_add_x4 + + mov %r11, 8*100($arg1) # update s[100] + +.Lshake128_absorb_exit: + # Clear sensitive registers + vpxorq %xmm16, %xmm16, %xmm16 + vmovdqa64 %ymm16, %ymm17 + vmovdqa64 %ymm16, %ymm18 + vmovdqa64 %ymm16, %ymm19 + vmovdqa64 %ymm16, %ymm20 + vmovdqa64 %ymm16, %ymm21 + vmovdqa64 %ymm16, %ymm22 + vmovdqa64 %ymm16, %ymm23 + vmovdqa64 %ymm16, %ymm24 + vmovdqa64 %ymm16, %ymm25 + vmovdqa64 %ymm16, %ymm26 + vmovdqa64 %ymm16, %ymm27 + vmovdqa64 %ymm16, %ymm28 + vmovdqa64 %ymm16, %ymm29 + vmovdqa64 %ymm16, %ymm30 + vmovdqa64 %ymm16, %ymm31 +.Lshake128_absorb_epilogue: + vzeroall +___ +$code .= <<___ if ($win64); + vmovups 0(%rsp), %xmm6 + vmovups 16(%rsp), %xmm7 + vmovups 32(%rsp), %xmm8 + vmovups 48(%rsp), %xmm9 + vmovups 64(%rsp), %xmm10 + vmovups 80(%rsp), %xmm11 + vmovups 96(%rsp), %xmm12 + vmovups 112(%rsp), %xmm13 + vmovups 128(%rsp), %xmm14 + vmovups 144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + + pop %r15 +.cfi_pop %r15 + pop %r14 +.cfi_pop %r14 + pop %r13 +.cfi_pop %r13 + pop %r12 +.cfi_pop %r12 + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake128_x4_inc_absorb_avx512vl,.-SHA3_shake128_x4_inc_absorb_avx512vl + + +# SHA3_shake128_x4_inc_finalize_avx512vl +# Finalize absorption phase for 4 parallel SHAKE-128 states +# Adds padding and terminator bytes and clears the absorb offset +# Arguments: +# arg1 (rdi): pointer to state context (808 bytes) +# Returns: void +# Note: After this call, state is ready for squeezing output +.globl SHA3_shake128_x4_inc_finalize_avx512vl +.type SHA3_shake128_x4_inc_finalize_avx512vl,\@function,1 +.align 32 +SHA3_shake128_x4_inc_finalize_avx512vl: +.L_SHA3_shake128_x4_inc_finalize_avx512vl: +.cfi_startproc + mov 8*100($arg1), %r11 # load state offset from s[100] + mov %r11, %r10 + and \$~7, %r10d # offset to the state register + and \$7, %r11d # offset within the register + + # add EOM byte right after the message + vmovdqu32 ($arg1,%r10,4), %ymm31 + lea shake_msg_pad_x4(%rip), %r9 + sub %r11, %r9 + vmovdqu32 (%r9), %ymm30 + vpxorq %ymm30, %ymm31, %ymm31 + vmovdqu32 %ymm31, ($arg1,%r10,4) + + # add terminating byte at offset equal to rate - 1 (SHAKE128_RATE = 168) + vmovdqu32 640($arg1), %ymm31 # 168*4 - 32 = 672 - 32 = 640 + vmovdqa32 shake_terminator_byte_x4(%rip), %ymm30 + vpxorq %ymm30, %ymm31, %ymm31 + vmovdqu32 %ymm31, 640($arg1) + + movq \$0, 8*100($arg1) # clear s[100] + vpxorq %ymm31, %ymm31, %ymm31 + ret +.cfi_endproc +.size SHA3_shake128_x4_inc_finalize_avx512vl,.-SHA3_shake128_x4_inc_finalize_avx512vl + +___ + +$code .= <<___ if ($win64); +# Internal Win64 shim for squeeze entry. It establishes xlate-compatible +# unwind state and then jumps to the function entry after the prologue. +# This is required for internal calls since the xlate ABI conversion +# is already done in the caller function. +.type SHA3_shake128_x4_inc_squeeze_avx512vl_internal,\@abi-omnipotent +.align 32 +.LSEH_begin_SHA3_shake128_x4_inc_squeeze_avx512vl_internal: +SHA3_shake128_x4_inc_squeeze_avx512vl_internal: + mov %rsp, %rax + mov $arg1, 8(%rsp) + mov $arg2, 16(%rsp) + jmp .L_SHA3_shake128_x4_inc_squeeze_avx512vl +.LSEH_end_SHA3_shake128_x4_inc_squeeze_avx512vl_internal: +.size SHA3_shake128_x4_inc_squeeze_avx512vl_internal,.-SHA3_shake128_x4_inc_squeeze_avx512vl_internal +___ +$code.=<<___; + +# SHA3_shake128_x4_inc_squeeze_avx512vl +# Squeeze output from 4 parallel SHAKE128 states +# Arguments: +# arg1 (rdi): pointer to lane 0 output buffer +# arg2 (rsi): pointer to lane 1 output buffer +# arg3 (rdx): pointer to lane 2 output buffer +# arg4 (rcx): pointer to lane 3 output buffer +# arg5 (r8): output length in bytes (must be same for all lanes) +# arg6 (r9): pointer to state context (808 bytes) +# Returns: void +# Note: Can be called multiple times to generate arbitrary-length output +.globl SHA3_shake128_x4_inc_squeeze_avx512vl +.type SHA3_shake128_x4_inc_squeeze_avx512vl,\@function,6 +.align 32 +SHA3_shake128_x4_inc_squeeze_avx512vl: +.L_SHA3_shake128_x4_inc_squeeze_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + +.Lshake128_squeeze_body: + or $arg5, $arg5 + jz .Lshake128_squeeze_done + + # check for partially processed block + mov 8*100($arg6), %r15 # s[100] - capacity + or %r15, %r15 + jnz .Lshake128_squeeze_no_init_permute + + mov $arg1, %r14 + mov $arg6, $arg1 + call keccak_1600_load_state_x4 + + mov %r14, $arg1 + + xor %rbp, %rbp + jmp .Lshake128_squeeze_loop + +.align 32 +.Lshake128_squeeze_no_init_permute: + # extract bytes: r10 - state/src, arg1-arg4 - output/dst, r12 - length = min(capacity, outlen), r11 - offset + mov $arg6, %r10 + mov $arg6, %r14 # preserve state pointer across extract helper + + mov %r15, %r12 + cmp %r15, $arg5 + cmovnae $arg5, %r12 # %r12 = min(capacity, outlen) + + sub %r12, $arg5 # outlen -= length + + mov \$168, %r11d # SHAKE128_RATE + sub %r15, %r11 # state offset + + sub %r12, %r15 # capacity -= length + mov %r15, 8*100($arg6) # update s[100] + + call keccak_1600_extract_bytes_x4 + mov %r14, $arg6 # restore state pointer after helper clobbers + + or %r15, %r15 + jnz .Lshake128_squeeze_done # check s[100] not zero + + mov $arg1, %r13 # preserve arg1 + mov %r14, $arg1 + call keccak_1600_load_state_x4 + + mov %r13, $arg1 + xor %rbp, %rbp + +.align 32 +.Lshake128_squeeze_loop: + cmp \$168, $arg5 # outlen > SHAKE128_RATE + jb .Lshake128_squeeze_final_extract + + call keccak_1600_permute + + # Extract SHAKE128 rate bytes (168 bytes = 21 x 8 bytes) inline +___ + +# Generate extract code for SHAKE128 rate (168 bytes = 21 ymm registers) +for (my $i = 0; $i < 21; $i++) { + my $offset = $i * 8; + $code.=<<___; + vextracti64x2 \$1, %ymm$i, %xmm31 + vmovq %xmm$i, $offset($arg1,%rbp) + vpextrq \$1, %xmm$i, $offset($arg2,%rbp) + vmovq %xmm31, $offset($arg3,%rbp) + vpextrq \$1, %xmm31, $offset($arg4,%rbp) +___ +} + +$code.=<<___; + add \$168, %rbp # dst offset += SHAKE128_RATE + sub \$168, $arg5 # outlen -= SHAKE128_RATE + jmp .Lshake128_squeeze_loop + +.align 32 +.Lshake128_squeeze_final_extract: + or $arg5, $arg5 + jz .Lshake128_squeeze_no_end_permute + + # update output pointers + add %rbp, $arg1 + add %rbp, $arg2 + add %rbp, $arg3 + add %rbp, $arg4 + + mov \$168, %r15d # SHAKE128_RATE + sub $arg5, %r15 + mov %r15, 8*100($arg6) # s[100] = capacity + + call keccak_1600_permute + + mov $arg1, %r14 + mov $arg6, $arg1 + call keccak_1600_save_state_x4 + + mov %r14, $arg1 + + # extract bytes: r10 - state/src, arg1-arg4 - output/dst, r12 - length, r11 - offset = 0 + mov $arg6, %r10 + mov $arg5, %r12 + xor %r11, %r11 + call keccak_1600_extract_bytes_x4 + + jmp .Lshake128_squeeze_done + +.Lshake128_squeeze_no_end_permute: + movq \$0, 8*100($arg6) # s[100] = 0 + mov $arg6, $arg1 + call keccak_1600_save_state_x4 + +.Lshake128_squeeze_done: + # Clear sensitive registers + vpxorq %xmm16, %xmm16, %xmm16 + vmovdqa64 %ymm16, %ymm17 + vmovdqa64 %ymm16, %ymm18 + vmovdqa64 %ymm16, %ymm19 + vmovdqa64 %ymm16, %ymm20 + vmovdqa64 %ymm16, %ymm21 + vmovdqa64 %ymm16, %ymm22 + vmovdqa64 %ymm16, %ymm23 + vmovdqa64 %ymm16, %ymm24 + vmovdqa64 %ymm16, %ymm25 + vmovdqa64 %ymm16, %ymm26 + vmovdqa64 %ymm16, %ymm27 + vmovdqa64 %ymm16, %ymm28 + vmovdqa64 %ymm16, %ymm29 + vmovdqa64 %ymm16, %ymm30 + vmovdqa64 %ymm16, %ymm31 +.Lshake128_squeeze_epilogue: + vzeroall +___ +$code .= <<___ if ($win64); + vmovups 0(%rsp), %xmm6 + vmovups 16(%rsp), %xmm7 + vmovups 32(%rsp), %xmm8 + vmovups 48(%rsp), %xmm9 + vmovups 64(%rsp), %xmm10 + vmovups 80(%rsp), %xmm11 + vmovups 96(%rsp), %xmm12 + vmovups 112(%rsp), %xmm13 + vmovups 128(%rsp), %xmm14 + vmovups 144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + + pop %r15 +.cfi_pop %r15 + pop %r14 +.cfi_pop %r14 + pop %r13 +.cfi_pop %r13 + pop %r12 +.cfi_pop %r12 + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake128_x4_inc_squeeze_avx512vl,.-SHA3_shake128_x4_inc_squeeze_avx512vl + + +# SHAKE256 x4 multi-buffer functions +# These functions process 4 independent SHAKE256 streams in parallel using AVX-512VL +# State layout: 25 ymm registers (200 bytes each) + 1 qword = 808 bytes per context +# Rate: 136 bytes for SHAKE256 + +# SHA3_shake256_x4_avx512vl +# One-shot SHAKE-256 x4 function: init + absorb + finalize + squeeze +# Arguments: +# arg1 (rdi): pointer to output lane 0 +# arg2 (rsi): pointer to output lane 1 +# arg3 (rdx): pointer to output lane 2 +# arg4 (rcx): pointer to output lane 3 +# arg5 (r8): output length in bytes (must be same for all lanes) +# arg6 (r9): pointer to input lane 0 +# [stack+0]: pointer to input lane 1 +# [stack+8]: pointer to input lane 2 +# [stack+16]: pointer to input lane 3 +# [stack+24]: input length in bytes (must be same for all lanes) +# Returns: void +.globl SHA3_shake256_x4_avx512vl +.type SHA3_shake256_x4_avx512vl,\@function,10 +.align 32 +SHA3_shake256_x4_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + mov %rsp, %rbp + push %rbx +.cfi_push %rbx +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + + sub \$$sf_size, %rsp + mov %rsp, %rbx + +.Lshake256_x4_body: + mov $arg1, $sf_arg1(%rbx) + mov $arg2, $sf_arg2(%rbx) + mov $arg3, $sf_arg3(%rbx) + mov $arg4, $sf_arg4(%rbx) + mov $arg5, $sf_arg5(%rbx) + + lea $sf_state_x4(%rbx), $arg1 # start of x4 state on the stack frame + mov $arg1, $sf_state_ptr(%rbx) + + # Initialize the state array to zero + call keccak_1600_init_state + + call keccak_1600_save_state_x4 + + movq \$0, 8*100($arg1) # clear s[100] + + mov $sf_state_ptr(%rbx), $arg1 + mov $arg6, $arg2 +___ +$code .= <<___ if ($win64); + # xlate prologue handles up to six arguments. For one-shot x4 wrappers + # (10 args), the remaining four stay in Win64 stack slots. + mov 64(%rbp), $arg3 # arg7 from stack + mov 72(%rbp), $arg4 # arg8 from stack + mov 80(%rbp), $arg5 # arg9 from stack + mov 88(%rbp), $arg6 # arg10 from stack +___ +$code .= <<___ if (!$win64); + mov 16(%rbp), $arg3 # arg7 from stack + mov 24(%rbp), $arg4 # arg8 from stack + mov 32(%rbp), $arg5 # arg9 from stack + mov 40(%rbp), $arg6 # arg10 from stack +___ +$code.=<<___; + # Internal entry avoids Win64 xlate prologue argument remapping. +___ +$code .= call_internal("SHA3_shake256_x4_inc_absorb_avx512vl_internal"); +$code.=<<___; + + mov $sf_state_ptr(%rbx), $arg1 + call .L_SHA3_shake256_x4_inc_finalize_avx512vl + + # squeeze + mov $sf_arg1(%rbx), $arg1 + mov $sf_arg2(%rbx), $arg2 + mov $sf_arg3(%rbx), $arg3 + mov $sf_arg4(%rbx), $arg4 + mov $sf_arg5(%rbx), $arg5 + mov $sf_state_ptr(%rbx), $arg6 +___ +$code .= call_internal("SHA3_shake256_x4_inc_squeeze_avx512vl_internal"); +$code.=<<___; + + # Clear the temporary buffer + lea $sf_state_x4(%rbx), %r9 + vpxorq %ymm31, %ymm31, %ymm31 + vmovdqu64 %ymm31, 32*0(%r9) + vmovdqu64 %ymm31, 32*1(%r9) + vmovdqu64 %ymm31, 32*2(%r9) + vmovdqu64 %ymm31, 32*3(%r9) + vmovdqu64 %ymm31, 32*4(%r9) + vmovdqu64 %ymm31, 32*5(%r9) + vmovdqu64 %ymm31, 32*6(%r9) + vmovdqu64 %ymm31, 32*7(%r9) + vmovdqu64 %ymm31, 32*8(%r9) + vmovdqu64 %ymm31, 32*9(%r9) + vmovdqu64 %ymm31, 32*10(%r9) + vmovdqu64 %ymm31, 32*11(%r9) + vmovdqu64 %ymm31, 32*12(%r9) + vmovdqu64 %ymm31, 32*13(%r9) + vmovdqu64 %ymm31, 32*14(%r9) + vmovdqu64 %ymm31, 32*15(%r9) + vmovdqu64 %ymm31, 32*16(%r9) + vmovdqu64 %ymm31, 32*17(%r9) + vmovdqu64 %ymm31, 32*18(%r9) + vmovdqu64 %ymm31, 32*19(%r9) + vmovdqu64 %ymm31, 32*20(%r9) + vmovdqu64 %ymm31, 32*21(%r9) + vmovdqu64 %ymm31, 32*22(%r9) + vmovdqu64 %ymm31, 32*23(%r9) + vmovdqu64 %ymm31, 32*24(%r9) + vmovq %xmm31, 32*25(%r9) + +.Lshake256_x4_epilogue: +___ +$code .= <<___ if ($win64); + vmovups $sf_size+0(%rsp), %xmm6 + vmovups $sf_size+16(%rsp), %xmm7 + vmovups $sf_size+32(%rsp), %xmm8 + vmovups $sf_size+48(%rsp), %xmm9 + vmovups $sf_size+64(%rsp), %xmm10 + vmovups $sf_size+80(%rsp), %xmm11 + vmovups $sf_size+96(%rsp), %xmm12 + vmovups $sf_size+112(%rsp), %xmm13 + vmovups $sf_size+128(%rsp), %xmm14 + vmovups $sf_size+144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + add \$$sf_size, %rsp + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake256_x4_avx512vl,.-SHA3_shake256_x4_avx512vl + +___ + +$code .= <<___ if ($win64); +# Internal Win64 shim for absorb entry. It establishes xlate-compatible +# unwind state and then jumps to the function entry after the prologue. +# This is required for internal calls since the xlate ABI conversion +# is already done in the caller function. +.type SHA3_shake256_x4_inc_absorb_avx512vl_internal,\@abi-omnipotent +.align 32 +.LSEH_begin_SHA3_shake256_x4_inc_absorb_avx512vl_internal: +SHA3_shake256_x4_inc_absorb_avx512vl_internal: + mov %rsp, %rax + mov $arg1, 8(%rsp) + mov $arg2, 16(%rsp) + jmp .L_SHA3_shake256_x4_inc_absorb_avx512vl +.LSEH_end_SHA3_shake256_x4_inc_absorb_avx512vl_internal: +.size SHA3_shake256_x4_inc_absorb_avx512vl_internal,.-SHA3_shake256_x4_inc_absorb_avx512vl_internal +___ +$code.=<<___; + +# SHA3_shake256_x4_inc_absorb_avx512vl +# Absorb input data into 4 parallel SHAKE256 states +# Arguments: +# arg1 (rdi): pointer to state context (808 bytes) +# arg2 (rsi): pointer to lane 0 input data +# arg3 (rdx): pointer to lane 1 input data +# arg4 (rcx): pointer to lane 2 input data +# arg5 (r8): pointer to lane 3 input data +# arg6 (r9): input length in bytes (must be same for all lanes) +# Returns: void +# Note: Input is XORed into state and Keccak permutation is applied for each rate-sized block +.globl SHA3_shake256_x4_inc_absorb_avx512vl +.type SHA3_shake256_x4_inc_absorb_avx512vl,\@function,6 +.align 32 +SHA3_shake256_x4_inc_absorb_avx512vl: +.L_SHA3_shake256_x4_inc_absorb_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + +.Lshake256_absorb_body: + # check for partially processed block + mov 8*100($arg1), %r14 + or %r14, %r14 # s[100] == 0? + je .Lshake256_absorb_main_loop_start + + # process remaining bytes if message long enough + mov \$136, %r12 # SHAKE256_RATE = 136 + sub %r14, %r12 # %r12 = capacity + + cmp %r12, $arg6 # if mlen <= capacity then no permute + jbe .Lshake256_absorb_skip_permute + + sub %r12, $arg6 + mov $arg6, %r11 # preserve remaining length across helper calls + + # r10/state, arg2-arg5/inputs, r12/length + mov $arg1, %r10 # %r10 = state + call keccak_1600_partial_add_x4 # arg2-arg5 are updated + + call keccak_1600_load_state_x4 + + call keccak_1600_permute + + movq \$0, 8*100($arg1) # clear s[100] + jmp .Lshake256_absorb_partial_block_done + +.Lshake256_absorb_skip_permute: + # r10/state, arg2-arg5/inputs, r12/length + mov $arg1, %r10 + mov $arg6, %r12 + mov $arg6, %r11 # preserve input length across helper call + call keccak_1600_partial_add_x4 + + lea (%r11,%r14), %r15 + mov %r15, 8*100($arg1) # s[100] += inlen + + cmp \$136, %r15 # check s[100] below SHAKE256_RATE + jb .Lshake256_absorb_exit + + call keccak_1600_load_state_x4 + + call keccak_1600_permute + + call keccak_1600_save_state_x4 + + movq \$0, 8*100($arg1) # clear s[100] + jmp .Lshake256_absorb_exit + +.Lshake256_absorb_main_loop_start: + call keccak_1600_load_state_x4 + mov $arg6, %r11 # full input length when no prior partial block + +.Lshake256_absorb_partial_block_done: + xor %r12, %r12 # zero message offset + + # Process the input message in blocks +.align 32 +.Lshake256_absorb_while_loop: + cmp \$136, %r11 # compare mlen to SHAKE256_RATE + jb .Lshake256_absorb_while_loop_done + + # Inline absorb_bytes_x4 for SHAKE256_RATE (136 bytes = 17 ymm registers) +___ + +# Generate absorb code for SHAKE256 rate (136 bytes) +for (my $i = 0; $i < 17; $i++) { + my $offset = $i * 8; + $code.=<<___; + vmovq $offset($arg2,%r12), %xmm31 + vpinsrq \$1, $offset($arg3,%r12), %xmm31, %xmm31 + vmovq $offset($arg4,%r12), %xmm30 + vpinsrq \$1, $offset($arg5,%r12), %xmm30, %xmm30 + vinserti32x4 \$1, %xmm30, %ymm31, %ymm31 + vpxorq %ymm31, %ymm$i, %ymm$i +___ +} + +$code.=<<___; + sub \$136, %r11 # Subtract the rate from the remaining length + add \$136, %r12 # Adjust offset to next block + call keccak_1600_permute # Perform the Keccak permutation + + jmp .Lshake256_absorb_while_loop + +.align 32 +.Lshake256_absorb_while_loop_done: + call keccak_1600_save_state_x4 + + mov %r11, 8*100($arg1) # update s[100] + or %r11, %r11 + jz .Lshake256_absorb_exit + + movq \$0, 8*100($arg1) # clear s[100] + + # r10/state, arg2-arg5/input, r12/length + mov $arg1, %r10 + add %r12, $arg2 + add %r12, $arg3 + add %r12, $arg4 + add %r12, $arg5 + mov %r11, %r12 + call keccak_1600_partial_add_x4 + + mov %r11, 8*100($arg1) # update s[100] + +.Lshake256_absorb_exit: + # Clear sensitive registers + vpxorq %xmm16, %xmm16, %xmm16 + vmovdqa64 %ymm16, %ymm17 + vmovdqa64 %ymm16, %ymm18 + vmovdqa64 %ymm16, %ymm19 + vmovdqa64 %ymm16, %ymm20 + vmovdqa64 %ymm16, %ymm21 + vmovdqa64 %ymm16, %ymm22 + vmovdqa64 %ymm16, %ymm23 + vmovdqa64 %ymm16, %ymm24 + vmovdqa64 %ymm16, %ymm25 + vmovdqa64 %ymm16, %ymm26 + vmovdqa64 %ymm16, %ymm27 + vmovdqa64 %ymm16, %ymm28 + vmovdqa64 %ymm16, %ymm29 + vmovdqa64 %ymm16, %ymm30 + vmovdqa64 %ymm16, %ymm31 +.Lshake256_absorb_epilogue: + vzeroall +___ +$code .= <<___ if ($win64); + vmovups 0(%rsp), %xmm6 + vmovups 16(%rsp), %xmm7 + vmovups 32(%rsp), %xmm8 + vmovups 48(%rsp), %xmm9 + vmovups 64(%rsp), %xmm10 + vmovups 80(%rsp), %xmm11 + vmovups 96(%rsp), %xmm12 + vmovups 112(%rsp), %xmm13 + vmovups 128(%rsp), %xmm14 + vmovups 144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + + pop %r15 +.cfi_pop %r15 + pop %r14 +.cfi_pop %r14 + pop %r13 +.cfi_pop %r13 + pop %r12 +.cfi_pop %r12 + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake256_x4_inc_absorb_avx512vl,.-SHA3_shake256_x4_inc_absorb_avx512vl + + +# SHA3_shake256_x4_inc_finalize_avx512vl +# Finalize absorption phase for 4 parallel SHAKE-256 states +# Adds padding and terminator bytes and clears the absorb offset +# Arguments: +# arg1 (rdi): pointer to state context (808 bytes) +# Returns: void +# Note: After this call, state is ready for squeezing output +.globl SHA3_shake256_x4_inc_finalize_avx512vl +.type SHA3_shake256_x4_inc_finalize_avx512vl,\@function,1 +.align 32 +SHA3_shake256_x4_inc_finalize_avx512vl: +.L_SHA3_shake256_x4_inc_finalize_avx512vl: +.cfi_startproc + mov 8*100($arg1), %r11 # load state offset from s[100] + mov %r11, %r10 + and \$~7, %r10d # offset to the state register + and \$7, %r11d # offset within the register + + # add EOM byte right after the message + vmovdqu32 ($arg1,%r10,4), %ymm31 + lea shake_msg_pad_x4(%rip), %r9 + sub %r11, %r9 + vmovdqu32 (%r9), %ymm30 + vpxorq %ymm30, %ymm31, %ymm31 + vmovdqu32 %ymm31, ($arg1,%r10,4) + + # add terminating byte at offset equal to rate - 1 (SHAKE256_RATE = 136) + vmovdqu32 512($arg1), %ymm31 # 136*4 - 32 = 544 - 32 = 512 + vmovdqa32 shake_terminator_byte_x4(%rip), %ymm30 + vpxorq %ymm30, %ymm31, %ymm31 + vmovdqu32 %ymm31, 512($arg1) + + movq \$0, 8*100($arg1) # clear s[100] + vpxorq %ymm31, %ymm31, %ymm31 + ret +.cfi_endproc +.size SHA3_shake256_x4_inc_finalize_avx512vl,.-SHA3_shake256_x4_inc_finalize_avx512vl + +___ + +$code .= <<___ if ($win64); +# Internal Win64 shim for squeeze entry. It establishes xlate-compatible +# unwind state and then jumps to the function entry after the prologue. +# This is required for internal calls since the xlate ABI conversion +# is already done in the caller function. +.type SHA3_shake256_x4_inc_squeeze_avx512vl_internal,\@abi-omnipotent +.align 32 +.LSEH_begin_SHA3_shake256_x4_inc_squeeze_avx512vl_internal: +SHA3_shake256_x4_inc_squeeze_avx512vl_internal: + mov %rsp, %rax + mov $arg1, 8(%rsp) + mov $arg2, 16(%rsp) + jmp .L_SHA3_shake256_x4_inc_squeeze_avx512vl +.LSEH_end_SHA3_shake256_x4_inc_squeeze_avx512vl_internal: +.size SHA3_shake256_x4_inc_squeeze_avx512vl_internal,.-SHA3_shake256_x4_inc_squeeze_avx512vl_internal +___ +$code.=<<___; + +# SHA3_shake256_x4_inc_squeeze_avx512vl +# Squeeze output from 4 parallel SHAKE256 states +# Arguments: +# arg1 (rdi): pointer to lane 0 output buffer +# arg2 (rsi): pointer to lane 1 output buffer +# arg3 (rdx): pointer to lane 2 output buffer +# arg4 (rcx): pointer to lane 3 output buffer +# arg5 (r8): output length in bytes (must be same for all lanes) +# arg6 (r9): pointer to state context (808 bytes) +# Returns: void +# Note: Can be called multiple times to generate arbitrary-length output +.globl SHA3_shake256_x4_inc_squeeze_avx512vl +.type SHA3_shake256_x4_inc_squeeze_avx512vl,\@function,6 +.align 32 +SHA3_shake256_x4_inc_squeeze_avx512vl: +.L_SHA3_shake256_x4_inc_squeeze_avx512vl: +.cfi_startproc + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +___ +$code .= <<___ if ($win64); + sub \$160, %rsp + vmovups %xmm6, 0(%rsp) + vmovups %xmm7, 16(%rsp) + vmovups %xmm8, 32(%rsp) + vmovups %xmm9, 48(%rsp) + vmovups %xmm10, 64(%rsp) + vmovups %xmm11, 80(%rsp) + vmovups %xmm12, 96(%rsp) + vmovups %xmm13, 112(%rsp) + vmovups %xmm14, 128(%rsp) + vmovups %xmm15, 144(%rsp) +___ +$code.=<<___; + +.Lshake256_squeeze_body: + or $arg5, $arg5 + jz .Lshake256_squeeze_done + + # check for partially processed block + mov 8*100($arg6), %r15 # s[100] - capacity + or %r15, %r15 + jnz .Lshake256_squeeze_no_init_permute + + mov $arg1, %r14 + mov $arg6, $arg1 + call keccak_1600_load_state_x4 + + mov %r14, $arg1 + + xor %rbp, %rbp + jmp .Lshake256_squeeze_loop + +.align 32 +.Lshake256_squeeze_no_init_permute: + # extract bytes: r10 - state/src, arg1-arg4 - output/dst, r12 - length = min(capacity, outlen), r11 - offset + mov $arg6, %r10 + mov $arg6, %r14 # preserve state pointer across extract helper + + mov %r15, %r12 + cmp %r15, $arg5 + cmovnae $arg5, %r12 # %r12 = min(capacity, outlen) + + sub %r12, $arg5 # outlen -= length + + mov \$136, %r11d # SHAKE256_RATE + sub %r15, %r11 # state offset + + sub %r12, %r15 # capacity -= length + mov %r15, 8*100($arg6) # update s[100] + + call keccak_1600_extract_bytes_x4 + mov %r14, $arg6 # restore state pointer after helper clobbers + + or %r15, %r15 + jnz .Lshake256_squeeze_done # check s[100] not zero + + mov $arg1, %r13 # preserve arg1 + mov %r14, $arg1 + call keccak_1600_load_state_x4 + + mov %r13, $arg1 + xor %rbp, %rbp + +.align 32 +.Lshake256_squeeze_loop: + cmp \$136, $arg5 # outlen > SHAKE256_RATE + jb .Lshake256_squeeze_final_extract + + call keccak_1600_permute + + # Extract SHAKE256 rate bytes (136 bytes = 17 x 8 bytes) inline +___ + +# Generate extract code for SHAKE256 rate (136 bytes = 17 ymm registers) +for (my $i = 0; $i < 17; $i++) { + my $offset = $i * 8; + $code.=<<___; + vextracti64x2 \$1, %ymm$i, %xmm31 + vmovq %xmm$i, $offset($arg1,%rbp) + vpextrq \$1, %xmm$i, $offset($arg2,%rbp) + vmovq %xmm31, $offset($arg3,%rbp) + vpextrq \$1, %xmm31, $offset($arg4,%rbp) +___ +} + +$code.=<<___; + add \$136, %rbp # dst offset += SHAKE256_RATE + sub \$136, $arg5 # outlen -= SHAKE256_RATE + jmp .Lshake256_squeeze_loop + +.align 32 +.Lshake256_squeeze_final_extract: + or $arg5, $arg5 + jz .Lshake256_squeeze_no_end_permute + + # update output pointers + add %rbp, $arg1 + add %rbp, $arg2 + add %rbp, $arg3 + add %rbp, $arg4 + + mov \$136, %r15d # SHAKE256_RATE + sub $arg5, %r15 + mov %r15, 8*100($arg6) # s[100] = capacity + + call keccak_1600_permute + + mov $arg1, %r14 + mov $arg6, $arg1 + call keccak_1600_save_state_x4 + + mov %r14, $arg1 + + # extract bytes: r10 - state/src, arg1-arg4 - output/dst, r12 - length, r11 - offset = 0 + mov $arg6, %r10 + mov $arg5, %r12 + xor %r11, %r11 + call keccak_1600_extract_bytes_x4 + + jmp .Lshake256_squeeze_done + +.Lshake256_squeeze_no_end_permute: + movq \$0, 8*100($arg6) # s[100] = 0 + mov $arg6, $arg1 + call keccak_1600_save_state_x4 + +.Lshake256_squeeze_done: + # Clear sensitive registers + vpxorq %xmm16, %xmm16, %xmm16 + vmovdqa64 %ymm16, %ymm17 + vmovdqa64 %ymm16, %ymm18 + vmovdqa64 %ymm16, %ymm19 + vmovdqa64 %ymm16, %ymm20 + vmovdqa64 %ymm16, %ymm21 + vmovdqa64 %ymm16, %ymm22 + vmovdqa64 %ymm16, %ymm23 + vmovdqa64 %ymm16, %ymm24 + vmovdqa64 %ymm16, %ymm25 + vmovdqa64 %ymm16, %ymm26 + vmovdqa64 %ymm16, %ymm27 + vmovdqa64 %ymm16, %ymm28 + vmovdqa64 %ymm16, %ymm29 + vmovdqa64 %ymm16, %ymm30 + vmovdqa64 %ymm16, %ymm31 +.Lshake256_squeeze_epilogue: + vzeroall +___ +$code .= <<___ if ($win64); + vmovups 0(%rsp), %xmm6 + vmovups 16(%rsp), %xmm7 + vmovups 32(%rsp), %xmm8 + vmovups 48(%rsp), %xmm9 + vmovups 64(%rsp), %xmm10 + vmovups 80(%rsp), %xmm11 + vmovups 96(%rsp), %xmm12 + vmovups 112(%rsp), %xmm13 + vmovups 128(%rsp), %xmm14 + vmovups 144(%rsp), %xmm15 + add \$160, %rsp +___ +$code.=<<___; + + pop %r15 +.cfi_pop %r15 + pop %r14 +.cfi_pop %r14 + pop %r13 +.cfi_pop %r13 + pop %r12 +.cfi_pop %r12 + pop %rbx +.cfi_pop %rbx + pop %rbp +.cfi_pop %rbp + ret +.cfi_endproc +.size SHA3_shake256_x4_inc_squeeze_avx512vl,.-SHA3_shake256_x4_inc_squeeze_avx512vl +___ + +if ($win64) { +my $context = "%r8"; +my $disp = "%r9"; + +$code.=<<___; +.extern __imp_RtlVirtualUnwind +.type keccak_se_handler,\@abi-omnipotent +.align 16 +keccak_se_handler: + push %rsi + push %rdi + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + pushfq + sub \$64, %rsp + + mov 120($context), %rax # context->Rax = original %rsp from xlate prologue + mov 248($context), %rbx # context->Rip + + mov 8($disp), %rsi # disp->ImageBase + mov 56($disp), %r11 # disp->HandlerData + + mov 0(%r11), %r10d # HandlerData[0]: body label (rva) + lea (%rsi,%r10), %r10 + cmp %r10, %rbx # Rip < body? + jb .Lkeccak_in_prologue + + mov 4(%r11), %r10d # HandlerData[1]: epilogue label (rva) + lea (%rsi,%r10), %r10 + cmp %r10, %rbx # Rip >= epilogue? + jae .Lkeccak_in_epilogue + + # In function body: + # HandlerData[2]: delta from context->Rsp(body) to original %rsp + # HandlerData[3]: offset of XMM6 save area from context->Rsp(body), -1 if none + # HandlerData[4]: number of saved non-volatiles in stack frame layout (2 or 6) + # HandlerData[5]: delta from context->Rsp(epilogue) to original %rsp + mov 152($context), %rdx # body rsp + mov 8(%r11), %r10d + lea (%rdx,%r10), %rax # original rsp + jmp .Lkeccak_restore_body_or_epilogue + +.Lkeccak_in_epilogue: + mov 152($context), %rdx # epilogue rsp + mov 20(%r11), %r10d + lea (%rdx,%r10), %rax # original rsp + +.Lkeccak_restore_body_or_epilogue: + mov 8(%rax), %rcx # xlate shadow save of original rdi + mov 16(%rax), %rsi # xlate shadow save of original rsi + mov %rax, 152($context) # context->Rsp = original rsp + mov %rsi, 168($context) # context->Rsi + mov %rcx, 176($context) # context->Rdi + + mov 16(%r11), %r10d # gpr save count + cmp \$6, %r10d + jne .Lkeccak_restore_two + + mov -24(%rax), %r12 + mov -32(%rax), %r13 + mov -40(%rax), %r14 + mov -48(%rax), %r15 + mov %r12, 216($context) # context->R12 + mov %r13, 224($context) # context->R13 + mov %r14, 232($context) # context->R14 + mov %r15, 240($context) # context->R15 + +.Lkeccak_restore_two: + mov -8(%rax), %rbp + mov -16(%rax), %rbx + mov %rbp, 160($context) # context->Rbp + mov %rbx, 144($context) # context->Rbx + + mov 12(%r11), %r10d # xmm save offset from body rsp + cmp \$-1, %r10d + je .Lkeccak_in_prologue + + lea (%rdx,%r10), %rsi # source = xmm save area + lea 512($context), %rdi # &context->Xmm6 + mov \$20, %ecx # 10 XMM * 2 qwords + .long 0xa548f3fc # cld; rep movsq + +.Lkeccak_in_prologue: + mov 8(%rax), %rcx + mov 16(%rax), %rdx + mov %rcx, 176($context) # context->Rdi + mov %rdx, 168($context) # context->Rsi + mov %rax, 152($context) # context->Rsp = original rsp + + mov 40($disp), %rdi # disp->ContextRecord + mov $context, %rsi + mov \$154, %ecx # sizeof(CONTEXT)/8 + .long 0xa548f3fc # cld; rep movsq + + mov $disp, %rsi + xor %rcx, %rcx # UNW_FLAG_NHANDLER + mov 8(%rsi), %rdx # disp->ImageBase + mov 0(%rsi), %r8 # disp->ControlPc + mov 16(%rsi), %r9 # disp->FunctionEntry + mov 40(%rsi), %r10 # disp->ContextRecord + lea 56(%rsi), %r11 # &disp->HandlerData + lea 24(%rsi), %r12 # &disp->EstablisherFrame + mov %r10, 32(%rsp) + mov %r11, 40(%rsp) + mov %r12, 48(%rsp) + mov %rcx, 56(%rsp) + call *__imp_RtlVirtualUnwind(%rip) + + mov \$1, %eax # ExceptionContinueSearch + add \$64, %rsp + popfq + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + pop %rdi + pop %rsi + ret +.size keccak_se_handler,.-keccak_se_handler + +.section .pdata +.align 4 + .rva .LSEH_begin_SHA3_shake128_x4_avx512vl + .rva .LSEH_end_SHA3_shake128_x4_avx512vl + .rva .LSEH_info_SHA3_shake128_x4_avx512vl + .rva .LSEH_begin_SHA3_shake128_x4_inc_absorb_avx512vl_internal + .rva .LSEH_end_SHA3_shake128_x4_inc_absorb_avx512vl_internal + .rva .LSEH_info_SHA3_shake128_x4_inc_absorb_avx512vl_internal + .rva .LSEH_begin_SHA3_shake128_x4_inc_absorb_avx512vl + .rva .LSEH_end_SHA3_shake128_x4_inc_absorb_avx512vl + .rva .LSEH_info_SHA3_shake128_x4_inc_absorb_avx512vl + .rva .LSEH_begin_SHA3_shake128_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_end_SHA3_shake128_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_info_SHA3_shake128_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_begin_SHA3_shake128_x4_inc_squeeze_avx512vl + .rva .LSEH_end_SHA3_shake128_x4_inc_squeeze_avx512vl + .rva .LSEH_info_SHA3_shake128_x4_inc_squeeze_avx512vl + .rva .LSEH_begin_SHA3_shake256_x4_avx512vl + .rva .LSEH_end_SHA3_shake256_x4_avx512vl + .rva .LSEH_info_SHA3_shake256_x4_avx512vl + .rva .LSEH_begin_SHA3_shake256_x4_inc_absorb_avx512vl_internal + .rva .LSEH_end_SHA3_shake256_x4_inc_absorb_avx512vl_internal + .rva .LSEH_info_SHA3_shake256_x4_inc_absorb_avx512vl_internal + .rva .LSEH_begin_SHA3_shake256_x4_inc_absorb_avx512vl + .rva .LSEH_end_SHA3_shake256_x4_inc_absorb_avx512vl + .rva .LSEH_info_SHA3_shake256_x4_inc_absorb_avx512vl + .rva .LSEH_begin_SHA3_shake256_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_end_SHA3_shake256_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_info_SHA3_shake256_x4_inc_squeeze_avx512vl_internal + .rva .LSEH_begin_SHA3_shake256_x4_inc_squeeze_avx512vl + .rva .LSEH_end_SHA3_shake256_x4_inc_squeeze_avx512vl + .rva .LSEH_info_SHA3_shake256_x4_inc_squeeze_avx512vl + +.section .xdata +.align 8 +.LSEH_info_SHA3_shake128_x4_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake128_x4_body,.Lshake128_x4_epilogue + .long 1032,856,2,1032 +.LSEH_info_SHA3_shake128_x4_inc_absorb_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake128_absorb_body,.Lshake128_absorb_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake128_x4_inc_absorb_avx512vl_internal: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake128_absorb_body,.Lshake128_absorb_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake128_x4_inc_squeeze_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake128_squeeze_body,.Lshake128_squeeze_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake128_x4_inc_squeeze_avx512vl_internal: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake128_squeeze_body,.Lshake128_squeeze_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake256_x4_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake256_x4_body,.Lshake256_x4_epilogue + .long 1032,856,2,1032 +.LSEH_info_SHA3_shake256_x4_inc_absorb_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake256_absorb_body,.Lshake256_absorb_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake256_x4_inc_absorb_avx512vl_internal: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake256_absorb_body,.Lshake256_absorb_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake256_x4_inc_squeeze_avx512vl: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake256_squeeze_body,.Lshake256_squeeze_epilogue + .long 208,0,6,208 +.LSEH_info_SHA3_shake256_x4_inc_squeeze_avx512vl_internal: + .byte 9,0,0,0 + .rva keccak_se_handler + .rva .Lshake256_squeeze_body,.Lshake256_squeeze_epilogue + .long 208,0,6,208 +___ +} + +$code.=<<___; + +.section .rodata align=128 +.align 128 +.type iotas,\@object +iotas: + .quad 0x0000000000000001 + .quad 0x0000000000008082 + .quad 0x800000000000808a + .quad 0x8000000080008000 + .quad 0x000000000000808b + .quad 0x0000000080000001 + .quad 0x8000000080008081 + .quad 0x8000000000008009 + .quad 0x000000000000008a + .quad 0x0000000000000088 + .quad 0x0000000080008009 + .quad 0x000000008000000a + .quad 0x000000008000808b + .quad 0x800000000000008b + .quad 0x8000000000008089 + .quad 0x8000000000008003 + .quad 0x8000000000008002 + .quad 0x8000000000000080 + .quad 0x000000000000800a + .quad 0x800000008000000a + .quad 0x8000000080008081 + .quad 0x8000000000008080 + .quad 0x0000000080000001 + .quad 0x8000000080008008 +.size iotas,.-iotas + +.align 8 +byte_kmask_0_to_7: + .byte 0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f + +.align 32 +shake_terminator_byte_x4: + .byte 0, 0, 0, 0, 0, 0, 0, 0x80 + .byte 0, 0, 0, 0, 0, 0, 0, 0x80 + .byte 0, 0, 0, 0, 0, 0, 0, 0x80 + .byte 0, 0, 0, 0, 0, 0, 0, 0x80 + +.align 8 + .byte 0, 0, 0, 0, 0, 0, 0, 0 +shake_msg_pad_x4: + .byte 0x1F, 0, 0, 0, 0, 0, 0, 0 + .byte 0x1F, 0, 0, 0, 0, 0, 0, 0 + .byte 0x1F, 0, 0, 0, 0, 0, 0, 0 + .byte 0x1F, 0, 0, 0, 0, 0, 0, 0 + +.asciz "Keccak-1600 absorb and squeeze for AVX512VL, CRYPTOGAMS by " +___ + +}}} else {{{ + +# When AVX512VL is not available, output stub functions +# The capable function returns 0, and the operation functions are not defined (will use C fallback) + +$code .= <<___; +.text + +.globl SHA3_avx512vl_capable +.type SHA3_avx512vl_capable,\@abi-omnipotent +SHA3_avx512vl_capable: + xor %eax, %eax + ret +.size SHA3_avx512vl_capable, .-SHA3_avx512vl_capable + +.globl SHA3_shake128_x4_inc_absorb_avx512vl +.globl SHA3_shake256_x4_inc_absorb_avx512vl +.globl SHA3_shake128_x4_inc_finalize_avx512vl +.globl SHA3_shake256_x4_inc_finalize_avx512vl +.globl SHA3_shake128_x4_inc_squeeze_avx512vl +.globl SHA3_shake256_x4_inc_squeeze_avx512vl +.globl SHA3_shake128_x4_avx512vl +.globl SHA3_shake256_x4_avx512vl +.type SHA3_shake128_x4_inc_absorb_avx512vl,\@abi-omnipotent +SHA3_shake128_x4_inc_absorb_avx512vl: +SHA3_shake256_x4_inc_absorb_avx512vl: +SHA3_shake128_x4_inc_finalize_avx512vl: +SHA3_shake256_x4_inc_finalize_avx512vl: +SHA3_shake128_x4_inc_squeeze_avx512vl: +SHA3_shake256_x4_inc_squeeze_avx512vl: +SHA3_shake128_x4_avx512vl: +SHA3_shake256_x4_avx512vl: + .byte 0x0f,0x0b # ud2 + ret +.size SHA3_shake128_x4_inc_absorb_avx512vl, .-SHA3_shake128_x4_inc_absorb_avx512vl +___ +}}} + +print $code; +close STDOUT or die "error closing STDOUT: $!"; diff --git a/crypto/sha/build.info b/crypto/sha/build.info index 457ac8d06ab..88e8b9cc5e7 100644 --- a/crypto/sha/build.info +++ b/crypto/sha/build.info @@ -65,7 +65,7 @@ ENDIF $KECCAK1600ASM=keccak1600.c IF[{- !$disabled{asm} -}] $KECCAK1600ASM_x86= - $KECCAK1600ASM_x86_64=keccak1600-x86_64.s + $KECCAK1600ASM_x86_64=keccak1600-x86_64.s keccak1600x4-avx512vl.s sha3_x4_avx512vl.c $KECCAK1600ASM_s390x=keccak1600-s390x.S @@ -198,4 +198,8 @@ GENERATE[keccak1600-avx512vl.S]=asm/keccak1600-avx512vl.pl GENERATE[keccak1600-mmx.S]=asm/keccak1600-mmx.pl GENERATE[keccak1600p8-ppc.S]=asm/keccak1600p8-ppc.pl +# keccak1600x4-avx512vl.s supports multi-squeeze +# Currently only used in ML-DSA on x86_64 with AVX-512VL support +GENERATE[keccak1600x4-avx512vl.s]=asm/keccak1600x4-avx512vl.pl + GENERATE[sha1-thumb.S]=asm/sha1-thumb.pl diff --git a/crypto/sha/sha3_x4_avx512vl.c b/crypto/sha/sha3_x4_avx512vl.c new file mode 100644 index 00000000000..86a8282814f --- /dev/null +++ b/crypto/sha/sha3_x4_avx512vl.c @@ -0,0 +1,213 @@ +/* + * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved. + * Copyright (c) 2026 Intel Corporation. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +/* + * SHAKE x4 multi-buffer implementation for AVX-512VL + * + * This file provides incremental API wrappers around the AVX-512VL + * assembly implementations for processing 4 SHAKE instances in parallel. + * + * Callers should check SHA3_avx512vl_capable() before calling. + */ + +#include "internal/sha3.h" +#include +#include + +#if defined(KECCAK1600_ASM) \ + && (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) \ + && !defined(OPENSSL_NO_ASM) + +/* External assembly function declarations */ +extern void SHA3_shake128_x4_inc_absorb_avx512vl( + uint64_t *state, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +extern void SHA3_shake256_x4_inc_absorb_avx512vl( + uint64_t *state, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +extern void SHA3_shake128_x4_inc_finalize_avx512vl(uint64_t *state); +extern void SHA3_shake256_x4_inc_finalize_avx512vl(uint64_t *state); + +extern void SHA3_shake128_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + uint64_t *state); + +extern void SHA3_shake256_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + uint64_t *state); + +/* One-shot assembly function declarations */ +extern void SHA3_shake128_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +extern void SHA3_shake256_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +/* + * SHAKE-128 x4 Implementation + */ + +void ossl_sha3_shake128_x4_inc_init_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + memset(ctx->A, 0, sizeof(ctx->A)); + ctx->rate = SHA3_BLOCKSIZE(128); + ctx->finalized = 0; +} + +void ossl_sha3_shake128_x4_inc_absorb_avx512vl( + KECCAK1600_X4_AVX512VL_CTX *ctx, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen) +{ + if (ctx->finalized) { + /* Error: cannot absorb after finalize */ + return; + } + + SHA3_shake128_x4_inc_absorb_avx512vl( + ctx->A, in0, in1, in2, in3, inlen); +} + +void ossl_sha3_shake128_x4_inc_cleanup_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + OPENSSL_cleanse(ctx, sizeof(*ctx)); +} + +static void ossl_sha3_shake128_x4_inc_finalize_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + if (ctx->finalized) { + return; /* Already finalized */ + } + + SHA3_shake128_x4_inc_finalize_avx512vl(ctx->A); + ctx->finalized = 1; +} + +void ossl_sha3_shake128_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + if (!ctx->finalized) { + /* Auto-finalize on first squeeze */ + ossl_sha3_shake128_x4_inc_finalize_avx512vl(ctx); + } + + SHA3_shake128_x4_inc_squeeze_avx512vl( + out0, out1, out2, out3, outlen, ctx->A); +} + +/* + * SHAKE-256 x4 Implementation + */ + +void ossl_sha3_shake256_x4_inc_init_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + memset(ctx->A, 0, sizeof(ctx->A)); + ctx->rate = SHA3_BLOCKSIZE(256); + ctx->finalized = 0; +} + +void ossl_sha3_shake256_x4_inc_absorb_avx512vl( + KECCAK1600_X4_AVX512VL_CTX *ctx, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen) +{ + if (ctx->finalized) { + /* Error: cannot absorb after finalize */ + return; + } + + SHA3_shake256_x4_inc_absorb_avx512vl( + ctx->A, in0, in1, in2, in3, inlen); +} + +void ossl_sha3_shake256_x4_inc_cleanup_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + OPENSSL_cleanse(ctx, sizeof(*ctx)); +} + +static void ossl_sha3_shake256_x4_inc_finalize_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + if (ctx->finalized) { + return; /* Already finalized */ + } + + SHA3_shake256_x4_inc_finalize_avx512vl(ctx->A); + ctx->finalized = 1; +} + +void ossl_sha3_shake256_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + KECCAK1600_X4_AVX512VL_CTX *ctx) +{ + if (!ctx->finalized) { + /* Auto-finalize on first squeeze */ + ossl_sha3_shake256_x4_inc_finalize_avx512vl(ctx); + } + + SHA3_shake256_x4_inc_squeeze_avx512vl( + out0, out1, out2, out3, outlen, ctx->A); +} + +/* + * Single-call wrapper APIs + */ + +void ossl_sha3_shake128_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen) +{ + SHA3_shake128_x4_avx512vl(out0, out1, out2, out3, outlen, + in0, in1, in2, in3, inlen); +} + +void ossl_sha3_shake256_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen) +{ + SHA3_shake256_x4_avx512vl(out0, out1, out2, out3, outlen, + in0, in1, in2, in3, inlen); +} + +#endif /* KECCAK1600_ASM && x86_64 && !OPENSSL_NO_ASM */ diff --git a/include/internal/sha3.h b/include/internal/sha3.h index f91d00a74f8..0e0ab76fb56 100644 --- a/include/internal/sha3.h +++ b/include/internal/sha3.h @@ -65,4 +65,75 @@ int ossl_shake_squeeze_default(KECCAK1600_CTX *ctx, unsigned char *out, size_t o size_t SHA3_absorb(uint64_t A[5][5], const unsigned char *inp, size_t len, size_t r); +/* Multi-buffer (x4) Keccak-f[1600] context and API */ +#if defined(KECCAK1600_ASM) \ + && (defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) \ + && !defined(OPENSSL_NO_ASM) + +/* Runtime capability check for AVX512VL */ +int SHA3_avx512vl_capable(void); + +/* Context for 4-way parallel SHAKE operations */ +typedef struct { + /* 4 interleaved Keccak states (800 bytes) + plus 8 bytes to store the number of + already absorbed or not yet squeezed bytes */ + uint64_t A[(25 * 4) + 1]; + size_t rate; /* Rate in bytes: 168 (SHAKE-128) or 136 (SHAKE-256) */ + unsigned finalized; /* Has finalize been called? 0=no, 1=yes */ +} KECCAK1600_X4_AVX512VL_CTX; + +/* SHAKE-128 x4 incremental API */ +void ossl_sha3_shake128_x4_inc_init_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx); + +void ossl_sha3_shake128_x4_inc_absorb_avx512vl( + KECCAK1600_X4_AVX512VL_CTX *ctx, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +void ossl_sha3_shake128_x4_inc_cleanup_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx); + +void ossl_sha3_shake128_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + KECCAK1600_X4_AVX512VL_CTX *ctx); + +/* SHAKE-256 x4 incremental API */ +void ossl_sha3_shake256_x4_inc_init_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx); + +void ossl_sha3_shake256_x4_inc_absorb_avx512vl( + KECCAK1600_X4_AVX512VL_CTX *ctx, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +void ossl_sha3_shake256_x4_inc_cleanup_avx512vl(KECCAK1600_X4_AVX512VL_CTX *ctx); + +void ossl_sha3_shake256_x4_inc_squeeze_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + KECCAK1600_X4_AVX512VL_CTX *ctx); + +/* Single-call SHAKE x4 APIs (wrapper functions) */ +void ossl_sha3_shake128_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +void ossl_sha3_shake256_x4_avx512vl( + void *out0, void *out1, + void *out2, void *out3, + size_t outlen, + const void *in0, const void *in1, + const void *in2, const void *in3, + size_t inlen); + +#endif /* KECCAK1600_ASM && x86_64 && !OPENSSL_NO_ASM */ + #endif /* OSSL_INTERNAL_SHA3_H */ diff --git a/providers/fips-sources.checksums b/providers/fips-sources.checksums index 527e124d765..598c3bb44dc 100644 --- a/providers/fips-sources.checksums +++ b/providers/fips-sources.checksums @@ -270,6 +270,7 @@ ff65c82c56e341f47df03d0c74de7fb537de0e68a4fa23fa07a9fdb51c511f1c crypto/ml_dsa/ 3e0980e67842c4d8637fa449ac41e9d650c614c1074c29f1021605d229a4f73d crypto/ml_dsa/ml_dsa_params.c 10e37ab3ee09a45d99007665e073efb2b062c819f30af8694c6b0f411eb33822 crypto/ml_dsa/ml_dsa_poly.h 26be5266a9f1a33999a5a68c96cffc7932ba64521d9554dabe7397591611c852 crypto/ml_dsa/ml_dsa_sample.c +9e57d844f2acedb490b6e8f32e125240078ddab380a7faa533baa9c447dee262 crypto/ml_dsa/ml_dsa_sample_hw_x86_64.inc 2127303173eff12cb2b71f92179d10370b555c26e7a305e87ce7066580d57689 crypto/ml_dsa/ml_dsa_sign.c 5217ef237e21872205703b95577290c34898423466a465c7bd609b2eb4627964 crypto/ml_dsa/ml_dsa_sign.h e3ef4cf1598420c94eee4f53d13491ff0f9dc8cfb1fe1cacd3a1225e2073fa56 crypto/ml_dsa/ml_dsa_vector.h @@ -359,6 +360,7 @@ b7f1f4e69d41812dba39226aea0942fb2e3a638308c3130b0c83bb636a258ce6 crypto/sha/asm 82a84e6ae5ffe90e76530de560f6261c5004c23923fa9fd9aa2aae3852976426 crypto/sha/asm/keccak1600-s390x.pl e485942ed7f7bf1f376059ed2e5e194907348f44a4308dfc4bbc2d8be05fed99 crypto/sha/asm/keccak1600-x86_64.pl af173b53537e18453705a3843e0629334b37e8ca03483a2b164fd48252289da3 crypto/sha/asm/keccak1600p8-ppc.pl +d27078e0478f34536f596e037887ad673b05537d9cd4b79376e902774a4c8340 crypto/sha/asm/keccak1600x4-avx512vl.pl 3dd5e288f70b684d337f4054119935ab46af4042a40dffdca203868fe943bae0 crypto/sha/asm/sha1-586.pl 58378cb694b61022d70956ab344331d7577f9a26431acd00a60bed91229b29eb crypto/sha/asm/sha1-alpha.pl 8576b406e8596c8ac56a351a92175accfcf4121dd3dbffc59bae7d0ece0b0dab crypto/sha/asm/sha1-armv4-large.pl @@ -401,6 +403,7 @@ feb4283e1bb2a36f0a98d4cd7b43ac78c40b72efeab8d58b4f0aad8b65a1d2ba crypto/sha/asm f6021e6e5f4694579ba5a014e2aba44b6d436b2fd25c18ecd474a7383042b027 crypto/sha/sha256.c ed6049496d9786296d011a4582ffcfe0859a609b37f1f4ce57cee36136ee526c crypto/sha/sha3.c db0f16a4cb9c86f971b4defbee9dccd77345766f64efc45e0f1bc8d448bfb3e3 crypto/sha/sha3_encode.c +8def0b2e5996f2a2e187abcb7443e5e98edd4be0f71ff918a1b8c717cb97eb27 crypto/sha/sha3_x4_avx512vl.c 833e0990a4b8590a40990bfe1cfe59542ece6923fde37d3dd6faddfe1b9418fc crypto/sha/sha512.c 7b99b3c9fa26c5e58a56cbbb62b6e2bfe62542a662799f910387e76e5688a13a crypto/sha/sha_local.h dfd99e02830973ab349409ac6ba0ee901ba7736216030965bd7e5a54356abd7c crypto/slh_dsa/slh_adrs.c @@ -497,7 +500,7 @@ ee75ecd35b3ae90c51ace957ab7ce06de3c7d5064b97a878241ff65cc943a6db include/intern 3f476694478c1125574dfb2e75c4c0c0d04c7d3d763176686a4730028513fcd5 include/internal/rcu.h b6e33da6011b2b74d27e39f27cf98e6f123fe47826562b6479d0dbd5c758c4c2 include/internal/refcount.h f77c0844cc44bd92965647cd8cb6addb210f0300a8d1090da8c26e4382e87c2c include/internal/safe_math.h -ea565ff98b36d3ae2d8eb709a52cba75720b30e2527cdfda3b90aa426291ac38 include/internal/sha3.h +17f6585bc81ad324d00aaacd558e0176dbe22cc7cbdb353c8c9072fa40000a58 include/internal/sha3.h 8e672cc0620606b044f63b8446125f5233d64e3eea59df54c1fcca6bc90ba537 include/internal/sizes.h 188be736ff23a2202fe594e6d49a7ccf4c23a7baa86e2dac5d58360f21d9c986 include/internal/skey.h abf03dc8635f2925bdc2299feabe115f8d5d6eaa450b421172ded222872386ba include/internal/ssl3_cbc.h diff --git a/providers/fips.checksum b/providers/fips.checksum index b70425fb459..2c7bab5eab1 100644 --- a/providers/fips.checksum +++ b/providers/fips.checksum @@ -1 +1 @@ -7e5428946b50e077e099dd7bd3891b854b37c623ede0cc8ace51c09b94871db7 providers/fips-sources.checksums +864d3535e188440ca60b8268ce8137e0cffb870be990531fd9f68a23f0ffa898 providers/fips-sources.checksums diff --git a/providers/fips.module.sources b/providers/fips.module.sources index 7b404dbecef..53584585732 100644 --- a/providers/fips.module.sources +++ b/providers/fips.module.sources @@ -270,6 +270,7 @@ crypto/ml_dsa/ml_dsa_ntt.c crypto/ml_dsa/ml_dsa_params.c crypto/ml_dsa/ml_dsa_poly.h crypto/ml_dsa/ml_dsa_sample.c +crypto/ml_dsa/ml_dsa_sample_hw_x86_64.inc crypto/ml_dsa/ml_dsa_sign.c crypto/ml_dsa/ml_dsa_sign.h crypto/ml_dsa/ml_dsa_vector.h @@ -359,6 +360,7 @@ crypto/sha/asm/keccak1600-ppc64.pl crypto/sha/asm/keccak1600-s390x.pl crypto/sha/asm/keccak1600-x86_64.pl crypto/sha/asm/keccak1600p8-ppc.pl +crypto/sha/asm/keccak1600x4-avx512vl.pl crypto/sha/asm/sha1-586.pl crypto/sha/asm/sha1-alpha.pl crypto/sha/asm/sha1-armv4-large.pl @@ -401,6 +403,7 @@ crypto/sha/sha1dgst.c crypto/sha/sha256.c crypto/sha/sha3.c crypto/sha/sha3_encode.c +crypto/sha/sha3_x4_avx512vl.c crypto/sha/sha512.c crypto/sha/sha_local.h crypto/slh_dsa/slh_adrs.c