From: Alexander Troosh Date: Fri, 10 Oct 2025 10:58:59 +0000 (+0300) Subject: GCM/ghash specialization for the e2kv6 (Elbrus2000) arch X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=66f75fbcccf522e7ea5ead6beb1e6a84375e6326;p=thirdparty%2Fopenssl.git GCM/ghash specialization for the e2kv6 (Elbrus2000) arch Co-authored-by: Gleb Popov Reviewed-by: Andrew Dinh Reviewed-by: Igor Ustinov MergeDate: Fri Jul 31 16:49:56 2026 (Merged from https://github.com/openssl/openssl/pull/31269) --- diff --git a/crypto/modes/asm/ghash-e2kv6.c b/crypto/modes/asm/ghash-e2kv6.c new file mode 100644 index 00000000000..25615693e22 --- /dev/null +++ b/crypto/modes/asm/ghash-e2kv6.c @@ -0,0 +1,180 @@ +/* + * Copyright 2026 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the Apache License 2.0 (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#include +#include + +#include "crypto/modes.h" + +static __attribute__((__always_inline__)) inline __v2di reverse_vector(const __v2di in) +{ + __v2di fmt = __builtin_e2k_qppackdl(0x0001020304050607LL, 0x08090a0b0c0d0e0fLL); + return __builtin_e2k_qppermb(in, in, fmt); +} + +static __attribute__((__always_inline__)) inline __v2di gcm_reduce(__v2di B0, __v2di B1) +{ + __v2di X0 = __builtin_e2k_qpsrlw(B1, 31); + __v2di X1 = __builtin_e2k_qpsllw(B1, 1); + __v2di X2 = __builtin_e2k_qpsrlw(B0, 31); + __v2di X3 = __builtin_e2k_qpsllw(B0, 1); + + X3 = X3 | __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x8080808080808080LL, 0x808080800f0e0d0cLL)) + | __builtin_e2k_qpshufb(X2, X2, __builtin_e2k_qppackdl(0x0b0a090807060504LL, 0x0302010080808080LL)); + + X1 = X1 | __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x0b0a090807060504LL, 0x0302010080808080LL)); + + X0 = __builtin_e2k_qpsllw(X1, 31) ^ __builtin_e2k_qpsllw(X1, 30) ^ __builtin_e2k_qpsllw(X1, 25); + + X1 ^= __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x0302010080808080LL, 0x8080808080808080LL)); + + X0 = X1 ^ X3 ^ __builtin_e2k_qpshufb(X0, X0, __builtin_e2k_qppackdl(0x808080800f0e0d0cLL, 0x0b0a090807060504LL)); + + X0 ^= __builtin_e2k_qpsrlw(X1, 7) ^ __builtin_e2k_qpsrlw(X1, 2) ^ __builtin_e2k_qpsrlw(X1, 1); + + return X0; +} + +static __attribute__((__always_inline__)) inline __v2di gcm_multiply(__v2di H, __v2di x) +{ + uint64_t Hh = H[1], Hl = H[0]; + uint64_t xh = x[1], xl = x[0]; + + uint64_t T0h = __builtin_e2k_clmulh(Hh, xh), T0l = __builtin_e2k_clmull(Hh, xh); + uint64_t T1h = __builtin_e2k_clmulh(Hh, xl), T1l = __builtin_e2k_clmull(Hh, xl); + uint64_t T2h = __builtin_e2k_clmulh(Hl, xh), T2l = __builtin_e2k_clmull(Hl, xh); + uint64_t T3h = __builtin_e2k_clmulh(Hl, xl), T3l = __builtin_e2k_clmull(Hl, xl); + + T1h = __builtin_e2k_pxord(T1h, T2h); + T1l = __builtin_e2k_pxord(T1l, T2l); + + T0l = __builtin_e2k_pxord(T0l, T1h); + T3h = __builtin_e2k_pxord(T3h, T1l); + + return gcm_reduce(__builtin_e2k_qppackdl(T0h, T0l), __builtin_e2k_qppackdl(T3h, T3l)); +} + +static __attribute__((__always_inline__)) inline __v2di gcm_multiply_x4(__v2di H1, __v2di H2, __v2di H3, __v2di H4, + __v2di X1, __v2di X2, __v2di X3, __v2di X4) +{ + /* + * Multiply with delayed reduction, algorithm by Krzysztof Jankowski + * and Pierre Laurent of Intel + */ + + const uint64_t loh = (__builtin_e2k_clmulh(H1[0], X1[0]) ^ __builtin_e2k_clmulh(H2[0], X2[0])) ^ (__builtin_e2k_clmulh(H3[0], X3[0]) ^ __builtin_e2k_clmulh(H4[0], X4[0])); + const uint64_t lol = (__builtin_e2k_clmull(H1[0], X1[0]) ^ __builtin_e2k_clmull(H2[0], X2[0])) ^ (__builtin_e2k_clmull(H3[0], X3[0]) ^ __builtin_e2k_clmull(H4[0], X4[0])); + + const uint64_t hih = (__builtin_e2k_clmulh(H1[1], X1[1]) ^ __builtin_e2k_clmulh(H2[1], X2[1])) ^ (__builtin_e2k_clmulh(H3[1], X3[1]) ^ __builtin_e2k_clmulh(H4[1], X4[1])); + const uint64_t hil = (__builtin_e2k_clmull(H1[1], X1[1]) ^ __builtin_e2k_clmull(H2[1], X2[1])) ^ (__builtin_e2k_clmull(H3[1], X3[1]) ^ __builtin_e2k_clmull(H4[1], X4[1])); + uint64_t Th, Tl; + + Th = __builtin_e2k_clmulh(H1[0] ^ H1[1], X1[0] ^ X1[1]); + Tl = __builtin_e2k_clmull(H1[0] ^ H1[1], X1[0] ^ X1[1]); + + Th ^= __builtin_e2k_clmulh(H2[0] ^ H2[1], X2[0] ^ X2[1]); + Tl ^= __builtin_e2k_clmull(H2[0] ^ H2[1], X2[0] ^ X2[1]); + + Th ^= __builtin_e2k_clmulh(H3[0] ^ H3[1], X3[0] ^ X3[1]); + Tl ^= __builtin_e2k_clmull(H3[0] ^ H3[1], X3[0] ^ X3[1]); + + Th ^= __builtin_e2k_clmulh(H4[0] ^ H4[1], X4[0] ^ X4[1]); + Tl ^= __builtin_e2k_clmull(H4[0] ^ H4[1], X4[0] ^ X4[1]); + + Th ^= loh; + Tl ^= lol; + Th ^= hih; + Tl ^= hil; + + return gcm_reduce(__builtin_e2k_qppackdl(hih, hil ^ Th), + __builtin_e2k_qppackdl(loh ^ Tl, lol)); +} + +/*############################################################################## +# void gcm_init_e2kv6_clmul(u128 Htable[16],const uint64_t H[2]); +# +# input: 128-bit H - secret parameter E(K,0^128) +# output: precomputed table filled with degrees of twisted H; +# H is twisted to handle reverse bitness of GHASH; +# only few of 16 slots of Htable[16] are used; +# data is opaque to outside world (which allows to +# optimize the code independently); +# +*/ +void gcm_init_e2kv6_clmul(u128 Htable[16], const uint64_t H[2]) +{ + __v2di *Hp = (__v2di *)Htable; + __v2di H1 = (__v2di) { H[1], H[0] }; /* H in LE, but need swap hi/lo */ + __v2di H2 = gcm_multiply(H1, H1); + __v2di H3 = gcm_multiply(H1, H2); + __v2di H4 = gcm_multiply(H2, H2); + + Hp[0] = H1; + Hp[1] = H2; + Hp[2] = H3; + Hp[3] = H4; +} + +/*############################################################################## +# void gcm_gmult_e2kv6_clmul(uint64_t Xi[2],const u128 Htable[16]); +# +# input: Xi - current hash value; +# Htable - table precomputed in gcm_init_e2kv6_clmul; +# output: Xi - next hash value Xi; +*/ +void gcm_gmult_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16]) +{ + __v2di *Xp = (__v2di *)Xi; + __v2di *Hp = (__v2di *)Htable; + *Xp = reverse_vector(gcm_multiply(Hp[0], reverse_vector(*Xp))); +} + +/*############################################################################## +# void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16], +# const uint8_t *inp,size_t len); +# +# input: table precomputed in gcm_init_e2kv6_clmul; +# current hash value Xi; +# pointer to input data; +# length of input data in bytes, but divisible by block size; +# output: next hash value Xi; +*/ +void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16], + const uint8_t *inp, size_t len) +{ + __v2di *Hp = (__v2di *)Htable; + __v2di *input = (__v2di *)inp; + __v2di x; + size_t i, blocks = (len >> 4); + + x = reverse_vector(*(__v2di *)Xi); + + while (blocks >= 4) { + __v2di m0 = reverse_vector(input[0]); + __v2di m1 = reverse_vector(input[1]); + __v2di m2 = reverse_vector(input[2]); + __v2di m3 = reverse_vector(input[3]); + + x ^= m0; + x = gcm_multiply_x4(Hp[0], Hp[1], Hp[2], Hp[3], m3, m2, m1, x); + + input += 4; + blocks -= 4; + } + +#pragma loop count(3) + for (i = 0; i < blocks; i++) { + __v2di m = reverse_vector(input[i]); + + x ^= m; + x = gcm_multiply(Hp[0], x); + } + + *(__v2di *)Xi = reverse_vector(x); +} diff --git a/crypto/modes/build.info b/crypto/modes/build.info index 9fb2d503d8d..cd4e0330dac 100644 --- a/crypto/modes/build.info +++ b/crypto/modes/build.info @@ -46,6 +46,11 @@ IF[{- !$disabled{asm} -}] $MODESASM_riscv64=ghash-riscv64.s ghash-riscv64-zvkb-zvbc.s ghash-riscv64-zvkg.s aes-gcm-riscv64-zvkb-zvkg-zvkned.s $MODESDEF_riscv64=GHASH_ASM + $MODESASM_e2kv6=asm/ghash-e2kv6.c + $MODESDEF_e2kv6=GHASH_ASM + $MODESASM_e2kv7=asm/ghash-e2kv6.c + $MODESDEF_e2kv7=GHASH_ASM + # Now that we have defined all the arch specific variables, use the # appropriate one, and define the appropriate macros IF[$MODESASM_{- $target{asm_arch} -}] diff --git a/crypto/modes/gcm128.c b/crypto/modes/gcm128.c index 1b77c2e27e5..b4cab1bc000 100644 --- a/crypto/modes/gcm128.c +++ b/crypto/modes/gcm128.c @@ -422,6 +422,12 @@ void gcm_init_rv64i_zvkg_zvkb(u128 Htable[16], const uint64_t Xi[2]); void gcm_gmult_rv64i_zvkg(uint64_t Xi[2], const u128 Htable[16]); void gcm_ghash_rv64i_zvkg(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp, size_t len); +#elif defined(__e2k__) && (__iset__ >= 6) +#define GHASH_ASM_E2KV6 +void gcm_init_e2kv6_clmul(u128 Htable[16], const uint64_t Xi[2]); +void gcm_gmult_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16]); +void gcm_ghash_e2kv6_clmul(uint64_t Xi[2], const u128 Htable[16], + const uint8_t *inp, size_t len); #endif #endif @@ -552,6 +558,11 @@ static void gcm_get_funcs(struct gcm_funcs_st *ctx) } } return; +#elif defined(GHASH_ASM_E2KV6) + ctx->ginit = gcm_init_e2kv6_clmul; + ctx->gmult = gcm_gmult_e2kv6_clmul; + ctx->ghash = gcm_ghash_e2kv6_clmul; + return; #elif defined(GHASH_ASM) /* all other architectures use the generic names */ ctx->gmult = gcm_gmult_4bit; diff --git a/include/crypto/modes.h b/include/crypto/modes.h index baa0f3a6c14..7076e196478 100644 --- a/include/crypto/modes.h +++ b/include/crypto/modes.h @@ -25,7 +25,7 @@ #define STRICT_ALIGNMENT 1 #ifndef PEDANTIC -#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || defined(__aarch64__) || defined(__s390__) || defined(__s390x__) +#if defined(__i386) || defined(__i386__) || defined(__x86_64) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || defined(__aarch64__) || defined(__s390__) || defined(__s390x__) || defined(__e2k__) #undef STRICT_ALIGNMENT #endif #endif @@ -72,6 +72,9 @@ #define BSWAP4(x) ({ uint32_t ret_=(x); \ asm ("rev8 %0,%0; srli %0,%0,32"\ : "+&r"(ret_)); ret_; }) +#elif defined(__e2k__) +#define BSWAP8(x) __builtin_bswap64(x) +#define BSWAP4(x) __builtin_bswap32(x) #endif #elif defined(_MSC_VER) #if _MSC_VER >= 1300