From: Selva Nair Date: Tue, 25 Jan 2022 02:51:27 +0000 (-0500) Subject: Fix max saltlen calculation in cryptoapi.c X-Git-Tag: v2.6_beta1~306 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=72daac6973c304b93e6516879948c5470d0c805a;p=thirdparty%2Fopenvpn.git Fix max saltlen calculation in cryptoapi.c (nbits - 1)/8 should have been rounded up. Fix and move it to an inlined function for reuse in pkcs11_openssl.c (used in the next commit). Note: The error is not triggered in normal use as OpenSSL always seems to use saltlen="digest" for signing. Signed-off-by: Selva Nair Acked-by: Arne Schwabe Message-Id: <20220125025128.2117-2-selva.nair@gmail.com> URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg23648.html Signed-off-by: Gert Doering --- diff --git a/src/openvpn/cryptoapi.c b/src/openvpn/cryptoapi.c index 08cb434f5..f8818963c 100644 --- a/src/openvpn/cryptoapi.c +++ b/src/openvpn/cryptoapi.c @@ -860,7 +860,7 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig int saltlen = tbslen; /* digest size by default */ if (!strcmp(sigalg.saltlen, "max")) { - saltlen = (EVP_PKEY_bits(cd->pubkey) - 1)/8 - tbslen - 2; + saltlen = xkey_max_saltlen(EVP_PKEY_bits(cd->pubkey), tbslen); if (saltlen < 0) { msg(M_NONFATAL, "Error in cryptoapicert: invalid salt length (%d)", saltlen); diff --git a/src/openvpn/xkey_common.h b/src/openvpn/xkey_common.h index 308c7050d..e58748b42 100644 --- a/src/openvpn/xkey_common.h +++ b/src/openvpn/xkey_common.h @@ -153,6 +153,20 @@ xkey_load_generic_key(OSSL_LIB_CTX *libctx, void *handle, EVP_PKEY *pubkey, extern OSSL_LIB_CTX *tls_libctx; /* Global */ +/** + * Maximum salt length for PSS signature. + * + * @param modBits Number of bits in RSA modulus + * @param hLen Length of digest to be signed + * @returns the maximum allowed salt length. Caller must check it's not < 0. + */ +static inline int +xkey_max_saltlen(int modBits, int hLen) +{ + int emLen = (modBits - 1 + 7)/8; /* ceil((modBits - 1)/8) */ + + return emLen - hLen - 2; +} #endif /* HAVE_XKEY_PROVIDER */ #endif /* XKEY_COMMON_H_ */