Fix max saltlen calculation in cryptoapi.c

(nbits - 1)/8 should have been rounded up. Fix and move it to
an inlined function for reuse in pkcs11_openssl.c (used in the
next commit).

Note: The error is not triggered in normal use as OpenSSL
always seems to use saltlen="digest" for signing.

Signed-off-by: Selva Nair <selva.nair@gmail.com>
Acked-by: Arne Schwabe <arne@rfc2549.org>
Message-Id: <20220125025128.2117-2-selva.nair@gmail.com>
URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg23648.html
Signed-off-by: Gert Doering <gert@greenie.muc.de>

diff --git a/src/openvpn/cryptoapi.c b/src/openvpn/cryptoapi.c
index 08cb434f5949c9cc5d27d0520acd4fd377a7e3c1..f8818963c1944cd4475f5f0d2511a356d290bf98 100644 (file)
--- a/src/openvpn/cryptoapi.c
+++ b/src/openvpn/cryptoapi.c
@@ -860,7 +860,7 @@ xkey_cng_rsa_sign(CAPI_DATA *cd, unsigned char *sig, size_t *siglen, const unsig
         int saltlen = tbslen; /* digest size by default */
         if (!strcmp(sigalg.saltlen, "max"))
         {
-            saltlen = (EVP_PKEY_bits(cd->pubkey) - 1)/8 - tbslen - 2;
+            saltlen = xkey_max_saltlen(EVP_PKEY_bits(cd->pubkey), tbslen);
             if (saltlen < 0)
             {
                 msg(M_NONFATAL, "Error in cryptoapicert: invalid salt length (%d)", saltlen);

diff --git a/src/openvpn/xkey_common.h b/src/openvpn/xkey_common.h
index 308c7050df9915d61fcbc9b805d59a18d445ac0e..e58748b4229eba58c53134b3b67fc7e55077aea2 100644 (file)
--- a/src/openvpn/xkey_common.h
+++ b/src/openvpn/xkey_common.h
@@ -153,6 +153,20 @@ xkey_load_generic_key(OSSL_LIB_CTX *libctx, void *handle, EVP_PKEY *pubkey,
 
 extern OSSL_LIB_CTX *tls_libctx; /* Global */
 
+/**
+ * Maximum salt length for PSS signature.
+ *
+ * @param modBits    Number of bits in RSA modulus
+ * @param hLen       Length of digest to be signed
+ * @returns the maximum allowed salt length. Caller must check it's not < 0.
+ */
+static inline int
+xkey_max_saltlen(int modBits, int hLen)
+{
+    int emLen = (modBits - 1 + 7)/8; /* ceil((modBits - 1)/8) */
+
+    return emLen - hLen - 2;
+}
 #endif /* HAVE_XKEY_PROVIDER */
 
 #endif /* XKEY_COMMON_H_ */