Support no-digest X509 certificate keys like ML-DSA/EdDSA (#2165)

Recent OpenSSL releases (e.g., OpenSSL v3.5) support several private key
types[^1] for which supplying a message digest algorithm is prohibited
when signing a certificate. Prior to this enhancement, Squid was
rejecting https_port and http_port configurations using such key types
(with the above FATAL message) because OpenSSL X509_sign() call made
with a prohibited (for the given key type) non-nil digest algorithm was
failing.

Technically, only listening ports with generate-host-certificates (and
ssl-bump) parameters need to generate X509 certificates and, hence, call
X509_sign(). However, current Squid code generates so called "untrusted"
certificates even for ports that do not support dynamic host certificate
generation or SslBump (XXX). Thus, this enhancement is applicable to
both regular and SslBump configurations.

[^1]: Known no-message-digest key types are ML-DSA-44, ML-DSA-65,
ML-DSA-87, ED25519, and ED448, but others might exist or will be added.
This change was tested against known types, but should support others.
ML-DSA key types are used in post-quantum cryptography.

diff --git a/acinclude/lib-checks.m4 b/acinclude/lib-checks.m4
index 9793b9a0556c93e226948486ead2cf46879aa049..538034b8b6973edec1efb737f2f6a87515841624 100644 (file)
--- a/acinclude/lib-checks.m4
+++ b/acinclude/lib-checks.m4
@@ -58,6 +58,7 @@ AC_DEFUN([SQUID_CHECK_LIBCRYPTO_API],[
   AH_TEMPLATE(HAVE_LIBCRYPTO_DH_UP_REF, "Define to 1 if the DH_up_ref() OpenSSL API function exists")
   AH_TEMPLATE(HAVE_LIBCRYPTO_X509_GET0_SIGNATURE, "Define to 1 if the X509_get0_signature() OpenSSL API function exists")
   AH_TEMPLATE(HAVE_SSL_GET0_PARAM, "Define to 1 of the SSL_get0_param() OpenSSL API function exists")
+  AH_TEMPLATE(HAVE_LIBCRYPTO_EVP_PKEY_GET_DEFAULT_DIGEST_NAME, "Define to 1 if the EVP_PKEY_get_default_digest_name() OpenSSL API function exists")
   SQUID_STATE_SAVE(check_openssl_libcrypto_api)
   LIBS="$LIBS $SSLLIB"
   AC_CHECK_LIB(crypto, OPENSSL_LH_strhash, AC_DEFINE(HAVE_LIBCRYPTO_OPENSSL_LH_STRHASH, 1))
@@ -77,6 +78,7 @@ AC_DEFUN([SQUID_CHECK_LIBCRYPTO_API],[
   AC_CHECK_LIB(crypto, DH_up_ref, AC_DEFINE(HAVE_LIBCRYPTO_DH_UP_REF, 1))
   AC_CHECK_LIB(crypto, X509_get0_signature, AC_DEFINE(HAVE_LIBCRYPTO_X509_GET0_SIGNATURE, 1), AC_DEFINE(SQUID_CONST_X509_GET0_SIGNATURE_ARGS,))
   AC_CHECK_LIB(crypto, SSL_get0_param, AC_DEFINE(HAVE_SSL_GET0_PARAM, 1))
+  AC_CHECK_LIB(crypto, EVP_PKEY_get_default_digest_name, AC_DEFINE(HAVE_LIBCRYPTO_EVP_PKEY_GET_DEFAULT_DIGEST_NAME, 1))
   SQUID_STATE_ROLLBACK(check_openssl_libcrypto_api)
 ])
 

diff --git a/src/ssl/gadgets.cc b/src/ssl/gadgets.cc
index a8406df39c85d4173bdc04ef8534b0d7728fa5d8..87a9f9dda56bf2ebe3af37f057021b5a81134ff7 100644 (file)
--- a/src/ssl/gadgets.cc
+++ b/src/ssl/gadgets.cc
@@ -15,6 +15,52 @@
 #include "security/Io.h"
 #include "ssl/gadgets.h"
 
+/// whether to supply a digest algorithm name when calling X509_sign() with the given key
+static bool
+signWithDigest(const Security::PrivateKeyPointer &key) {
+#if HAVE_LIBCRYPTO_EVP_PKEY_GET_DEFAULT_DIGEST_NAME
+    Assure(key); // TODO: Add and use Security::PrivateKey (here and in caller).
+    const auto pkey = key.get();
+
+    // OpenSSL does not define a maximum name size, but does terminate longer
+    // names without returning an error to the caller. Many similar callers in
+    // OpenSSL sources use 80-byte buffers.
+    char defaultDigestName[80] = "";
+    const auto nameGetterResult = EVP_PKEY_get_default_digest_name(pkey, defaultDigestName, sizeof(defaultDigestName));
+    debugs(83, 3, "nameGetterResult=" << nameGetterResult << " defaultDigestName=" << defaultDigestName);
+    if (nameGetterResult <= 0) {
+        debugs(83, 3, "ERROR: EVP_PKEY_get_default_digest_name() failure: " << Ssl::ReportAndForgetErrors);
+        // Backward compatibility: On error, assume digest should be used.
+        // TODO: Return false for -2 nameGetterResult as it "indicates the
+        // operation is not supported by the public key algorithm"?
+        return true;
+    }
+
+    // The name "UNDEF" signifies that a digest must (for return value 2) or may
+    // (for return value 1) be left unspecified.
+    if (nameGetterResult == 2 && strcmp(defaultDigestName, "UNDEF") == 0)
+        return false;
+
+    // Defined mandatory algorithms and "may be left unspecified" cases mentioned above.
+    return true;
+
+#else
+    // TODO: Drop this legacy code when we stop supporting OpenSSL v1;
+    // EVP_PKEY_get_default_digest_name() is available starting with OpenSSL v3.
+    (void)key;
+
+    // assume that digest is required for all key types supported by older OpenSSL versions
+    return true;
+#endif // HAVE_LIBCRYPTO_EVP_PKEY_GET_DEFAULT_DIGEST_NAME
+}
+
+/// OpenSSL X509_sign() wrapper
+static auto
+Sign(Security::Certificate &cert, const Security::PrivateKeyPointer &key, const EVP_MD &availableDigest) {
+    const auto digestOrNil = signWithDigest(key) ? &availableDigest : nullptr;
+    return X509_sign(&cert, key.get(), digestOrNil);
+}
+
 void
 Ssl::ForgetErrors()
 {
@@ -677,9 +723,9 @@ static bool generateFakeSslCertificate(Security::CertPointer & certToStore, Secu
     assert(hash);
     /*Now sign the request */
     if (properties.signAlgorithm != Ssl::algSignSelf && properties.signWithPkey.get())
-        ret = X509_sign(cert.get(), properties.signWithPkey.get(), hash);
+        ret = Sign(*cert, properties.signWithPkey, *hash);
     else //else sign with self key (self signed request)
-        ret = X509_sign(cert.get(), pkey.get(), hash);
+        ret = Sign(*cert, pkey, *hash);
 
     if (!ret)
         return false;