Refactor OpenSSL RSA exponent bits checking to a helper function

- Make it a separate opensslrsa_check_exponent_bits() function to
  clean up the code a bit
- Always use provider API first if using openssl 3.0, and fallback
  to EVP API for older openssl or if built with engine support
- Use RSA_get0_key() (with shim for openssl 1.0) to avoid memory
  allocations

diff --git a/configure.ac b/configure.ac
index c0a364e54b7284a1ec1b27daa9cf8de30beb0626..4576c64da1447d6c50ca572be887cbbfea868be4 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -726,7 +726,8 @@ AC_CHECK_FUNCS([EVP_aes_128_ecb EVP_aes_192_ecb EVP_aes_256_ecb], [:],
 #
 # Check for OpenSSL 1.1.x/LibreSSL functions
 #
-AC_CHECK_FUNCS([DH_get0_key ECDSA_SIG_get0 RSA_set0_key EVP_PKEY_get0_EC_KEY])
+AC_CHECK_FUNCS([DH_get0_key ECDSA_SIG_get0 EVP_PKEY_get0_EC_KEY])
+AC_CHECK_FUNCS([RSA_set0_key EVP_PKEY_get0_RSA])
 
 AC_CHECK_FUNCS([TLS_server_method TLS_client_method])
 

diff --git a/lib/dns/openssl_shim.h b/lib/dns/openssl_shim.h
index e716b3a7a372abb6bad7a0b2443dadbe6015b360..ad07638c3e5225d269f16e22fe9424397d8414a4 100644 (file)
--- a/lib/dns/openssl_shim.h
+++ b/lib/dns/openssl_shim.h
@@ -17,6 +17,7 @@
 #include <openssl/dh.h>
 #include <openssl/ecdsa.h>
 #include <openssl/err.h>
+#include <openssl/evp.h>
 #include <openssl/opensslv.h>
 #include <openssl/rsa.h>
 
@@ -27,6 +28,13 @@
 #define RSA_MAX_PUBEXP_BITS 35
 #endif /* ifndef RSA_MAX_PUBEXP_BITS */
 
+#if !HAVE_EVP_PKEY_GET0_RSA && OPENSSL_VERSION_NUMBER < 0x10100000L
+static inline const RSA *
+EVP_PKEY_get0_RSA(const EVP_PKEY *pkey) {
+       return (pkey->type == EVP_PKEY_RSA ? pkey->pkey.rsa : NULL);
+}
+#endif
+
 #if !HAVE_RSA_SET0_KEY && OPENSSL_VERSION_NUMBER < 0x30000000L
 int
 RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d);

diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index e6813ed3e1620e1cd514eb202b9fb6641f766080..94f4f4573186d4b107c0924d8d9a4f1816a8c692 100644 (file)
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -17,19 +17,17 @@
 #include <stdbool.h>
 
 #include <openssl/bn.h>
+#include <openssl/err.h>
+#include <openssl/objects.h>
 #include <openssl/opensslv.h>
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_API_LEVEL >= 30000
-#include <openssl/core_names.h>
-#endif
+#include <openssl/rsa.h>
 #if !defined(OPENSSL_NO_ENGINE) && OPENSSL_API_LEVEL < 30000
 #include <openssl/engine.h>
 #endif /* if !defined(OPENSSL_NO_ENGINE) && OPENSSL_API_LEVEL < 30000 */
-#include <openssl/err.h>
-#include <openssl/objects.h>
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_API_LEVEL >= 30000
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+#include <openssl/core_names.h>
 #include <openssl/param_build.h>
 #endif
-#include <openssl/rsa.h>
 
 #include <isc/mem.h>
 #include <isc/result.h>
@@ -177,44 +175,41 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
        return (ISC_R_SUCCESS);
 }
 
+static bool
+opensslrsa_check_exponent_bits(EVP_PKEY *pkey, int maxbits) {
+       /* Always use the new API first with OpenSSL 3.x. */
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+       BIGNUM *e = NULL;
+       if (EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &e) == 1) {
+               int bits = BN_num_bits(e);
+               BN_free(e);
+               return (bits < maxbits);
+       }
+#endif
+       /* Use old API for the OpenSSL ENGINE support, even with OpenSSL 3.x */
+#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
+       const RSA *rsa = EVP_PKEY_get0_RSA(pkey);
+       if (rsa != NULL) {
+               const BIGNUM *ce = NULL;
+               RSA_get0_key(rsa, NULL, &ce, NULL);
+               if (ce != NULL) {
+                       return (BN_num_bits(ce) < maxbits);
+               }
+       }
+#endif
+       return (false);
+}
+
 static isc_result_t
 opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
        dst_key_t *key = dctx->key;
        int status = 0;
-#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
-       RSA *rsa;
-       const BIGNUM *e = NULL;
-#else
-       BIGNUM *e = NULL;
-#endif /* OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000 */
        EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
        EVP_PKEY *pkey = key->keydata.pkey;
-       int bits;
 
        REQUIRE(opensslrsa_valid_key_alg(dctx->key->key_alg));
 
-#if OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000
-       rsa = EVP_PKEY_get1_RSA(pkey);
-       if (rsa == NULL) {
-               return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
-       }
-       RSA_get0_key(rsa, NULL, &e, NULL);
-       if (e == NULL) {
-               RSA_free(rsa);
-               return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-       }
-       bits = BN_num_bits(e);
-       RSA_free(rsa);
-#else
-       EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &e);
-       if (e == NULL) {
-               return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-       }
-       bits = BN_num_bits(e);
-       BN_free(e);
-#endif /* OPENSSL_VERSION_NUMBER < 0x30000000L || OPENSSL_API_LEVEL < 30000 */
-
-       if (bits > maxbits && maxbits != 0) {
+       if (maxbits != 0 && !opensslrsa_check_exponent_bits(pkey, maxbits)) {
                return (DST_R_VERIFYFAILURE);
        }