Properly handle the return value of BN_num_bits()

BN_num_bits() returns 0 when passed NULL and a negative value on
internal error.  The OpenSSL wrappers stored the result in a size_t,
so a 0 return falsely satisfied the bit-length check and a negative
return wrapped to a huge value.  Capture the int return, reject
non-positive values, then compare against the limit.

diff --git a/lib/isc/ossl_wrap/ossl1_1.c b/lib/isc/ossl_wrap/ossl1_1.c
index 4154739611966298572f6071a1608548378e78e8..de325c47c4b6aaf856373bfd9bd37703a06824d4 100644 (file)
--- a/lib/isc/ossl_wrap/ossl1_1.c
+++ b/lib/isc/ossl_wrap/ossl1_1.c
@@ -419,7 +419,6 @@ bool
 isc_ossl_wrap_rsa_key_bits_leq(EVP_PKEY *pkey, size_t limit) {
        const RSA *rsa;
        const BIGNUM *ce;
-       size_t bits = SIZE_MAX;
 
        REQUIRE(pkey != NULL);
 
@@ -428,11 +427,13 @@ isc_ossl_wrap_rsa_key_bits_leq(EVP_PKEY *pkey, size_t limit) {
                ce = NULL;
                RSA_get0_key(rsa, NULL, &ce, NULL);
                if (ce != NULL) {
-                       bits = BN_num_bits(ce);
+                       int bits = BN_num_bits(ce);
+
+                       return bits > 0 && (size_t)bits <= limit;
                }
        }
 
-       return bits <= limit;
+       return false;
 }
 
 isc_result_t

diff --git a/lib/isc/ossl_wrap/ossl3.c b/lib/isc/ossl_wrap/ossl3.c
index b8da16e8e03086b7bd6c73bd7bea870b32f29ff9..92334b2d1cb9a0ca96e200a0e8fa762d5b16cd24 100644 (file)
--- a/lib/isc/ossl_wrap/ossl3.c
+++ b/lib/isc/ossl_wrap/ossl3.c
@@ -572,13 +572,14 @@ cleanup:
 
 bool
 isc_ossl_wrap_rsa_key_bits_leq(EVP_PKEY *pkey, size_t limit) {
-       size_t bits = SIZE_MAX;
        BIGNUM *e = NULL;
        if (EVP_PKEY_get_bn_param(pkey, OSSL_PKEY_PARAM_RSA_E, &e) == 1) {
-               bits = BN_num_bits(e);
+               int bits = BN_num_bits(e);
                BN_free(e);
+
+               return bits > 0 && (size_t)bits <= limit;
        }
-       return bits <= limit;
+       return false;
 }
 
 isc_result_t