Check RSA component sizes before conversion

The modulus and exponent size limits were only enforced after
BN_bin2bn() had converted both components into BIGNUMs, so parsing an
attacker-supplied DNSKEY allocated memory proportional to the wire data
before any limit applied.  Bound the byte lengths first: the largest
accepted exponent (2^32 + 1) fits in five octets and a 4096-bit modulus
in 512, so oversized key material is now rejected before conversion.

This also rejects encodings that previously stayed within the bit-size
limits only because of leading zero octets, which RFC 3110 prohibits
anyway.

diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index 65a8b481b708401ace34a5f4b5026c803f151d13..4b4a333d56175a575f93669270830fcdca1759f5 100644 (file)
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -33,8 +33,10 @@
 #include "dst_parse.h"
 #include "openssl_shim.h"
 
-#define OPENSSLRSA_MAX_MODULUS_BITS 4096
-#define OPENSSLRSA_MIN_MODULUS_BITS 512
+constexpr int rsa_max_modulus_bits = 4096;
+constexpr int rsa_min_modulus_bits = 512;
+constexpr unsigned int rsa_max_modulus_bytes = (rsa_max_modulus_bits + 7) / 8;
+constexpr unsigned int rsa_max_exponent_bytes = 5; /* 2^32 + 1 */
 
 static BIGNUM *rsa_exponent_min = NULL;
 static BIGNUM *rsa_exponent_max = NULL;
@@ -257,9 +259,8 @@ opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
        evp_md_ctx = dctx->ctxdata.evp_md_ctx;
        pkey = key->keydata.pkeypair.pub;
 
-       if (!isc_ossl_wrap_rsa_modulus_bits_in_range(
-                   pkey, OPENSSLRSA_MIN_MODULUS_BITS,
-                   OPENSSLRSA_MAX_MODULUS_BITS) ||
+       if (!isc_ossl_wrap_rsa_modulus_bits_in_range(pkey, rsa_min_modulus_bits,
+                                                    rsa_max_modulus_bits) ||
            !isc_ossl_wrap_rsa_exponent_is_allowed(pkey))
        {
                return DST_R_VERIFYFAILURE;
@@ -484,14 +485,19 @@ opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
        if (r.length < e_bytes) {
                CLEANUP(DST_R_INVALIDPUBLICKEY);
        }
+       if (e_bytes > rsa_max_exponent_bytes ||
+           r.length - e_bytes > rsa_max_modulus_bytes)
+       {
+               CLEANUP(ISC_R_RANGE);
+       }
        c.e = BN_bin2bn(r.base, e_bytes, NULL);
        isc_region_consume(&r, e_bytes);
        c.n = BN_bin2bn(r.base, r.length, NULL);
        if (c.e == NULL || c.n == NULL) {
                CLEANUP(ISC_R_NOMEMORY);
        }
-       if (BN_num_bits(c.n) < OPENSSLRSA_MIN_MODULUS_BITS ||
-           BN_num_bits(c.n) > OPENSSLRSA_MAX_MODULUS_BITS)
+       if (BN_num_bits(c.n) < rsa_min_modulus_bits ||
+           BN_num_bits(c.n) > rsa_max_modulus_bits)
        {
                CLEANUP(ISC_R_RANGE);
        }
@@ -735,8 +741,8 @@ opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
        if (c.n == NULL || c.e == NULL) {
                CLEANUP(DST_R_INVALIDPRIVATEKEY);
        }
-       if (BN_num_bits(c.n) < OPENSSLRSA_MIN_MODULUS_BITS ||
-           BN_num_bits(c.n) > OPENSSLRSA_MAX_MODULUS_BITS)
+       if (BN_num_bits(c.n) < rsa_min_modulus_bits ||
+           BN_num_bits(c.n) > rsa_max_modulus_bits)
        {
                CLEANUP(ISC_R_RANGE);
        }
@@ -782,8 +788,7 @@ opensslrsa_fromlabel(dst_key_t *key, const char *label, const char *pin) {
                CLEANUP(ISC_R_RANGE);
        }
        if (!isc_ossl_wrap_rsa_modulus_bits_in_range(
-                   pubpkey, OPENSSLRSA_MIN_MODULUS_BITS,
-                   OPENSSLRSA_MAX_MODULUS_BITS))
+                   pubpkey, rsa_min_modulus_bits, rsa_max_modulus_bits))
        {
                CLEANUP(ISC_R_RANGE);
        }