From: Ondřej Surý <ondrej@isc.org>
Date: Mon, 10 Aug 2026 14:40:54 +0000 (+0200)
Subject: Check RSA component sizes before conversion
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=343764f9f4f3577cd47aaeb24726e5a5a06f7d3a;p=thirdparty%2Fbind9.git

Check RSA component sizes before conversion

The modulus and exponent size limits were only enforced after
BN_bin2bn() had converted both components into BIGNUMs, so parsing an
attacker-supplied DNSKEY allocated memory proportional to the wire data
before any limit applied.  Bound the byte lengths first: the largest
accepted exponent (2^32 + 1) fits in five octets and a 4096-bit modulus
in 512, so oversized key material is now rejected before conversion.

This also rejects encodings that previously stayed within the bit-size
limits only because of leading zero octets, which RFC 3110 prohibits
anyway.
---

diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index 65a8b481b70..4b4a333d561 100644
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -33,8 +33,10 @@
 #include "dst_parse.h"
 #include "openssl_shim.h"
 
-#define OPENSSLRSA_MAX_MODULUS_BITS 4096
-#define OPENSSLRSA_MIN_MODULUS_BITS 512
+constexpr int rsa_max_modulus_bits = 4096;
+constexpr int rsa_min_modulus_bits = 512;
+constexpr unsigned int rsa_max_modulus_bytes = (rsa_max_modulus_bits + 7) / 8;
+constexpr unsigned int rsa_max_exponent_bytes = 5; /* 2^32 + 1 */
 
 static BIGNUM *rsa_exponent_min = NULL;
 static BIGNUM *rsa_exponent_max = NULL;
@@ -257,9 +259,8 @@ opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 	evp_md_ctx = dctx->ctxdata.evp_md_ctx;
 	pkey = key->keydata.pkeypair.pub;
 
-	if (!isc_ossl_wrap_rsa_modulus_bits_in_range(
-		    pkey, OPENSSLRSA_MIN_MODULUS_BITS,
-		    OPENSSLRSA_MAX_MODULUS_BITS) ||
+	if (!isc_ossl_wrap_rsa_modulus_bits_in_range(pkey, rsa_min_modulus_bits,
+						     rsa_max_modulus_bits) ||
 	    !isc_ossl_wrap_rsa_exponent_is_allowed(pkey))
 	{
 		return DST_R_VERIFYFAILURE;
@@ -484,14 +485,19 @@ opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
 	if (r.length < e_bytes) {
 		CLEANUP(DST_R_INVALIDPUBLICKEY);
 	}
+	if (e_bytes > rsa_max_exponent_bytes ||
+	    r.length - e_bytes > rsa_max_modulus_bytes)
+	{
+		CLEANUP(ISC_R_RANGE);
+	}
 	c.e = BN_bin2bn(r.base, e_bytes, NULL);
 	isc_region_consume(&r, e_bytes);
 	c.n = BN_bin2bn(r.base, r.length, NULL);
 	if (c.e == NULL || c.n == NULL) {
 		CLEANUP(ISC_R_NOMEMORY);
 	}
-	if (BN_num_bits(c.n) < OPENSSLRSA_MIN_MODULUS_BITS ||
-	    BN_num_bits(c.n) > OPENSSLRSA_MAX_MODULUS_BITS)
+	if (BN_num_bits(c.n) < rsa_min_modulus_bits ||
+	    BN_num_bits(c.n) > rsa_max_modulus_bits)
 	{
 		CLEANUP(ISC_R_RANGE);
 	}
@@ -735,8 +741,8 @@ opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 	if (c.n == NULL || c.e == NULL) {
 		CLEANUP(DST_R_INVALIDPRIVATEKEY);
 	}
-	if (BN_num_bits(c.n) < OPENSSLRSA_MIN_MODULUS_BITS ||
-	    BN_num_bits(c.n) > OPENSSLRSA_MAX_MODULUS_BITS)
+	if (BN_num_bits(c.n) < rsa_min_modulus_bits ||
+	    BN_num_bits(c.n) > rsa_max_modulus_bits)
 	{
 		CLEANUP(ISC_R_RANGE);
 	}
@@ -782,8 +788,7 @@ opensslrsa_fromlabel(dst_key_t *key, const char *label, const char *pin) {
 		CLEANUP(ISC_R_RANGE);
 	}
 	if (!isc_ossl_wrap_rsa_modulus_bits_in_range(
-		    pubpkey, OPENSSLRSA_MIN_MODULUS_BITS,
-		    OPENSSLRSA_MAX_MODULUS_BITS))
+		    pubpkey, rsa_min_modulus_bits, rsa_max_modulus_bits))
 	{
 		CLEANUP(ISC_R_RANGE);
 	}