- Fix DNSKEY size calculation for noncanonical RSA DNSKEYs

  with leading zeroes for n. Thanks to Xin Wang and Jiajia Liu,
  Northwestern Polytechnical University, for the report.

diff --git a/doc/Changelog b/doc/Changelog
index 862e4ce7f13e5032c79be8f33875cafb1ade8439..42a2192ac275b5e7111235dcc2cbc4907d8bcbd6 100644 (file)
--- a/doc/Changelog
+++ b/doc/Changelog
@@ -18,6 +18,9 @@
          DNSKEYs with leading zeroes for n. Thanks to Xin Wang and
          Jiajia Liu, Northwestern Polytechnical University, for
          the report.
+       - Fix DNSKEY size calculation for noncanonical RSA DNSKEYs
+         with leading zeroes for n. Thanks to Xin Wang and Jiajia Liu,
+         Northwestern Polytechnical University, for the report.
 
 11 May 2026: Yorgos
        - Fix comment and verbose logging for EDNS fallback buffer size.

diff --git a/sldns/keyraw.c b/sldns/keyraw.c
index 42a9262a30da753ed79f2db4a1f289daf6d2bf27..ab5c459147cf1af651018f179206b10fa1daea0d 100644 (file)
--- a/sldns/keyraw.c
+++ b/sldns/keyraw.c
@@ -67,19 +67,28 @@ sldns_rr_dnskey_key_size_raw(const unsigned char* keydata,
        case LDNS_RSASHA512:
 #endif
                if (len > 0) {
+                       size_t nlen, offset;
                        if (keydata[0] == 0) {
                                /* big exponent */
                                if (len > 3) {
                                        memmove(&int16, keydata + 1, 2);
                                        exp = ntohs(int16);
-                                       return (len - exp - 3)*8;
+                                       offset = 3;
                                } else {
                                        return 0;
                                }
                        } else {
                                exp = keydata[0];
-                               return (len-exp-1)*8;
+                               offset = 1;
                        }
+                       if(exp+offset > len)
+                               return 0;
+                       nlen = len - exp - offset;
+                       /* prefixed zeroes mean a smaller value */
+                       while(nlen > 0 &&
+                               keydata[len-nlen] == 0)
+                               nlen--;
+                       return nlen*8;
                } else {
                        return 0;
                }