validator: refuse to validate answers with more than 8 NSEC3 records

(cherry picked from commit a05cf1d379d1af0958587bd111f791b72f404364)
This partially mitigates CVE-2023-50868 (release 5.7.1 is better)

diff --git a/lib/layer/validate.c b/lib/layer/validate.c
index 92ddb5a9ddb5ded3b0fe3fb36b9573ab21641fce..06758f0fef0c2f2439f6eb71f1c7af01e6c63cbf 100644 (file)
--- a/lib/layer/validate.c
+++ b/lib/layer/validate.c
@@ -1090,6 +1090,21 @@ static int validate(kr_layer_t *ctx, knot_pkt_t *pkt)
                }
        }
 
+       /* Check for too many NSEC3 records.  That's an issue, as some parts of validation
+        * are quadratic in their count, doing nontrivial computations inside.
+        * Also there seems to be no use in sending many NSEC3 records. */
+       if (!qry->flags.CACHED) {
+               const knot_pktsection_t *sec = knot_pkt_section(pkt, KNOT_AUTHORITY);
+               int count = 0;
+               for (int i = 0; i < sec->count; ++i)
+                       count += (knot_pkt_rr(sec, i)->type == KNOT_RRTYPE_NSEC3);
+               if (count > 8) {
+                       VERBOSE_MSG(qry, "<= too many NSEC3 records in AUTHORITY (%d)\n", count);
+                       qry->flags.DNSSEC_BOGUS = true;
+                       return KR_STATE_FAIL;
+               }
+       }
+
        if (knot_wire_get_aa(pkt->wire) && qtype == KNOT_RRTYPE_DNSKEY) {
                const knot_rrset_t *ds = qry->zone_cut.trust_anchor;
                if (ds && !kr_ds_algo_support(ds)) {