From: Willem Toorop Date: Mon, 19 Aug 2013 20:55:30 +0000 (+0200) Subject: ldns-verify-zone NSEC3 checking optimization X-Git-Tag: release-1.6.17rc1~71 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7a1f71d2d2a7794544d05f6b9eac551fa5cde641;p=thirdparty%2Fldns.git ldns-verify-zone NSEC3 checking optimization from NIC.MX --- diff --git a/Changelog b/Changelog index 3de96964..9a9104bc 100644 --- a/Changelog +++ b/Changelog @@ -33,6 +33,8 @@ file pointer. * Fix memory leak in contrib/python: ldns_pkt.new_query. * Fix buffer overflow in fget_token and bget_token. + * ldns-verify-zone NSEC3 checking from quadratic to linear performance. + Thanks NIC MX (nicmexico.mx) 1.6.16 2012-11-13 * Fix Makefile to build pyldns with BSD make diff --git a/dnssec_sign.c b/dnssec_sign.c index abce4852..6b479d8d 100644 --- a/dnssec_sign.c +++ b/dnssec_sign.c @@ -771,6 +771,12 @@ ldns_dnssec_zone_create_nsecs(ldns_dnssec_zone *zone, } #ifdef HAVE_SSL +static void +ldns_hashed_names_node_free(ldns_rbnode_t *node, void *arg) { + (void) arg; + LDNS_FREE(node); +} + static ldns_status ldns_dnssec_zone_create_nsec3s_mkmap(ldns_dnssec_zone *zone, ldns_rr_list *new_rrs, @@ -810,21 +816,24 @@ ldns_dnssec_zone_create_nsec3s_mkmap(ldns_dnssec_zone *zone, nsec_ttl = LDNS_DEFAULT_TTL; } - if (map) { - if ((*map = ldns_rbtree_create(ldns_dname_compare_v)) - == NULL) { - map = NULL; - }; + if (zone->hashed_names) { + ldns_traverse_postorder(zone->hashed_names, + ldns_hashed_names_node_free, NULL); + LDNS_FREE(zone->hashed_names); + } + zone->hashed_names = ldns_rbtree_create(ldns_dname_compare_v); + if (zone->hashed_names && map) { + *map = zone->hashed_names; } - nsec3_list = ldns_rr_list_new(); first_name_node = ldns_dnssec_name_node_next_nonglue( ldns_rbtree_first(zone->names)); current_name_node = first_name_node; - while (current_name_node && - current_name_node != LDNS_RBTREE_NULL) { + while (current_name_node && current_name_node != LDNS_RBTREE_NULL && + result == LDNS_STATUS_OK) { + current_name = (ldns_dnssec_name *) current_name_node->data; nsec_rr = ldns_dnssec_create_nsec3(current_name, NULL, @@ -842,28 +851,49 @@ ldns_dnssec_zone_create_nsec3s_mkmap(ldns_dnssec_zone *zone, ldns_rr_set_ttl(nsec_rr, nsec_ttl); result = ldns_dnssec_name_add_rr(current_name, nsec_rr); ldns_rr_list_push_rr(new_rrs, nsec_rr); - ldns_rr_list_push_rr(nsec3_list, nsec_rr); - if (map) { + if (ldns_rr_owner(nsec_rr)) { hashmap_node = LDNS_MALLOC(ldns_rbnode_t); - if (hashmap_node && ldns_rr_owner(nsec_rr)) { - hashmap_node->key = ldns_dname_label( - ldns_rr_owner(nsec_rr), 0); - if (hashmap_node->key) { - hashmap_node->data = current_name->name; - (void) ldns_rbtree_insert( - *map, hashmap_node); - } + if (hashmap_node == NULL) { + return LDNS_STATUS_MEM_ERR; + } + current_name->hashed_name = + ldns_dname_label(ldns_rr_owner(nsec_rr), 0); + + if (current_name->hashed_name == NULL) { + LDNS_FREE(hashmap_node); + return LDNS_STATUS_MEM_ERR; + } + hashmap_node->key = current_name->hashed_name; + hashmap_node->data = current_name; + + if (! ldns_rbtree_insert(zone->hashed_names + , hashmap_node)) { + LDNS_FREE(hashmap_node); } } current_name_node = ldns_dnssec_name_node_next_nonglue( ldns_rbtree_next(current_name_node)); } if (result != LDNS_STATUS_OK) { - ldns_rr_list_free(nsec3_list); return result; } - ldns_rr_list_sort_nsec3(nsec3_list); + /* Make sorted list of nsec3s (via zone->hashed_names) + */ + nsec3_list = ldns_rr_list_new(); + if (nsec3_list == NULL) { + return LDNS_STATUS_MEM_ERR; + } + for ( hashmap_node = ldns_rbtree_first(zone->hashed_names) + ; hashmap_node != LDNS_RBTREE_NULL + ; hashmap_node = ldns_rbtree_next(hashmap_node) + ) { + current_name = (ldns_dnssec_name *) hashmap_node->data; + nsec_rr = ((ldns_dnssec_name *) hashmap_node->data)->nsec; + if (nsec_rr) { + ldns_rr_list_push_rr(nsec3_list, nsec_rr); + } + } result = ldns_dnssec_chain_nsec3_list(nsec3_list); ldns_rr_list_free(nsec3_list); diff --git a/dnssec_zone.c b/dnssec_zone.c index 11053b27..48f9c2ce 100644 --- a/dnssec_zone.c +++ b/dnssec_zone.c @@ -450,8 +450,6 @@ ldns_dnssec_name_add_rr(ldns_dnssec_name *name, ldns_rr *rr) { ldns_status result = LDNS_STATUS_OK; - ldns_rdf *name_name; - bool hashed_name = false; ldns_rr_type rr_type; ldns_rr_type typecovered = 0; @@ -467,19 +465,6 @@ ldns_dnssec_name_add_rr(ldns_dnssec_name *name, typecovered = ldns_rdf2rr_type(ldns_rr_rrsig_typecovered(rr)); } -#ifdef HAVE_SSL - if (rr_type == LDNS_RR_TYPE_NSEC3 || - typecovered == LDNS_RR_TYPE_NSEC3) { - name_name = ldns_nsec3_hash_name_frm_nsec3(rr, - ldns_dnssec_name_name(name)); - hashed_name = true; - } else { - name_name = ldns_dnssec_name_name(name); - } -#else - name_name = ldns_dnssec_name_name(name); -#endif /* HAVE_SSL */ - if (rr_type == LDNS_RR_TYPE_NSEC || rr_type == LDNS_RR_TYPE_NSEC3) { /* XX check if is already set (and error?) */ @@ -501,11 +486,6 @@ ldns_dnssec_name_add_rr(ldns_dnssec_name *name, result = ldns_dnssec_rrsets_add_rr(name->rrsets, rr); } } - - if (hashed_name) { - ldns_rdf_deep_free(name_name); - } - return result; } @@ -593,6 +573,8 @@ ldns_dnssec_zone_new(void) if(!zone) return NULL; zone->soa = NULL; zone->names = NULL; + zone->hashed_names = NULL; + zone->_nsec3params = NULL; return zone; } @@ -800,31 +782,99 @@ ldns_dname_compare_v(const void *a, const void *b) { return ldns_dname_compare((ldns_rdf *)a, (ldns_rdf *)b); } -static ldns_rbnode_t * -ldns_dnssec_zone_find_nsec3_original(ldns_dnssec_zone *zone, - ldns_rr *rr) { - ldns_rbnode_t *current_node = ldns_rbtree_first(zone->names); - ldns_dnssec_name *current_name; - ldns_rdf *hashed_name; +static void +ldns_dnssec_name_make_hashed_name(ldns_dnssec_zone *zone, + ldns_dnssec_name* name, ldns_rr* nsec3rr); - hashed_name = ldns_dname_label(ldns_rr_owner(rr), 0); +static void +ldns_hashed_names_node_free(ldns_rbnode_t *node, void *arg) { + (void) arg; + LDNS_FREE(node); +} + +static void +ldns_dnssec_zone_hashed_names_from_nsec3( + ldns_dnssec_zone* zone, ldns_rr* nsec3rr) +{ + ldns_rbnode_t* current_node; + ldns_dnssec_name* current_name; - while (current_node != LDNS_RBTREE_NULL) { + assert(zone); + assert(nsec3rr); + + if (zone->hashed_names) { + ldns_traverse_postorder(zone->hashed_names, + ldns_hashed_names_node_free, NULL); + LDNS_FREE(zone->hashed_names); + } + zone->_nsec3params = nsec3rr; + + /* So this is a NSEC3 zone. + * Calculate hashes for all names already in the zone + */ + zone->hashed_names = ldns_rbtree_create(ldns_dname_compare_v); + if (zone->hashed_names == NULL) { + return; + } + for ( current_node = ldns_rbtree_first(zone->names) + ; current_node != LDNS_RBTREE_NULL + ; current_node = ldns_rbtree_next(current_node) + ) { current_name = (ldns_dnssec_name *) current_node->data; - if (!current_name->hashed_name) { - current_name->hashed_name = - ldns_nsec3_hash_name_frm_nsec3(rr, current_name->name); + ldns_dnssec_name_make_hashed_name(zone, current_name, nsec3rr); + + } +} + +static void +ldns_dnssec_name_make_hashed_name(ldns_dnssec_zone *zone, + ldns_dnssec_name* name, ldns_rr* nsec3rr) +{ + ldns_rbnode_t* new_node; + + assert(name); + if (! zone->_nsec3params) { + if (! nsec3rr) { + return; } - if (ldns_dname_compare(hashed_name, - current_name->hashed_name) - == 0) { - ldns_rdf_deep_free(hashed_name); - return current_node; + ldns_dnssec_zone_hashed_names_from_nsec3(zone, nsec3rr); + + } else if (! nsec3rr) { + nsec3rr = zone->_nsec3params; + } + name->hashed_name = ldns_nsec3_hash_name_frm_nsec3(nsec3rr, name->name); + + /* Also store in zone->hashed_names */ + if ((new_node = LDNS_MALLOC(ldns_rbnode_t))) { + + new_node->key = name->hashed_name; + new_node->data = name; + + if (ldns_rbtree_insert(zone->hashed_names, new_node) == NULL) { + + LDNS_FREE(new_node); } - current_node = ldns_rbtree_next(current_node); } - ldns_rdf_deep_free(hashed_name); - return NULL; +} + + +static ldns_rbnode_t * +ldns_dnssec_zone_find_nsec3_original(ldns_dnssec_zone *zone, ldns_rr *rr) { + ldns_rdf *hashed_name; + + hashed_name = ldns_dname_label(ldns_rr_owner(rr), 0); + if (hashed_name == NULL) { + return NULL; + } + if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_NSEC3 && ! zone->_nsec3params){ + + ldns_dnssec_zone_hashed_names_from_nsec3(zone, rr); + } + if (zone->hashed_names == NULL) { + ldns_rdf_deep_free(hashed_name); + return NULL; + } + return ldns_rbtree_search(zone->hashed_names, hashed_name); } ldns_status @@ -851,15 +901,13 @@ ldns_dnssec_zone_add_rr(ldns_dnssec_zone *zone, ldns_rr *rr) } if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_NSEC3 || type_covered == LDNS_RR_TYPE_NSEC3) { - cur_node = ldns_dnssec_zone_find_nsec3_original(zone, - rr); + cur_node = ldns_dnssec_zone_find_nsec3_original(zone, rr); if (!cur_node) { return LDNS_STATUS_DNSSEC_NSEC3_ORIGINAL_NOT_FOUND; } } else { cur_node = ldns_rbtree_search(zone->names, ldns_rr_owner(rr)); } - if (!cur_node) { /* add */ cur_name = ldns_dnssec_name_new_frm_rr(rr); @@ -872,21 +920,14 @@ ldns_dnssec_zone_add_rr(ldns_dnssec_zone *zone, ldns_rr *rr) cur_node->key = ldns_rr_owner(rr); cur_node->data = cur_name; (void)ldns_rbtree_insert(zone->names, cur_node); + ldns_dnssec_name_make_hashed_name(zone, cur_name, NULL); } else { cur_name = (ldns_dnssec_name *) cur_node->data; result = ldns_dnssec_name_add_rr(cur_name, rr); } - - if (result != LDNS_STATUS_OK) { - fprintf(stderr, "error adding rr: "); - ldns_rr_print(stderr, rr); - } - - /*TODO ldns_dnssec_name_print_names(stdout, zone->names, 0);*/ if (ldns_rr_get_type(rr) == LDNS_RR_TYPE_SOA) { zone->soa = cur_name; } - return result; } @@ -1032,6 +1073,8 @@ ldns_dnssec_zone_add_empty_nonterminals(ldns_dnssec_zone *zone) new_node->key = new_name->name; new_node->data = new_name; (void)ldns_rbtree_insert(zone->names, new_node); + ldns_dnssec_name_make_hashed_name( + zone, new_name, NULL); } ldns_rdf_deep_free(l1); ldns_rdf_deep_free(l2); diff --git a/examples/ldns-verify-zone.c b/examples/ldns-verify-zone.c index 0bbb97f4..09e6fe6f 100644 --- a/examples/ldns-verify-zone.c +++ b/examples/ldns-verify-zone.c @@ -11,6 +11,7 @@ #include "config.h" #include #include +#include #include @@ -245,74 +246,25 @@ verify_next_hashed_name(ldns_dnssec_zone* zone, ldns_dnssec_name *name) { ldns_rbnode_t *next_node; ldns_dnssec_name *next_name; - ldns_dnssec_name *cur_next_name = NULL; - ldns_dnssec_name *cur_first_name = NULL; int cmp; char *next_owner_str; ldns_rdf *next_owner_dname; - if (!name->hashed_name) { - name->hashed_name = ldns_nsec3_hash_name_frm_nsec3( - name->nsec, name->name); - } - next_node = ldns_rbtree_first(zone->names); - while (next_node != LDNS_RBTREE_NULL) { - next_name = (ldns_dnssec_name *)next_node->data; - /* skip over names that have no NSEC3 records (whether it - * actually should or should not should have been checked - * already */ - if (!next_name->nsec) { - next_node = ldns_rbtree_next(next_node); - continue; - } - if (!next_name->hashed_name) { - next_name->hashed_name = - ldns_nsec3_hash_name_frm_nsec3(name->nsec, - next_name->name); - } - /* we keep track of what 'so far' is the next hashed name; - * it must of course be 'larger' than the current name - * if we find one that is larger, but smaller than what we - * previously thought was the next one, that one is the next - */ - cmp = ldns_dname_compare(name->hashed_name, - next_name->hashed_name); - if (cmp < 0) { - if (!cur_next_name) { - cur_next_name = next_name; - } else { - cmp = ldns_dname_compare( - next_name->hashed_name, - cur_next_name->hashed_name); - if (cmp < 0) { - cur_next_name = next_name; - } - } - } - /* in case the hashed name of the nsec we are checking is the - * last one, we need the first hashed name of the zone */ - if (!cur_first_name) { - cur_first_name = next_name; - } else { - cmp = ldns_dname_compare(next_name->hashed_name, - cur_first_name->hashed_name); - if (cmp < 0) { - cur_first_name = next_name; - } - } + assert(name->hashed_name); + + next_node = ldns_rbtree_search(zone->hashed_names, name->hashed_name); + assert(next_node); + do { next_node = ldns_rbtree_next(next_node); - } - if (!cur_next_name) { - cur_next_name = cur_first_name; - } - assert(cur_next_name != NULL); - /* Because this function is called on nsec occurrence, - * there must be a cur_next_name! - */ + if (next_node == LDNS_RBTREE_NULL) { + next_node = ldns_rbtree_first(zone->hashed_names); + } + next_name = (ldns_dnssec_name *) next_node->data; + } while (! next_name->nsec); next_owner_str = ldns_rdf2str(ldns_nsec3_next_owner(name->nsec)); next_owner_dname = ldns_dname_new_frm_str(next_owner_str); - cmp = ldns_dname_compare(next_owner_dname, cur_next_name->hashed_name); + cmp = ldns_dname_compare(next_owner_dname, next_name->hashed_name); ldns_rdf_deep_free(next_owner_dname); LDNS_FREE(next_owner_str); if (cmp != 0) { @@ -321,9 +273,9 @@ verify_next_hashed_name(ldns_dnssec_zone* zone, ldns_dnssec_name *name) ldns_rdf_print(stdout, name->name); fprintf(myerr, " points to the wrong next hashed owner" " name\n\tshould point to "); - ldns_rdf_print(myerr, cur_next_name->name); + ldns_rdf_print(myerr, next_name->name); fprintf(myerr, ", whose hashed name is "); - ldns_rdf_print(myerr, cur_next_name->hashed_name); + ldns_rdf_print(myerr, next_name->hashed_name); fprintf(myerr, "\n"); } return LDNS_STATUS_ERR; @@ -458,7 +410,7 @@ verify_dnssec_name(ldns_rdf *zone_name, ldns_dnssec_zone* zone, /* for NSEC chain checks */ name = (ldns_dnssec_name *) cur_node->data; - if (verbosity >= 3) { + if (verbosity >= 5) { fprintf(myout, "Checking: "); ldns_rdf_print(myout, name->name); fprintf(myout, "\n"); @@ -703,6 +655,23 @@ error: return result; } +void timelog(const char* msg) /* DEBUGING (remove me) */ +{ + static struct timeval start; + static double dstart; + struct timeval now; + double dnow; + + if (! msg) { + gettimeofday(&start, NULL); + dstart = start.tv_sec + ((double)start.tv_usec / 1000000); + } else if (verbosity > 3) { + gettimeofday(&now, NULL); + dnow = now.tv_sec + ((double)now.tv_usec / 1000000); + fprintf(myout, "%10.6f %s\n", (dnow - dstart), msg); + } +} + int main(int argc, char **argv) { @@ -720,6 +689,7 @@ main(int argc, char **argv) ldns_rr_list *keys = ldns_rr_list_new(); size_t nkeys = 0; + timelog(NULL); /* DEBUGING (remove me) */ check_time = ldns_time(NULL); myout = stdout; myerr = stderr; @@ -897,6 +867,7 @@ main(int argc, char **argv) s = ldns_dnssec_zone_new_frm_fp_l(&dnssec_zone, fp, NULL, 0, LDNS_RR_CLASS_IN, &line_nr); if (s == LDNS_STATUS_OK) { + timelog("zone loaded"); /* DEBUGING (remove me) */ if (!dnssec_zone->soa) { if (verbosity > 0) { fprintf(myerr, @@ -913,7 +884,7 @@ main(int argc, char **argv) "glue in the zone\n"); } } - + timelog("glue marked"); if (verbosity >= 5) { ldns_dnssec_zone_print(myout, dnssec_zone); } @@ -921,6 +892,7 @@ main(int argc, char **argv) result = verify_dnssec_zone(dnssec_zone, dnssec_zone->soa->name, keys, apexonly, percentage); + timelog("zone verified"); if (result == LDNS_STATUS_OK) { if (verbosity >= 3) { diff --git a/host2str.c b/host2str.c index a79df2c2..0a3c73b3 100644 --- a/host2str.c +++ b/host2str.c @@ -1603,7 +1603,10 @@ ldns_rr2buffer_str_fmt(ldns_buffer *output, "from: "); (void) ldns_rdf2buffer_str( output, - (ldns_rdf*)node->data); + ldns_dnssec_name_name( + (ldns_dnssec_name*) + node->data + )); } ldns_rdf_free(key); } @@ -1617,8 +1620,11 @@ ldns_rr2buffer_str_fmt(ldns_buffer *output, ldns_buffer_printf(output, " to: "); (void) ldns_rdf2buffer_str( - output, - (ldns_rdf*)node->data); + output, + ldns_dnssec_name_name( + (ldns_dnssec_name*) + node->data + )); } ldns_rdf_free(key); } diff --git a/ldns/dnssec.h b/ldns/dnssec.h index 3e992c68..9cd22f13 100644 --- a/ldns/dnssec.h +++ b/ldns/dnssec.h @@ -387,7 +387,7 @@ ldns_status ldns_nsec_bitmap_set_type(ldns_rdf* bitmap, ldns_rr_type type); * \return LDNS_STATUS_OK on success. LDNS_STATUS_TYPE_NOT_IN_BITMAP is * returned when the bitmap does not contain the bit to set. */ -ldns_status ldns_nsec_bitmap_clear_type(ldns_rdf* rdf, ldns_rr_type t); +ldns_status ldns_nsec_bitmap_clear_type(ldns_rdf* bitmap, ldns_rr_type type); /** * Checks coverage of NSEC(3) RR name span diff --git a/ldns/dnssec_zone.h b/ldns/dnssec_zone.h index 257bfba2..42947355 100644 --- a/ldns/dnssec_zone.h +++ b/ldns/dnssec_zone.h @@ -93,6 +93,13 @@ struct ldns_struct_dnssec_zone { ldns_dnssec_name *soa; /** tree of ldns_dnssec_names */ ldns_rbtree_t *names; + /** tree of ldns_dnssec_names by nsec3 hashes (when applicible) */ + ldns_rbtree_t *hashed_names; + /** points to the first added NSEC3 rr whose parameters will be + * assumed for all subsequent NSEC3 rr's and which will be used + * to calculate hashed names + */ + ldns_rr *_nsec3params; }; typedef struct ldns_struct_dnssec_zone ldns_dnssec_zone; diff --git a/ldns/host2str.h b/ldns/host2str.h index 4c7d1452..724d2cbc 100644 --- a/ldns/host2str.h +++ b/ldns/host2str.h @@ -99,7 +99,7 @@ typedef struct ldns_struct_output_format ldns_output_format; */ struct ldns_struct_output_format_storage { int flags; - ldns_rbtree_t* hashmap; /* for LDNS_FMT_NSEC3_CHAIN */ + ldns_rbtree_t* hashmap; /* for LDNS_COMMENT_NSEC3_CHAIN */ ldns_rdf* bitmap; /* for LDNS_FMT_RFC3597 */ }; typedef struct ldns_struct_output_format_storage ldns_output_format_storage; diff --git a/test/test_all.sh b/test/test_all.sh index 37cda999..b4593af2 100755 --- a/test/test_all.sh +++ b/test/test_all.sh @@ -19,12 +19,13 @@ test_tool_avail "dig" echo start the test at `date` in `pwd` $TPKG clean $TPKG -a ../.. fake 01-compile.tpkg +$TPKG -a ../.. fake 02-lint.tpkg # Works only on FreeBSD really $TPKG -a ../.. fake 07-compile-examples.tpkg $TPKG -a ../.. fake 16-compile-builddir.tpkg $TPKG -a ../.. fake 30-load-pyldns.tpkg $TPKG -a ../.. fake 31-load-pyldnsx.tpkg -$TPKG -a ../.. fake 32-unbound-1.4.20-regression.tpkg -$TPKG -a ../.. fake 33-wget-compile-test-unbound-latest.tpkg +$TPKG -a ../.. fake 32-unbound-regression.tpkg +$TPKG -a ../.. fake 33-test-unbound-latest.tpkg $TPKG -a ../.. fake 999-compile-nossl.tpkg for tests in *.tpkg