From: Vladimír Čunát Date: Mon, 9 Mar 2020 14:10:24 +0000 (+0100) Subject: kr_zonecut_add(): allow not checking for duplicates X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f48d568220a66b62b645c9409992eaa16174bed8;p=thirdparty%2Fknot-resolver.git kr_zonecut_add(): allow not checking for duplicates Large block-lists in hints will end up in keeping a huge reverse map for some IP address(es), and our simple duplicate-checking algorithm can't handle that efficiently. The duplicate checking in hints seems better removed, so that add-del pairs work well in that case. --- diff --git a/NEWS b/NEWS index db4cd7a28..8f79e1963 100644 --- a/NEWS +++ b/NEWS @@ -4,11 +4,16 @@ Knot Resolver 5.x.y (2020-0m-0d) Improvements ------------ - cache garbage collector: reduce filesystem operations when idle (!946) +- hints module: improve speed for large block-lists (!955) Bugfixes -------- - cache: missing filesystem support for pre-allocation is no longer fatal (#549) +Incompatible changes +-------------------- +- module API: kr_zonecut_add() takes another parameter (!955) + Knot Resolver 5.0.1 (2020-02-05) ================================ diff --git a/daemon/engine.c b/daemon/engine.c index 87c9ffc78..e0907b9cc 100644 --- a/daemon/engine.c +++ b/daemon/engine.c @@ -231,7 +231,7 @@ static void roothints_add(zs_scanner_t *zs) return; } if (zs->r_type == KNOT_RRTYPE_A || zs->r_type == KNOT_RRTYPE_AAAA) { - kr_zonecut_add(hints, zs->r_owner, zs->r_data, zs->r_data_length); + kr_zonecut_add(hints, zs->r_owner, zs->r_data, zs->r_data_length, true); } } const char* engine_hint_root_file(struct kr_context *ctx, const char *file) diff --git a/daemon/lua/kres-gen.lua b/daemon/lua/kres-gen.lua index 973a7cfa0..08395ad12 100644 --- a/daemon/lua/kres-gen.lua +++ b/daemon/lua/kres-gen.lua @@ -358,7 +358,7 @@ int kr_ranked_rrarray_add(ranked_rr_array_t *, const knot_rrset_t *, uint8_t, _B int kr_ranked_rrarray_finalize(ranked_rr_array_t *, uint32_t, knot_mm_t *); void kr_qflags_set(struct kr_qflags *, struct kr_qflags); void kr_qflags_clear(struct kr_qflags *, struct kr_qflags); -int kr_zonecut_add(struct kr_zonecut *, const knot_dname_t *, const void *, int); +int kr_zonecut_add(struct kr_zonecut *, const knot_dname_t *, const void *, int, _Bool); _Bool kr_zonecut_is_empty(struct kr_zonecut *); void kr_zonecut_set(struct kr_zonecut *, const knot_dname_t *); uint64_t kr_now(); diff --git a/lib/layer/iterate.c b/lib/layer/iterate.c index 790b9e530..2f168025b 100644 --- a/lib/layer/iterate.c +++ b/lib/layer/iterate.c @@ -173,7 +173,8 @@ static int update_nsaddr(const knot_rrset_t *rr, struct kr_query *query, int *gl "'%s': '%s'\n", name_str, addr_str); return KR_STATE_CONSUME; /* Ignore invalid addresses */ } - int ret = kr_zonecut_add(&query->zone_cut, rr->owner, rdata->data, rdata->len); + int ret = kr_zonecut_add(&query->zone_cut, rr->owner, rdata->data, + rdata->len, true); if (ret != 0) { return KR_STATE_FAIL; } @@ -302,7 +303,7 @@ static int update_cut(knot_pkt_t *pkt, const knot_rrset_t *rr, } continue; } - int ret = kr_zonecut_add(cut, ns_name, NULL, 0); + int ret = kr_zonecut_add(cut, ns_name, NULL, 0, true); assert(!ret); (void)ret; /* Choose when to use glue records. */ diff --git a/lib/test_zonecut.c b/lib/test_zonecut.c index 0c85b0c69..829248086 100644 --- a/lib/test_zonecut.c +++ b/lib/test_zonecut.c @@ -17,7 +17,7 @@ static void test_zonecut_params(void **state) kr_zonecut_set(NULL, NULL); kr_zonecut_set(&cut, NULL); /* TODO triggerring inner assert: - assert_int_not_equal(kr_zonecut_add(NULL, NULL, NULL, 0), 0); + assert_int_not_equal(kr_zonecut_add(NULL, NULL, NULL, 0, true), 0); */ assert_null((void *)kr_zonecut_find(NULL, NULL)); assert_null((void *)kr_zonecut_find(&cut, NULL)); @@ -35,8 +35,8 @@ static void test_zonecut_copy(void **state) const knot_dname_t *n_1 = (const uint8_t *)"\4dead", *n_2 = (const uint8_t *)"\3bee\1f"; - assert_int_equal(kr_zonecut_add(&cut1, n_1, NULL, 0), 0); - assert_int_equal(kr_zonecut_add(&cut1, n_2, NULL, 0), 0); + assert_int_equal(kr_zonecut_add(&cut1, n_1, NULL, 0, true), 0); + assert_int_equal(kr_zonecut_add(&cut1, n_2, NULL, 0, true), 0); /* Copy */ assert_int_equal(kr_zonecut_copy(&cut2, &cut1), 0); /* Check if exist */ diff --git a/lib/zonecut.c b/lib/zonecut.c index 5839b9743..fc62e3c08 100644 --- a/lib/zonecut.c +++ b/lib/zonecut.c @@ -162,7 +162,8 @@ int kr_zonecut_copy_trust(struct kr_zonecut *dst, const struct kr_zonecut *src) return kr_ok(); } -int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *data, int len) +int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, + const void *data, int len, bool check_dup) { if (!cut || !ns || !cut->nsset || (data && len <= 0)) { assert(!EINVAL); @@ -188,7 +189,7 @@ int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *d return kr_ok(); } /* Check for duplicates */ - if (pack_obj_find(*pack, data, len)) { + if (check_dup && pack_obj_find(*pack, data, len)) { return kr_ok(); } /* Push new address */ diff --git a/lib/zonecut.h b/lib/zonecut.h index 8800e6703..3967e3aa0 100644 --- a/lib/zonecut.h +++ b/lib/zonecut.h @@ -97,10 +97,12 @@ int kr_zonecut_copy_trust(struct kr_zonecut *dst, const struct kr_zonecut *src); * @param ns nameserver name * @param data typically knot_rdata_t::data * @param len typically knot_rdata_t::len + * @param check_dup check for duplicates and return 0 without without adding in that case * @return 0 or error code */ KR_EXPORT -int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *data, int len); +int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, + const void *data, int len, bool check_dup); /** * Delete nameserver/address pair from the zone cut. diff --git a/modules/hints/hints.c b/modules/hints/hints.c index 7473a34c7..bd39e14ec 100644 --- a/modules/hints/hints.c +++ b/modules/hints/hints.c @@ -237,7 +237,7 @@ static int add_pair(struct kr_zonecut *hints, const char *name, const char *addr return kr_error(EINVAL); } - return kr_zonecut_add(hints, key, kr_inaddr(&ia.ip), kr_inaddr_len(&ia.ip)); + return kr_zonecut_add(hints, key, kr_inaddr(&ia.ip), kr_inaddr_len(&ia.ip), false); } static int add_reverse_pair(struct kr_zonecut *hints, const char *name, const char *addr) @@ -253,7 +253,7 @@ static int add_reverse_pair(struct kr_zonecut *hints, const char *name, const ch return kr_error(EINVAL); } - return kr_zonecut_add(hints, key, ptr_name, knot_dname_size(ptr_name)); + return kr_zonecut_add(hints, key, ptr_name, knot_dname_size(ptr_name), false); } /** For a given name, remove either one address or all of them (if == NULL). diff --git a/modules/priming/priming.lua b/modules/priming/priming.lua index c60f25540..79d640cdd 100644 --- a/modules/priming/priming.lua +++ b/modules/priming/priming.lua @@ -23,7 +23,7 @@ local function publish_hints(nsset) for dname, addrsets in pairs(nsset) do for i = 0, addrsets:rdcount() - 1 do local rdpt = addrsets:rdata_pt(i) - ffi.C.kr_zonecut_add(roothints, dname, rdpt.data, rdpt.len) + ffi.C.kr_zonecut_add(roothints, dname, rdpt.data, rdpt.len, true) end end end