kr_zonecut_add(): allow not checking for duplicates

Large block-lists in hints will end up in keeping a huge reverse map
for some IP address(es), and our simple duplicate-checking algorithm
can't handle that efficiently.  The duplicate checking in hints seems
better removed, so that add-del pairs work well in that case.

diff --git a/NEWS b/NEWS
index db4cd7a2803f41ffc6b7b16a3d4a4d3176df5f5c..8f79e19631fd1cd27d72988113dc625531c350ce 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -4,11 +4,16 @@ Knot Resolver 5.x.y (2020-0m-0d)
 Improvements
 ------------
 - cache garbage collector: reduce filesystem operations when idle (!946)
+- hints module: improve speed for large block-lists (!955)
 
 Bugfixes
 --------
 - cache: missing filesystem support for pre-allocation is no longer fatal (#549)
 
+Incompatible changes
+--------------------
+- module API: kr_zonecut_add() takes another parameter (!955)
+
 
 Knot Resolver 5.0.1 (2020-02-05)
 ================================

diff --git a/daemon/engine.c b/daemon/engine.c
index 87c9ffc788455615920f17239e1a47d53c15bab6..e0907b9cc8a54a223994d36a3a941d1ffcc32d05 100644 (file)
--- a/daemon/engine.c
+++ b/daemon/engine.c
@@ -231,7 +231,7 @@ static void roothints_add(zs_scanner_t *zs)
                return;
        }
        if (zs->r_type == KNOT_RRTYPE_A || zs->r_type == KNOT_RRTYPE_AAAA) {
-               kr_zonecut_add(hints, zs->r_owner, zs->r_data, zs->r_data_length);
+               kr_zonecut_add(hints, zs->r_owner, zs->r_data, zs->r_data_length, true);
        }
 }
 const char* engine_hint_root_file(struct kr_context *ctx, const char *file)

diff --git a/daemon/lua/kres-gen.lua b/daemon/lua/kres-gen.lua
index 973a7cfa013268102ab0e9cca053d6982f420445..08395ad12a1b3135d4fd80f19b1e6e331600908f 100644 (file)
--- a/daemon/lua/kres-gen.lua
+++ b/daemon/lua/kres-gen.lua
@@ -358,7 +358,7 @@ int kr_ranked_rrarray_add(ranked_rr_array_t *, const knot_rrset_t *, uint8_t, _B
 int kr_ranked_rrarray_finalize(ranked_rr_array_t *, uint32_t, knot_mm_t *);
 void kr_qflags_set(struct kr_qflags *, struct kr_qflags);
 void kr_qflags_clear(struct kr_qflags *, struct kr_qflags);
-int kr_zonecut_add(struct kr_zonecut *, const knot_dname_t *, const void *, int);
+int kr_zonecut_add(struct kr_zonecut *, const knot_dname_t *, const void *, int, _Bool);
 _Bool kr_zonecut_is_empty(struct kr_zonecut *);
 void kr_zonecut_set(struct kr_zonecut *, const knot_dname_t *);
 uint64_t kr_now();

diff --git a/lib/layer/iterate.c b/lib/layer/iterate.c
index 790b9e5300f09dcaaf96e6c73a3a41f0177d8bba..2f168025bb1fa2b2e4593601a858506c813329c9 100644 (file)
--- a/lib/layer/iterate.c
+++ b/lib/layer/iterate.c
@@ -173,7 +173,8 @@ static int update_nsaddr(const knot_rrset_t *rr, struct kr_query *query, int *gl
                                     "'%s': '%s'\n", name_str, addr_str);
                        return KR_STATE_CONSUME; /* Ignore invalid addresses */
                }
-               int ret = kr_zonecut_add(&query->zone_cut, rr->owner, rdata->data, rdata->len);
+               int ret = kr_zonecut_add(&query->zone_cut, rr->owner, rdata->data,
+                                        rdata->len, true);
                if (ret != 0) {
                        return KR_STATE_FAIL;
                }
@@ -302,7 +303,7 @@ static int update_cut(knot_pkt_t *pkt, const knot_rrset_t *rr,
                        }
                        continue;
                }
-               int ret = kr_zonecut_add(cut, ns_name, NULL, 0);
+               int ret = kr_zonecut_add(cut, ns_name, NULL, 0, true);
                assert(!ret); (void)ret;
 
                /* Choose when to use glue records. */

diff --git a/lib/test_zonecut.c b/lib/test_zonecut.c
index 0c85b0c6928e708fdba3e523137196fe7c8d41cb..829248086fd01a74cb496a8a896df1918e4dca63 100644 (file)
--- a/lib/test_zonecut.c
+++ b/lib/test_zonecut.c
@@ -17,7 +17,7 @@ static void test_zonecut_params(void **state)
        kr_zonecut_set(NULL, NULL);
        kr_zonecut_set(&cut, NULL);
        /* TODO triggerring inner assert:
-       assert_int_not_equal(kr_zonecut_add(NULL, NULL, NULL, 0), 0);
+       assert_int_not_equal(kr_zonecut_add(NULL, NULL, NULL, 0, true), 0);
        */
        assert_null((void *)kr_zonecut_find(NULL, NULL));
        assert_null((void *)kr_zonecut_find(&cut, NULL));
@@ -35,8 +35,8 @@ static void test_zonecut_copy(void **state)
        const knot_dname_t
                *n_1 = (const uint8_t *)"\4dead",
                *n_2 = (const uint8_t *)"\3bee\1f";
-       assert_int_equal(kr_zonecut_add(&cut1, n_1, NULL, 0), 0);
-       assert_int_equal(kr_zonecut_add(&cut1, n_2, NULL, 0), 0);
+       assert_int_equal(kr_zonecut_add(&cut1, n_1, NULL, 0, true), 0);
+       assert_int_equal(kr_zonecut_add(&cut1, n_2, NULL, 0, true), 0);
        /* Copy */
        assert_int_equal(kr_zonecut_copy(&cut2, &cut1), 0);
        /* Check if exist */

diff --git a/lib/zonecut.c b/lib/zonecut.c
index 5839b9743d496cde98af372075b44a6e870fb845..fc62e3c082ec1c18b4b03624dde3da94209ecc11 100644 (file)
--- a/lib/zonecut.c
+++ b/lib/zonecut.c
@@ -162,7 +162,8 @@ int kr_zonecut_copy_trust(struct kr_zonecut *dst, const struct kr_zonecut *src)
        return kr_ok();
 }
 
-int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *data, int len)
+int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns,
+                  const void *data, int len, bool check_dup)
 {
        if (!cut || !ns || !cut->nsset || (data && len <= 0)) {
                assert(!EINVAL);
@@ -188,7 +189,7 @@ int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *d
                return kr_ok();
        }
        /* Check for duplicates */
-       if (pack_obj_find(*pack, data, len)) {
+       if (check_dup && pack_obj_find(*pack, data, len)) {
                return kr_ok();
        }
        /* Push new address */

diff --git a/lib/zonecut.h b/lib/zonecut.h
index 8800e6703ec5125e8eccda99aa3110fe4e7fe263..3967e3aa082103d70fc0638227737ec1fcc158ca 100644 (file)
--- a/lib/zonecut.h
+++ b/lib/zonecut.h
@@ -97,10 +97,12 @@ int kr_zonecut_copy_trust(struct kr_zonecut *dst, const struct kr_zonecut *src);
  * @param ns     nameserver name
  * @param data   typically knot_rdata_t::data
  * @param len    typically knot_rdata_t::len
+ * @param check_dup check for duplicates and return 0 without without adding in that case
  * @return 0 or error code
  */
 KR_EXPORT
-int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns, const void *data, int len);
+int kr_zonecut_add(struct kr_zonecut *cut, const knot_dname_t *ns,
+                  const void *data, int len, bool check_dup);
 
 /**
  * Delete nameserver/address pair from the zone cut.

diff --git a/modules/hints/hints.c b/modules/hints/hints.c
index 7473a34c780011898160686840418930e6a708f0..bd39e14ec62d1395a440200eccc6a8e2b14003bb 100644 (file)
--- a/modules/hints/hints.c
+++ b/modules/hints/hints.c
@@ -237,7 +237,7 @@ static int add_pair(struct kr_zonecut *hints, const char *name, const char *addr
                return kr_error(EINVAL);
        }
 
-       return kr_zonecut_add(hints, key, kr_inaddr(&ia.ip), kr_inaddr_len(&ia.ip));
+       return kr_zonecut_add(hints, key, kr_inaddr(&ia.ip), kr_inaddr_len(&ia.ip), false);
 }
 
 static int add_reverse_pair(struct kr_zonecut *hints, const char *name, const char *addr)
@@ -253,7 +253,7 @@ static int add_reverse_pair(struct kr_zonecut *hints, const char *name, const ch
                return kr_error(EINVAL);
        }
 
-       return kr_zonecut_add(hints, key, ptr_name, knot_dname_size(ptr_name));
+       return kr_zonecut_add(hints, key, ptr_name, knot_dname_size(ptr_name), false);
 }
 
 /** For a given name, remove either one address or all of them (if == NULL).

diff --git a/modules/priming/priming.lua b/modules/priming/priming.lua
index c60f2554057c3243614ba5782963f5aa0bc242f2..79d640cdd12e89d17589b958b5304ca49f92900a 100644 (file)
--- a/modules/priming/priming.lua
+++ b/modules/priming/priming.lua
@@ -23,7 +23,7 @@ local function publish_hints(nsset)
        for dname, addrsets in pairs(nsset) do
                for i = 0, addrsets:rdcount() - 1 do
                        local rdpt = addrsets:rdata_pt(i)
-                       ffi.C.kr_zonecut_add(roothints, dname, rdpt.data, rdpt.len)
+                       ffi.C.kr_zonecut_add(roothints, dname, rdpt.data, rdpt.len, true)
                end
        end
 end