From 931b8be2537b70088e34587b1d6975544c51c24b Mon Sep 17 00:00:00 2001 From: Grigorii Demidov Date: Tue, 6 Mar 2018 16:49:00 +0100 Subject: [PATCH] daemon: cache prefill module --- daemon/bindings.c | 47 ++-- daemon/worker.h | 3 + daemon/zimport.c | 415 ++++++++++++++++++++++-------------- modules/modules.mk | 3 +- modules/prefill/prefill.lua | 157 ++++++++++++++ modules/prefill/prefill.mk | 2 + 6 files changed, 452 insertions(+), 175 deletions(-) create mode 100644 modules/prefill/prefill.lua create mode 100644 modules/prefill/prefill.mk diff --git a/daemon/bindings.c b/daemon/bindings.c index 23ac4f7a4..e3dd72584 100644 --- a/daemon/bindings.c +++ b/daemon/bindings.c @@ -1165,30 +1165,36 @@ static void cache_zone_import_cb(int state, void *param) /** Import zone from file. */ static int cache_zone_import(lua_State *L) { + int ret = -1; + char msg[128]; + struct worker_ctx *worker = wrk_luaget(L); if (!worker) { - return 0; + strncpy(msg, "internal error, empty worker pointer", sizeof(msg)); + goto finish; } if (worker->z_import && zi_import_started(worker->z_import)) { - format_error(L, "import has already started"); - lua_error(L); + strncpy(msg, "import already started", sizeof(msg)); + goto finish; } struct engine *engine = engine_luaget(L); if (!engine) { - return 0; + strncpy(msg, "internal error, empty engine pointer", sizeof(msg)); + goto finish; } struct kr_cache *cache = &engine->resolver.cache; if (!kr_cache_is_open(cache)) { - return 0; + strncpy(msg, "cache isn't open", sizeof(msg)); + goto finish; } /* Check parameters */ int n = lua_gettop(L); if (n < 1 || !lua_isstring(L, 1)) { - format_error(L, "expected 'cache.zone_import(string key)'"); - lua_error(L); + strncpy(msg, "expected 'cache.zone_import(path to zone file)'", sizeof(msg)); + goto finish; } /* Parse zone file */ @@ -1201,20 +1207,31 @@ static int cache_zone_import(lua_State *L) if (worker->z_import == NULL) { worker->z_import = zi_allocate(worker, cache_zone_import_cb, worker); if (worker->z_import == NULL) { - format_error(L, "can't allocate zone import context"); - lua_error(L); + strncpy(msg, "can't allocate zone import context", sizeof(msg)); + goto finish; } } - int ret = zi_zone_import(worker->z_import, zone_file, default_origin, - default_rclass, default_ttl); + ret = zi_zone_import(worker->z_import, zone_file, default_origin, + default_rclass, default_ttl); - if (ret != 0) { - format_error(L, "error parsing zone file"); - lua_error(L); + lua_newtable(L); + if (ret == 0) { + strncpy(msg, "zone file successfully parsed, import started", sizeof(msg)); + } else if (ret == 1) { + strncpy(msg, "TA not found", sizeof(msg)); + } else { + strncpy(msg, "error parsing zone file", sizeof(msg)); } - lua_pushstring(L, "zone file successfully parsed, import started"); +finish: + msg[sizeof(msg) - 1] = 0; + lua_newtable(L); + lua_pushstring(L, msg); + lua_setfield(L, -2, "msg"); + lua_pushnumber(L, ret); + lua_setfield(L, -2, "code"); + return 1; } diff --git a/daemon/worker.h b/daemon/worker.h index 7a2d17cd2..ec3e0f885 100644 --- a/daemon/worker.h +++ b/daemon/worker.h @@ -96,6 +96,9 @@ ssize_t worker_gnutls_push(gnutls_transport_ptr_t h, const void *buf, size_t len ssize_t worker_gnutls_client_push(gnutls_transport_ptr_t h, const void *buf, size_t len); +/** Finalize given task */ +int worker_task_finalize(struct qr_task *task, int state); + /** @cond internal */ /** Number of request within timeout window. */ diff --git a/daemon/zimport.c b/daemon/zimport.c index 1fed48cbd..45de989ac 100644 --- a/daemon/zimport.c +++ b/daemon/zimport.c @@ -14,6 +14,36 @@ along with this program. If not, see . */ +/* Module is intended to import resource records from file into resolver's cache. + * File supposed to be a standard DNS zone file + * which contains text representations of resource records. + * For now only root zone import is supported. + * + * Import process consists of two stages. + * 1) Zone file parsing. + * 2) Import of parsed entries into the cache. + * + * These stages are implemented as two separate functions + * (zi_zone_import and zi_zone_process) which runs sequentially with the + * pause between them. This is done because resolver is a single-threaded + * application, so it can't process user's requests during the whole import + * process. Separation into two stages allows to reduce the + * continuous time interval when resolver can't serve user requests. + * Since root zone isn't large it is imported as single + * chunk. If it would be considered as necessary, import stage can be + * split into shorter stages. + * + * zi_zone_import() uses libzscanner to parse zone file. + * Parsed records are stored to internal storage from where they are imported to + * cache during the second stage. + * + * zi_zone_process() imports parsed resource records to cache. + * It imports rrset by creating request that will never be sent to upstream. + * After request creation resolver creates pseudo-answer which must contain + * all necessary data for validation. Then resolver process answer as if he had + * been received from network. + */ + #include #include #include @@ -29,6 +59,8 @@ #define VERBOSE_MSG(qry, fmt...) QRVERBOSE(qry, "zimport", fmt) +/* Pause between parse and import stages, milliseconds. + * See comment in zi_zone_import() */ #define ZONE_IMPORT_PAUSE 100 typedef array_t(knot_rrset_t *) qr_rrsetlist_t; @@ -42,8 +74,8 @@ struct zone_import_ctx { uint64_t start_timestamp; size_t rrset_idx; uv_timer_t timer; - map_t rrset_sorted; - qr_rrsetlist_t rrset_list; + map_t rrset_indexed; + qr_rrsetlist_t rrset_sorted; knot_mm_t pool; zi_callback cb; void *cb_param; @@ -51,8 +83,6 @@ struct zone_import_ctx { typedef struct zone_import_ctx zone_import_ctx_t; -int worker_task_finalize(struct qr_task *task, int state); - static int RRSET_IS_ALREADY_IMPORTED = 1; /** @internal Allocate zone import context. @@ -74,7 +104,7 @@ static void zi_ctx_free(zone_import_ctx_t *z_import) * Flushes memory pool, but doesn't reallocate memory pool buffer. * Doesn't affect timer handle, pointers to callback and callback parameter. * @return 0 if success; -1 if failed. */ -static int zi_reset(struct zone_import_ctx *z_import, size_t rrset_list_size) +static int zi_reset(struct zone_import_ctx *z_import, size_t rrset_sorted_list_size) { mp_flush(z_import->pool.ctx); @@ -82,13 +112,13 @@ static int zi_reset(struct zone_import_ctx *z_import, size_t rrset_list_size) z_import->start_timestamp = 0; z_import->rrset_idx = 0; z_import->pool.alloc = (knot_mm_alloc_t) mp_alloc; - z_import->rrset_sorted = map_make(&z_import->pool); + z_import->rrset_indexed = map_make(&z_import->pool); - array_init(z_import->rrset_list); + array_init(z_import->rrset_sorted); int ret = 0; - if (rrset_list_size) { - ret = array_reserve_mm(z_import->rrset_list, rrset_list_size, + if (rrset_sorted_list_size) { + ret = array_reserve_mm(z_import->rrset_sorted, rrset_sorted_list_size, kr_memreserve, &z_import->pool); } @@ -127,7 +157,7 @@ zone_import_ctx_t *zi_allocate(struct worker_ctx *worker, if (ret < 0) { mp_delete(mp); zi_ctx_free(z_import); - z_import = NULL; + return NULL; } uv_timer_init(z_import->worker->loop, &z_import->timer); z_import->timer.data = z_import; @@ -164,52 +194,144 @@ static inline bool zi_rrset_is_marked_as_imported(knot_rrset_t *rr) return (rr->additional == &RRSET_IS_ALREADY_IMPORTED); } -/** @internal Try to find rrset with required properties amongst parsed rrsets - * and put it to given packet, If rrset found and successfully put, it marked as - * "already imported" to avoid repeated import. +/** @internal Try to find rrset with given requisites amongst parsed rrsets + * and put it to given packet. If there is RRSIG which covers that rrset, it + * will be added as well. If rrset found and successfully put, it marked as + * "already imported" to avoid repeated import. The same is true for RRSIG. * @return -1 if failed * 0 if required record been actually put into the packet * 1 if required record could not be found */ -static int zi_put_supplementary(struct zone_import_ctx *z_import, - knot_pkt_t *pkt, const knot_dname_t *owner, - uint16_t class, uint16_t supp_type) +static int zi_rrset_find_put(struct zone_import_ctx *z_import, + knot_pkt_t *pkt, const knot_dname_t *owner, + uint16_t class, uint16_t type, uint16_t additional) { - assert(supp_type != KNOT_RRTYPE_RRSIG); + if (type != KNOT_RRTYPE_RRSIG) { + /* If required rrset isn't rrsig, these must be the same values */ + additional = type; + } char key[KR_RRKEY_LEN]; - int err = kr_rrkey(key, class, owner, supp_type, supp_type); + int err = kr_rrkey(key, class, owner, type, additional); if (err <= 0) { return -1; } - knot_rrset_t *additional_rr = map_get(&z_import->rrset_sorted, key); - err = kr_rrkey(key, class, owner, KNOT_RRTYPE_RRSIG, supp_type); - if (err <= 0) { + knot_rrset_t *rr = map_get(&z_import->rrset_indexed, key); + if (!rr) { + return 1; + } + err = knot_pkt_put(pkt, 0, rr, 0); + if (err != KNOT_EOK) { return -1; } - knot_rrset_t *rrsig = map_get(&z_import->rrset_sorted, key); - if (additional_rr) { - err = knot_pkt_put(pkt, 0, additional_rr, 0); - if (err != KNOT_EOK) { - return -1; + zi_rrset_mark_as_imported(rr); + + if (type != KNOT_RRTYPE_RRSIG) { + /* Try to find corresponding rrsig */ + err = zi_rrset_find_put(z_import, pkt, owner, + class, KNOT_RRTYPE_RRSIG, type); + if (err < 0) { + return err; } - zi_rrset_mark_as_imported(additional_rr); } - if (rrsig) { - err = knot_pkt_put(pkt, 0, rrsig, 0); - if (err != KNOT_EOK) { - return -1; + + return 0; +} + +/** @internal Try to put given rrset to the given packet. + * If there is RRSIG which covers that rrset, it will be added as well. + * If rrset successfully put in the packet, it marked as + * "already imported" to avoid repeated import. + * The same is true for RRSIG. + * @return -1 if failed + * 0 if required record been actually put into the packet */ +static int zi_rrset_put(struct zone_import_ctx *z_import, knot_pkt_t *pkt, + knot_rrset_t *rr) +{ + assert(rr); + assert(rr->type != KNOT_RRTYPE_RRSIG); + int err = knot_pkt_put(pkt, 0, rr, 0); + if (err != KNOT_EOK) { + return -1; + } + zi_rrset_mark_as_imported(rr); + /* Try to find corresponding RRSIG */ + err = zi_rrset_find_put(z_import, pkt, rr->owner, rr->rclass, + KNOT_RRTYPE_RRSIG, rr->type); + return (err < 0) ? err : 0; +} + +/** @internal Try to put DS & NSEC* for rset->owner to given packet. + * @return -1 if failed; + * 0 if no errors occurred (it doesn't mean + * that records were actually added). */ +static int zi_put_delegation(zone_import_ctx_t *z_import, knot_pkt_t *pkt, + knot_rrset_t *rr) +{ + int err = zi_rrset_find_put(z_import, pkt, rr->owner, + rr->rclass, KNOT_RRTYPE_DS, 0); + if (err == 1) { + /* DS not found, maybe there are NSEC* */ + err = zi_rrset_find_put(z_import, pkt, rr->owner, + rr->rclass, KNOT_RRTYPE_NSEC, 0); + if (err >= 0) { + err = zi_rrset_find_put(z_import, pkt, rr->owner, + rr->rclass, KNOT_RRTYPE_NSEC3, 0); + } + } + return err < 0 ? err : 0; +} + +/** @internal Try to put A & AAAA records for rset->owner to given packet. + * @return -1 if failed; + * 0 if no errors occurred (it doesn't mean + * that records were actually added). */ +static int zi_put_glue(zone_import_ctx_t *z_import, knot_pkt_t *pkt, + knot_rrset_t *rr) +{ + int err = 0; + for (uint16_t i = 0; i < rr->rrs.rr_count; ++i) { + const knot_dname_t *ns_name = knot_ns_name(&rr->rrs, i); + err = zi_rrset_find_put(z_import, pkt, ns_name, + rr->rclass, KNOT_RRTYPE_A, 0); + if (err < 0) { + break; + } + + err = zi_rrset_find_put(z_import, pkt, ns_name, + rr->rclass, KNOT_RRTYPE_AAAA, 0); + if (err < 0) { + break; } - zi_rrset_mark_as_imported(rrsig); } - return additional_rr == NULL ? 1 : 0; + return err < 0 ? err : 0; +} + +/** @internal Create query. */ +static knot_pkt_t *zi_query_create(zone_import_ctx_t *z_import, knot_rrset_t *rr) +{ + knot_mm_t *pool = &z_import->pool; + + uint32_t msgid = kr_rand_uint(0); + + knot_pkt_t *query = knot_pkt_new(NULL, KNOT_WIRE_MAX_PKTSIZE, pool); + if (!query) { + return NULL; + } + + knot_pkt_put_question(query, rr->owner, rr->rclass, rr->type); + knot_pkt_begin(query, KNOT_ANSWER); + knot_wire_set_rd(query->wire); + knot_wire_set_id(query->wire, msgid); + int err = knot_pkt_parse(query, 0); + if (err != KNOT_EOK) { + knot_pkt_free(&query); + return NULL; + } + + return query; } /** @internal Import given rrset to cache. - * The main goal of import procedure is to store parsed records to the cache. - * Resolver imports rrset by creating request that will never be sent to upstream. - * After request creation resolver creates pseudo-answer which must contain - * all necessary data for validation. Then resolver process answer as if he had - * been received from network. * @return -1 if failed; 0 if success */ static int zi_rrset_import(zone_import_ctx_t *z_import, knot_rrset_t *rr) { @@ -217,34 +339,33 @@ static int zi_rrset_import(zone_import_ctx_t *z_import, knot_rrset_t *rr) assert(worker); - knot_mm_t *pool = &z_import->pool; + /* Create "pseudo query" which asks for given rrset. */ + knot_pkt_t *query = zi_query_create(z_import, rr); + if (!query) { + return -1; + } + knot_mm_t *pool = &z_import->pool; uint8_t *dname = rr->owner; uint16_t rrtype = rr->type; uint16_t rrclass = rr->rclass; - struct kr_qflags options; - memset(&options, 0, sizeof(options)); - options.DNSSEC_WANT = true; - options.NO_MINIMIZE = true; - uint32_t msgid = kr_rand_uint(0); - - /* Create query */ - knot_pkt_t *query = knot_pkt_new(NULL, KNOT_WIRE_MAX_PKTSIZE, pool); - if (!query) { - return -1; - } + /* Create "pseudo answer". */ knot_pkt_t *answer = knot_pkt_new(NULL, KNOT_WIRE_MAX_PKTSIZE, pool); if (!answer) { knot_pkt_free(&query); return -1; } + knot_pkt_put_question(answer, dname, rrclass, rrtype); + knot_pkt_begin(answer, KNOT_ANSWER); - knot_pkt_put_question(query, dname, rrclass, rrtype); - knot_pkt_begin(query, KNOT_ANSWER); - knot_wire_set_rd(query->wire); - knot_wire_set_id(query->wire, msgid); + struct kr_qflags options; + memset(&options, 0, sizeof(options)); + options.DNSSEC_WANT = true; + options.NO_MINIMIZE = true; + /* This call creates internal structures which necessary for + * resolving - qr_task & request_ctx. */ struct qr_task *task = worker_resolve_start(worker, query, options); if (!task) { knot_pkt_free(&query); @@ -252,16 +373,20 @@ static int zi_rrset_import(zone_import_ctx_t *z_import, knot_rrset_t *rr) return -1; } + /* Push query to the request resolve plan. + * Actually query will never been sent to upstream. */ struct kr_request *request = worker_task_request(task); struct kr_rplan *rplan = &request->rplan; struct kr_query *qry = kr_rplan_push(rplan, NULL, dname, rrclass, rrtype); - char key[KR_RRKEY_LEN]; - int err = 0; int state = KR_STATE_FAIL; bool origin_is_owner = knot_dname_is_equal(rr->owner, z_import->origin); bool is_referral = (rrtype == KNOT_RRTYPE_NS && !origin_is_owner); + uint32_t msgid = knot_wire_get_id(query->wire); qry->id = msgid; + + /* Prepare zonecut. It must have all the necessary requisites for + * successful validation - matched zone name & keys & trust-anchors. */ kr_zonecut_init(&qry->zone_cut, z_import->origin, pool); qry->zone_cut.key = z_import->key; qry->zone_cut.trust_anchor = z_import->ta; @@ -270,53 +395,26 @@ static int zi_rrset_import(zone_import_ctx_t *z_import, knot_rrset_t *rr) goto cleanup; } - knot_pkt_put_question(answer, dname, rrclass, rrtype); - knot_pkt_begin(answer, KNOT_ANSWER); - + /* Since "pseudo" query asks for NS for subzone, + * "pseudo" answer must simulate referral. */ if (is_referral) { knot_pkt_begin(answer, KNOT_AUTHORITY); } - err = knot_pkt_put(answer, 0, rr, 0); + /* Put target rrset to ANSWER\AUTHORIRY as well as corresponding RRSIG */ + int err = zi_rrset_put(z_import, answer, rr); if (err != 0) { goto cleanup; } - zi_rrset_mark_as_imported(rr); - - err = kr_rrkey(key, rr->rclass, rr->owner, KNOT_RRTYPE_RRSIG, rr->type); - if (err <= 0) { - goto cleanup; - } - knot_rrset_t *rrsig = map_get(&z_import->rrset_sorted, key); - if (rrsig) { - err = knot_pkt_put(answer, 0, rrsig, 0); - if (err != 0) { - goto cleanup; - } - zi_rrset_mark_as_imported(rrsig); - } if (!is_referral) { knot_wire_set_aa(answer->wire); } else { /* Type is KNOT_RRTYPE_NS and owner is not equal to origin. - * It will be "referral" answer, so try to add DS or NSEC* to it. */ - err = zi_put_supplementary(z_import, answer, rr->owner, - rr->rclass, KNOT_RRTYPE_DS); + * It will be "referral" answer and must contain delegation. */ + err = zi_put_delegation(z_import, answer, rr); if (err < 0) { goto cleanup; - } else if (err == 1) { - /* DS not found */ - err = zi_put_supplementary(z_import, answer, rr->owner, - rr->rclass, KNOT_RRTYPE_NSEC); - if (err < 0) { - goto cleanup; - } - err = zi_put_supplementary(z_import, answer, rr->owner, - rr->rclass, KNOT_RRTYPE_NSEC3); - if (err < 0) { - goto cleanup; - } } } @@ -324,24 +422,18 @@ static int zi_rrset_import(zone_import_ctx_t *z_import, knot_rrset_t *rr) if (rrtype == KNOT_RRTYPE_NS) { /* Try to find glue addresses. */ - for (uint16_t i = 0; i < rr->rrs.rr_count; ++i) { - const knot_dname_t *ns_name = knot_ns_name(&rr->rrs, i); - err = zi_put_supplementary(z_import, answer, ns_name, - rr->rclass, KNOT_RRTYPE_A); - if (err < 0) { - goto cleanup; - } - - err = zi_put_supplementary(z_import, answer, ns_name, - rr->rclass, KNOT_RRTYPE_AAAA); - if (err < 0) { - goto cleanup; - } + err = zi_put_glue(z_import, answer, rr); + if (err < 0) { + goto cleanup; } } knot_wire_set_id(answer->wire, msgid); answer->parsed = answer->size; + err = knot_pkt_parse(answer, 0); + if (err != KNOT_EOK) { + goto cleanup; + } /* Importing doesn't imply communication with upstream at all. * "answer" contains pseudo-answer from upstream and must be successfully @@ -362,7 +454,7 @@ static int zi_mapwalk_preprocess(const char *k, void *v, void *baton) { zone_import_ctx_t *z_import = (zone_import_ctx_t *)baton; - int ret = array_push_mm(z_import->rrset_list, v, kr_memreserve, &z_import->pool); + int ret = array_push_mm(z_import->rrset_sorted, v, kr_memreserve, &z_import->pool); return (ret < 0); } @@ -390,26 +482,11 @@ static void zi_zone_process(uv_timer_t* handle) goto finish; } - if (z_import->rrset_list.len <= 0) { + if (z_import->rrset_sorted.len <= 0) { VERBOSE_MSG(NULL, "zone is empty\n"); goto finish; } - /* z_import.rrset_list now contains sorted rrset list. - * Records are sorted by the key returned by kr_rrkey() function. - * Find out if zone is secured. - * Try to find TA for worker->z_import.origin. */ - map_t *trust_anchors = &z_import->worker->engine->resolver.trust_anchors; - knot_rrset_t *rr = kr_ta_get(trust_anchors, z_import->origin); - if (!rr) { - /* For now - fail. - * TODO - query DS and continue after answer had been obtained. */ - kr_log_error("[zimport] TA not found for `%s`, fail\n", zone_name_str); - failed = 1; - goto finish; - } - z_import->ta = rr; - /* TA have been found, zone is secured. * DNSKEY must be somewhere amongst the imported records. Find it. * TODO - For those zones that provenly do not have TA this step must be skipped. */ @@ -421,7 +498,7 @@ static void zi_zone_process(uv_timer_t* handle) goto finish; } - rr = map_get(&z_import->rrset_sorted, key); + knot_rrset_t *rr = map_get(&z_import->rrset_indexed, key); if (!rr) { /* DNSKEY MUST be here. If not found - fail. */ kr_log_error("[zimport] DNSKEY not found for `%s`, fail\n", zone_name_str); @@ -451,8 +528,8 @@ static void zi_zone_process(uv_timer_t* handle) } /* Import all NS records */ - for (size_t i = 0; i < z_import->rrset_list.len; ++i) { - knot_rrset_t *rr = z_import->rrset_list.at[i]; + for (size_t i = 0; i < z_import->rrset_sorted.len; ++i) { + knot_rrset_t *rr = z_import->rrset_sorted.at[i]; if (rr->type != KNOT_RRTYPE_NS) { continue; @@ -470,14 +547,14 @@ static void zi_zone_process(uv_timer_t* handle) qname_str, type_str); ++failed; } - z_import->rrset_list.at[i] = NULL; + z_import->rrset_sorted.at[i] = NULL; } /* NS records have been imported as well as relative DS, NSEC* and glue. * Now import what's left. */ - for (size_t i = 0; i < z_import->rrset_list.len; ++i) { + for (size_t i = 0; i < z_import->rrset_sorted.len; ++i) { - knot_rrset_t *rr = z_import->rrset_list.at[i]; + knot_rrset_t *rr = z_import->rrset_sorted.at[i]; if (rr == NULL) { continue; } @@ -556,31 +633,34 @@ static int zi_record_store(zs_scanner_t *s) kr_log_error("[zscanner] line %lu: error creating rrset\n", s->line_counter); return -1; } - int ret = knot_rrset_add_rdata(new_rr, s->r_data, s->r_data_length, + int res = knot_rrset_add_rdata(new_rr, s->r_data, s->r_data_length, s->r_ttl, &z_import->pool); - if (ret != KNOT_EOK) { + if (res != KNOT_EOK) { kr_log_error("[zscanner] line %lu: error adding rdata to rrset\n", s->line_counter); return -1; } + /* Records in zone file may not be grouped by name and RR type. + * Use map to create search key and + * avoid ineffective searches across all the imported records. */ char key[KR_RRKEY_LEN]; uint16_t additional_key_field = kr_rrset_type_maysig(new_rr); - ret = kr_rrkey(key, new_rr->rclass, new_rr->owner, new_rr->type, + res = kr_rrkey(key, new_rr->rclass, new_rr->owner, new_rr->type, additional_key_field); - if (ret <= 0) { + if (res <= 0) { kr_log_error("[zscanner] line %lu: error constructing rrkey\n", s->line_counter); return -1; } - knot_rrset_t *saved_rr = map_get(&z_import->rrset_sorted, key); + knot_rrset_t *saved_rr = map_get(&z_import->rrset_indexed, key); if (saved_rr) { - ret = knot_rdataset_merge(&saved_rr->rrs, &new_rr->rrs, + res = knot_rdataset_merge(&saved_rr->rrs, &new_rr->rrs, &z_import->pool); } else { - ret = map_set(&z_import->rrset_sorted, key, new_rr); + res = map_set(&z_import->rrset_indexed, key, new_rr); } - if (ret != 0) { + if (res != 0) { kr_log_error("[zscanner] line %lu: error saving parsed rrset\n", s->line_counter); return -1; } @@ -608,14 +688,20 @@ static int zi_state_parsing(zs_scanner_t *s) } break; case ZS_STATE_ERROR: - kr_log_error("[zscanner] line: %lu: error parsing record\n", s->line_counter); + kr_log_error("[zscanner] line: %lu: parse error; code: %i ('%s')\n", + s->line_counter, s->error.code, zs_strerror(s->error.code)); return -1; - break; case ZS_STATE_INCLUDE: + kr_log_error("[zscanner] line: %lu: INCLUDE is not supported\n", + s->line_counter); return -1; - break; - default: + case ZS_STATE_EOF: + case ZS_STATE_STOP: return (s->error.counter == 0) ? 0 : -1; + default: + kr_log_error("[zscanner] line: %lu: unexpected parse state: %i\n", + s->line_counter, s->state); + return -1; } } @@ -626,46 +712,45 @@ int zi_zone_import(struct zone_import_ctx *z_import, const char *zone_file, const char *origin, uint16_t rclass, uint32_t ttl) { - if (z_import->worker == NULL) { - kr_log_error("[zscanner] invalid parameter\n"); - return -1; - } + assert (z_import != NULL && "[zimport] empty parameter"); + assert (z_import->worker != NULL && "[zimport] invalid parameter\n"); + assert (zone_file != NULL && "[zimport] empty parameter\n"); zs_scanner_t *s = malloc(sizeof(zs_scanner_t)); if (s == NULL) { - kr_log_error("[zscanner] error creating instance of zone scanner\n"); + kr_log_error("[zscanner] error creating instance of zone scanner (malloc() fails)\n"); return -1; } - if (zs_init(s, origin, rclass, ttl) != 0) { + /* zs_init(), zs_set_input_file(), zs_set_processing() returns -1 in case of error, + * so don't print error code as it meaningless. */ + int res = zs_init(s, origin, rclass, ttl); + if (res != 0) { free(s); - kr_log_error("[zscanner] error initializing zone scanner instance\n"); + kr_log_error("[zscanner] error initializing zone scanner instance, error: %i (%s)\n", + s->error.code, zs_strerror(s->error.code)); return -1; } - if (zs_set_input_file(s, zone_file) != 0) { + res = zs_set_input_file(s, zone_file); + if (res != 0) { zs_deinit(s); free(s); - kr_log_error("[zscanner] error opening zone file `%s`\n", zone_file); + kr_log_error("[zscanner] error opening zone file `%s`, error: %i (%s)\n", + zone_file, s->error.code, zs_strerror(s->error.code)); return -1; } - /* Don't set callbacks as we don't use automatic parsing. - * Store pointer to zone import context. */ + /* Don't set processing and error callbacks as we don't use automatic parsing. + * Parsing as well error processing will be performed in zi_state_parsing(). + * Store pointer to zone import context for further use. */ if (zs_set_processing(s, NULL, NULL, (void *)z_import) != 0) { zs_deinit(s); free(s); - kr_log_error("[zscanner] error opening zone file `%s`\n", zone_file); + kr_log_error("[zscanner] zs_set_input_file() fails\n"); return -1; } - /* To reduce time spent in the callback, import is split - * into two stages. In the first stage zone file is parsed and prepared - * for importing to cache. In the second stage parsed zone is imported - * into the cache. Since root zone isn't large it is imported as single - * chunk. If it would be considered as necessary, second stage can be - * split into shorter stages. */ - uint64_t elapsed = 0; int ret = zi_reset(z_import, 4096); if (ret == 0) { @@ -674,6 +759,19 @@ int zi_zone_import(struct zone_import_ctx *z_import, VERBOSE_MSG(NULL, "[zscanner] started; zone file `%s`\n", zone_file); ret = zi_state_parsing(s); + /* Try to find TA for worker->z_import.origin. */ + map_t *trust_anchors = &z_import->worker->engine->resolver.trust_anchors; + knot_rrset_t *rr = kr_ta_get(trust_anchors, z_import->origin); + if (rr) { + z_import->ta = rr; + } else { + /* For now - fail. + * TODO - query DS and continue after answer had been obtained. */ + char zone_name_str[KNOT_DNAME_MAXLEN]; + knot_dname_to_str(zone_name_str, z_import->origin, sizeof(zone_name_str)); + kr_log_error("[zimport] no TA found for `%s`, fail\n", zone_name_str); + ret = 1; + } elapsed = kr_now() - z_import->start_timestamp; elapsed = elapsed > UINT_MAX ? UINT_MAX : elapsed; } @@ -688,10 +786,9 @@ int zi_zone_import(struct zone_import_ctx *z_import, VERBOSE_MSG(NULL, "[zscanner] finished in %lu ms; zone file `%s`\n", elapsed, zone_file); + map_walk(&z_import->rrset_indexed, zi_mapwalk_preprocess, z_import); - map_walk(&z_import->rrset_sorted, zi_mapwalk_preprocess, z_import); - - /* Start import */ + /* Zone have been parsed already, so start the import. */ uv_timer_start(&z_import->timer, zi_zone_process, ZONE_IMPORT_PAUSE, ZONE_IMPORT_PAUSE); diff --git a/modules/modules.mk b/modules/modules.mk index 864058f67..e4e5f9268 100644 --- a/modules/modules.mk +++ b/modules/modules.mk @@ -38,7 +38,8 @@ modules_TARGETS += etcd \ priming \ serve_stale \ detect_time_skew \ - detect_time_jump + detect_time_jump \ + prefill endif # Make C module diff --git a/modules/prefill/prefill.lua b/modules/prefill/prefill.lua new file mode 100644 index 000000000..ada7b7d0c --- /dev/null +++ b/modules/prefill/prefill.lua @@ -0,0 +1,157 @@ +local https = require('ssl.https') +local ltn12 = require('ltn12') +local lfs = require('lfs') + +local rz_url = "https://www.internic.net/domain/root.zone" +local rz_local_fname = "root.zone" +local rz_ca_path = nil +local rz_event_id = nil +local rz_auth = false + +local rz_default_interval = 86400 +local rz_https_fail_interval = 600 +local rz_no_ta_interval = 600 +local rz_initial_interval = 15 +local rz_cur_interval = rz_default_interval +local rz_interval_randomizator_limit = 10 +local rz_next_refresh = 86400 +local rz_interval_threshold = 5 +local rz_interval_min = 3600 + +local prefetch = { +} + + +-- Fetch over HTTPS with peert cert checked +local function https_fetch(auth, url, ca) + local resp = {} + local verify = {'none'} + local capath = nil + if auth then + verify = {'peer', 'fail_if_no_peer_cert' } + capath = ca + end + local r, c = https.request{ + url = url, + verify = verify, + capath = capath, + protocol = 'tlsv1_2', + sink = ltn12.sink.table(resp), + } + if r == nil then + return r, c + end + return resp, "[prefill] "..url.." downloaded" +end + +-- Write root zone to a file. +local function rzone_write(rzone) + local tmp_rz_fname = os.tmpname() + local file = assert(io.open(tmp_rz_fname, 'w')) + for i = 1, #rzone do + local rzone_chunk = rzone[i] + file:write(rzone_chunk) + end + file:close() + os.rename(tmp_rz_fname, rz_local_fname) + -- TODO: IO error handling +end + +local function display_delay(time) + local days = math.floor(time / 86400) + local hours = math.floor((time % 86400) / 3600) + local minutes = math.floor((time % 3600) / 60) + local seconds = math.floor(time % 60) + if days > 0 then + return string.format("%d days %02d hours", days, hours) + elseif hours > 0 then + return string.format("%02d hours %02d minutes", hours, minutes) + elseif minutes > 0 then + return string.format("%02d minutes %02d seconds", minutes, seconds) + end + return string.format("%02d seconds", seconds) +end + +local function check_time() + local expected_refresh = rz_next_refresh + local attrs = lfs.attributes(rz_local_fname) + if attrs then + expected_refresh = attrs.modification + rz_cur_interval + end + + local delay = expected_refresh - os.time() + if (delay > rz_interval_threshold) then + log("[prefill] next refresh for . in %s" , display_delay(delay)) + event.reschedule(rz_event_id, delay * sec) + return + end + + log("[prefill] downloading root zone...") + local rzone, err = https_fetch(rz_auth, rz_url, rz_ca_path) + if rzone == nil then + log(string.format("[prefill] fetch of `%s` failed: %s", rz_url, err)) + rz_cur_interval = rz_https_fail_interval; + rz_next_refresh = os.time() + rz_cur_interval + event.reschedule(rz_event_id, rz_cur_interval * sec) + log("[prefill] next refresh for . in %s", display_delay(rz_cur_interval)) + return + end + + log("[prefill] saving root zone...") + rzone_write(rzone) + local res = cache.zone_import('root.zone') + if res.code == 1 then -- no TA found, wait + log("[prefill] no TA found for root zone") + rz_cur_interval = rz_no_ta_interval + elseif res.code == 0 then + log("[prefill] root zone successfully parsed, import started") + rz_cur_interval = rz_default_interval + else + log("[prefill] root zone import failed (%s)", res.msg) + rz_cur_interval = rz_default_interval + end + + rz_cur_interval = rz_cur_interval + math.random(rz_interval_randomizator_limit) + rz_next_refresh = os.time() + rz_cur_interval + event.reschedule(rz_event_id, rz_cur_interval * sec) + log("[prefill] next refresh for . in %s", display_delay(rz_cur_interval)) + return +end + +function prefetch.init() + if rz_event_id then + error('[prefill] module is already loaded.') + end + math.randomseed(os.time()) + rz_event_id = event.after(rz_initial_interval * sec , check_time) +end + +function prefetch.deinit() + if rz_event_id then + event.cancel(rz_event_id) + rz_event_id = nil + end +end + +function prefetch.config(config) + if config and config.interval then + rz_default_interval = config.interval + if rz_default_interval < rz_interval_min then + log("[prefill] too small refresh interval (%d s), use default value", + rz_default_interval) + rz_default_interval = rz_interval_min + end + rz_cur_interval = rz_default_interval + end + if config and config.ca_path then + rz_ca_path = config.ca_path + rz_auth = true + end + log("[prefill] refresh interval: %i s; authentication: %s", + rz_default_interval, tostring(rz_auth)) + if rz_auth then + log("[prefill] ca path: %s", rz_ca_path) + end +end + +return prefetch diff --git a/modules/prefill/prefill.mk b/modules/prefill/prefill.mk new file mode 100644 index 000000000..7b10ba9ec --- /dev/null +++ b/modules/prefill/prefill.mk @@ -0,0 +1,2 @@ +prefill_SOURCES := prefill.lua +$(call make_lua_module,prefill) -- 2.47.2