journal\-max\-usage: SIZE
journal\-max\-depth: INT
zone\-max\-size : SIZE
+ adjust\-threads: INT
dnssec\-signing: BOOL
dnssec\-policy: STR
serial\-policy: increment | unixtime | dateserial
size of the zone must satisfy the configured value.
.sp
\fIDefault:\fP 2^64
+.SS adjust\-threads
+.sp
+Parallelize internal zone adjusting procedures. This is useful with huge
+zones with NSEC3. Speedup observable at server startup and while processing
+NSEC3 re\-salt.
+.sp
+\fIDefault:\fP 1
.SS dnssec\-signing
.sp
If enabled, automatic DNSSEC signing for the zone is turned on.
journal-max-usage: SIZE
journal-max-depth: INT
zone-max-size : SIZE
+ adjust-threads: INT
dnssec-signing: BOOL
dnssec-policy: STR
serial-policy: increment | unixtime | dateserial
*Default:* 2^64
+.. _zone_adjust-threads:
+
+adjust-threads
+--------------
+
+Parallelize internal zone adjusting procedures. This is useful with huge
+zones with NSEC3. Speedup observable at server startup and while processing
+NSEC3 re-salt.
+
+*Default:* 1
+
.. _zone_dnssec-signing:
dnssec-signing
{ C_SERIAL_POLICY, YP_TOPT, YP_VOPT = { serial_policies, SERIAL_POLICY_INCREMENT } }, \
{ C_REFRESH_MAX_INTERVAL,YP_TINT, YP_VINT = { 2, UINT32_MAX, UINT32_MAX, YP_STIME } }, \
{ C_REFRESH_MIN_INTERVAL,YP_TINT, YP_VINT = { 2, UINT32_MAX, 2, YP_STIME } }, \
+ { C_ADJUST_THR, YP_TINT, YP_VINT = { 1, UINT16_MAX, 1 } }, \
{ C_MODULE, YP_TDATA, YP_VDATA = { 0, NULL, mod_id_to_bin, mod_id_to_txt }, \
YP_FMULTI | FLAGS, { check_modref } }, \
{ C_COMMENT, YP_TSTR, YP_VNONE }, \
#define C_ACL "\x03""acl"
#define C_ACTION "\x06""action"
#define C_ADDR "\x07""address"
+#define C_ADJUST_THR "\x0E""adjust-threads"
#define C_ALG "\x09""algorithm"
#define C_ANS_ROTATION "\x0F""answer-rotation"
#define C_ANY "\x03""any"
return ret;
}
- ret = zone_adjust_contents(update->new_cont, adjust_cb_void, NULL, false, update->a_ctx->node_ptrs);
+ ret = zone_adjust_contents(update->new_cont, adjust_cb_void, NULL, false, true, 1, update->a_ctx->node_ptrs);
if (ret != KNOT_EOK) {
return ret;
}
return ret;
}
- ret = zone_adjust_contents(update->new_cont, NULL, adjust_cb_void, false, update->a_ctx->node_ptrs);
+ ret = zone_adjust_contents(update->new_cont, NULL, adjust_cb_void, false, true, 1, update->a_ctx->node_ptrs);
if (ret != KNOT_EOK) {
return ret;
}
goto done;
}
- result = zone_adjust_contents(update->new_cont, adjust_cb_flags, NULL, false, update->a_ctx->node_ptrs);
+ result = zone_adjust_contents(update->new_cont, adjust_cb_flags, NULL,
+ false, false, 1, update->a_ctx->node_ptrs);
if (result != KNOT_EOK) {
return result;
}
goto done;
}
- result = zone_adjust_contents(update->new_cont, adjust_cb_flags, NULL, false, update->a_ctx->node_ptrs);
+ result = zone_adjust_contents(update->new_cont, adjust_cb_flags, NULL, false, false, 1, update->a_ctx->node_ptrs);
if (result != KNOT_EOK) {
goto done;
}
return interval;
}
-static int xfr_validate(zone_contents_t *zone, struct refresh_data *data)
+static int xfr_validate(zone_contents_t *zone)
{
+ // adjust_cb_nsec3_pointer not needed as we don't check DNSSEC here
+ int ret = zone_adjust_contents(zone, adjust_cb_flags, NULL, false, false, 1, NULL);
+ if (ret != KNOT_EOK) {
+ return ret;
+ }
+
sem_handler_t handler = {
.cb = err_handler_logger
};
- int ret = sem_checks_process(zone, false, &handler, time(NULL));
+ ret = sem_checks_process(zone, false, &handler, time(NULL));
if (ret != KNOT_EOK) {
// error is logged by the error handler
return ret;
{
zone_contents_t *new_zone = data->axfr.zone;
- int ret = zone_adjust_contents(new_zone, adjust_cb_flags, NULL, false, NULL); // adjust_cb_nsec3_pointer not needed as we don't check DNSSEC in xfr_validate()
- if (ret == KNOT_EOK) {
- ret = xfr_validate(new_zone, data);
- }
+ int ret = xfr_validate(new_zone);
if (ret != KNOT_EOK) {
return ret;
}
}
}
- ret = zone_adjust_contents(up.new_cont, adjust_cb_flags, NULL, false, NULL); // adjust_cb_nsec3_pointer not needed as we don't check DNSSEC in xfr_validate()
- if (ret == KNOT_EOK) {
- ret = xfr_validate(up.new_cont, data);
- }
+ ret = xfr_validate(up.new_cont);
if (ret != KNOT_EOK) {
zone_update_clear(&up);
return ret;
return ret;
}
+ conf_val_t thr = conf_zone_get(conf, C_ADJUST_THR, update->zone->name);
if ((update->flags & (UPDATE_HYBRID | UPDATE_FULL))) {
- ret = zone_adjust_full(update->new_cont);
+ ret = zone_adjust_full(update->new_cont, conf_int(&thr));
} else {
ret = zone_adjust_incremental_update(update);
}
adjust_cb_t adjust_cb;
bool adjust_prevs;
measure_t *m;
+
+ // just for parallel
+ unsigned threads;
+ unsigned thr_id;
+ size_t i;
+ pthread_t thread;
+ int ret;
+ zone_tree_t *tree;
} zone_adjust_arg_t;
static int adjust_single(zone_node_t *node, void *data)
zone_adjust_arg_t *args = (zone_adjust_arg_t *)data;
- knot_measure_node(node, args->m);
+ // parallel adjust support
+ if (args->threads > 1) {
+ if (args->i++ % args->threads != args->thr_id) {
+ return KNOT_EOK;
+ }
+ }
+
+ if (args->m != NULL) {
+ knot_measure_node(node, args->m);
+ }
if ((node->flags & NODE_FLAGS_DELETED)) {
return KNOT_EOK;
return KNOT_EOK;
}
+static void *adjust_tree_thread(void *ctx)
+{
+ zone_adjust_arg_t *arg = ctx;
+
+ arg->ret = zone_tree_apply(arg->tree, adjust_single, ctx);
+
+ return NULL;
+}
+
+static int zone_adjust_tree_parallel(zone_tree_t *tree, adjust_ctx_t *ctx,
+ adjust_cb_t adjust_cb, unsigned threads)
+{
+ if (zone_tree_is_empty(tree)) {
+ return KNOT_EOK;
+ }
+
+ zone_adjust_arg_t arg[threads];
+ int ret = KNOT_EOK;
+
+ for (unsigned i = 0; i < threads; i++) {
+ arg[i].ctx = ctx;
+ arg[i].adjust_cb = adjust_cb;
+ arg[i].adjust_prevs = false;
+ arg[i].first_node = NULL;
+ arg[i].m = NULL;
+ arg[i].tree = tree;
+ arg[i].threads = threads;
+ arg[i].i = 0;
+ arg[i].thr_id = i;
+ arg[i].ret = pthread_create(&arg[i].thread, NULL, adjust_tree_thread, &arg[i]);
+ }
+
+ for (unsigned i = 0; i < threads; i++) {
+ if (arg[i].ret != -KNOT_EAGAIN) {
+ pthread_join(arg[i].thread, NULL);
+ }
+ if (ret == KNOT_EOK) {
+ ret = arg[i].ret;
+ }
+ }
+
+ return ret;
+}
+
int zone_adjust_contents(zone_contents_t *zone, adjust_cb_t nodes_cb, adjust_cb_t nsec3_cb,
- bool measure_zone, zone_tree_t *add_changed)
+ bool measure_zone, bool adjust_prevs, unsigned threads,
+ zone_tree_t *add_changed)
{
int ret = zone_contents_load_nsec3param(zone);
if (ret != KNOT_EOK) {
measure_t m = knot_measure_init(measure_zone, false);
adjust_ctx_t ctx = { zone, add_changed, true };
- if (nsec3_cb != NULL) {
- ret = zone_adjust_tree(zone->nsec3_nodes, &ctx, nsec3_cb, true, &m);
- }
- if (ret == KNOT_EOK && nodes_cb != NULL) {
- ret = zone_adjust_tree(zone->nodes, &ctx, nodes_cb, true, &m);
+ if (threads > 1) {
+ assert(nodes_cb != adjust_cb_flags); // This cb demands parent to be adjusted before child
+ // => required sequential adjusting (also true for
+ // adjust_cb_flags_and_nsec3) !!
+ assert(!measure_zone);
+ assert(!adjust_prevs);
+ assert(add_changed == NULL);
+ if (nsec3_cb != NULL) {
+ ret = zone_adjust_tree_parallel(zone->nsec3_nodes, &ctx, nsec3_cb, threads);
+ }
+ if (ret == KNOT_EOK && nodes_cb != NULL) {
+ ret = zone_adjust_tree_parallel(zone->nodes, &ctx, nodes_cb, threads);
+ }
+ } else {
+ if (nsec3_cb != NULL) {
+ ret = zone_adjust_tree(zone->nsec3_nodes, &ctx, nsec3_cb, adjust_prevs, &m);
+ }
+ if (ret == KNOT_EOK && nodes_cb != NULL) {
+ ret = zone_adjust_tree(zone->nodes, &ctx, nodes_cb, adjust_prevs, &m);
+ }
}
+
if (ret == KNOT_EOK && measure_zone && nodes_cb != NULL && nsec3_cb != NULL) {
knot_measure_finish_zone(&m, zone);
}
return ret;
}
-int zone_adjust_full(zone_contents_t *zone)
+int zone_adjust_full(zone_contents_t *zone, unsigned threads)
{
- int ret = zone_adjust_contents(zone, adjust_cb_flags, adjust_cb_nsec3_flags, true, NULL);
+ int ret = zone_adjust_contents(zone, adjust_cb_flags, adjust_cb_nsec3_flags,
+ true, true, 1, NULL);
if (ret == KNOT_EOK) {
- ret = zone_adjust_contents(zone, adjust_cb_nsec3_and_additionals, NULL, false, NULL);
+ ret = zone_adjust_contents(zone, adjust_cb_nsec3_and_additionals, NULL,
+ false, false, threads, NULL);
}
if (ret == KNOT_EOK) {
additionals_tree_free(zone->adds_tree);
bool nsec3change = zone_update_changed_nsec3param(update);
adjust_ctx_t ctx = { update->new_cont, update->a_ctx->adjust_ptrs, nsec3change };
- ret = zone_adjust_contents(update->new_cont, adjust_cb_flags, adjust_cb_nsec3_flags, false, update->a_ctx->adjust_ptrs);
+ ret = zone_adjust_contents(update->new_cont, adjust_cb_flags, adjust_cb_nsec3_flags,
+ false, true, 1, update->a_ctx->adjust_ptrs);
if (ret == KNOT_EOK) {
if (nsec3change) {
- ret = zone_adjust_contents(update->new_cont, adjust_cb_wildcard_nsec3, adjust_cb_void, true, update->a_ctx->adjust_ptrs);
+ ret = zone_adjust_contents(update->new_cont, adjust_cb_wildcard_nsec3, adjust_cb_void, true, false, 1, update->a_ctx->adjust_ptrs);
} else {
ret = zone_adjust_update(update, adjust_cb_wildcard_nsec3, adjust_cb_void, true);
}
}
if (ret == KNOT_EOK) {
if (nsec3change) {
- ret = zone_adjust_contents(update->new_cont, adjust_cb_nsec3_pointer, adjust_cb_void, false, update->a_ctx->adjust_ptrs);
+ ret = zone_adjust_contents(update->new_cont, adjust_cb_nsec3_pointer, adjust_cb_void, false, false, 1, update->a_ctx->adjust_ptrs);
} else {
ret = additionals_reverse_apply_multi(
update->new_cont->adds_tree,
* \param nodes_cb Callback for NORMAL nodes.
* \param nsec3_cb Callback for NSEC3 nodes.
* \param measure_zone While adjusting, count the size and max TTL of the zone.
+ * \param adjust_prevs Also (re-)generate node->prev pointers.
+ * \param threads Operate in parallel using specified threads.
* \param add_changed Special tree to add any changed node (by adjusting) into.
*
* \return KNOT_E*
*/
int zone_adjust_contents(zone_contents_t *zone, adjust_cb_t nodes_cb, adjust_cb_t nsec3_cb,
- bool measure_zone, zone_tree_t *add_changed);
+ bool measure_zone, bool adjust_prevs, unsigned threads,
+ zone_tree_t *add_changed);
/*!
* \brief Apply callback to nodes affected by the zone update.
* This operates in two phases, first fix basic node flags and prev pointers,
* than nsec3-related pointers and additionals.
*
- * \param zone Zone to be adjusted.
+ * \param zone Zone to be adjusted.
+ * \param threads Parallelize some adjusting using specified threads.
*
* \return KNOT_E*
*/
-int zone_adjust_full(zone_contents_t *zone);
+int zone_adjust_full(zone_contents_t *zone, unsigned threads);
/*!
* \brief Do a generally approved adjust after incremental update.
char *zonefile = conf_zonefile(conf, zone_name);
conf_val_t val = conf_zone_get(conf, C_SEM_CHECKS, zone_name);
+ conf_val_t thr = conf_zone_get(conf, C_ADJUST_THR, zone_name);
zloader_t zl;
- int ret = zonefile_open(&zl, zonefile, zone_name, conf_bool(&val), time(NULL));
+ int ret = zonefile_open(&zl, zonefile, zone_name, conf_bool(&val), conf_int(&thr), time(NULL));
free(zonefile);
if (ret != KNOT_EOK) {
return ret;
}
int zonefile_open(zloader_t *loader, const char *source,
- const knot_dname_t *origin, bool semantic_checks, time_t time)
+ const knot_dname_t *origin, bool semantic_checks,
+ unsigned adjusting_threads, time_t time)
{
if (!loader) {
return KNOT_EINVAL;
loader->source = strdup(source);
loader->creator = zc;
loader->semantic_checks = semantic_checks;
+ loader->adjust_threads = adjusting_threads;
loader->time = time;
return KNOT_EOK;
goto fail;
}
- ret = zone_adjust_contents(zc->z, adjust_cb_flags_and_nsec3, adjust_cb_nsec3_flags, true, NULL);
+ ret = zone_adjust_contents(zc->z, adjust_cb_flags_and_nsec3, adjust_cb_nsec3_flags, true, true, 1, NULL);
if (ret != KNOT_EOK) {
ERROR(zname, "failed to finalize zone contents (%s)",
knot_strerror(ret));
/* The contents will now change possibly messing up NSEC3 tree, it will
be adjusted again at zone_update_commit. */
- ret = zone_adjust_contents(zc->z, unadjust_cb_point_to_nsec3, NULL, false, NULL);
+ ret = zone_adjust_contents(zc->z, unadjust_cb_point_to_nsec3, NULL, false, false, loader->adjust_threads, NULL);
if (ret != KNOT_EOK) {
ERROR(zname, "failed to finalize zone contents (%s)",
knot_strerror(ret));
typedef struct {
char *source; /*!< Zone source file. */
bool semantic_checks; /*!< Do semantic checks. */
+ unsigned adjust_threads; /*!< Parallelize adjusting. */
sem_handler_t *err_handler; /*!< Semantic checks error handler. */
zcreator_t *creator; /*!< Loader context. */
zs_scanner_t scanner; /*!< Zone scanner. */
* \param source Source file name.
* \param origin Zone origin.
* \param semantic_checks Perform semantic checks.
+ * \param adjusting_threads Paralellize adjusting.
* \param time Time for semantic check.
*
* \retval Initialized loader on success.
* \retval NULL on error.
*/
int zonefile_open(zloader_t *loader, const char *source,
- const knot_dname_t *origin, bool semantic_checks, time_t time);
+ const knot_dname_t *origin, bool semantic_checks,
+ unsigned adjusting_threads, time_t time);
/*!
* \brief Loads zone from a zone file.
};
zloader_t zl;
- int ret = zonefile_open(&zl, zone_file, zone_name, true, time);
+ int ret = zonefile_open(&zl, zone_file, zone_name, true, 1, time);
if (ret != KNOT_EOK) {
return ret;
}
s.item_str("storage", self.dir)
s.item_str("zonefile-sync", self.zonefile_sync)
s.item_str("journal-max-usage", self.journal_max_usage)
+ s.item_str("adjust-threads", str(random.randint(1,4)))
s.item_str("semantic-checks", "on" if self.semantic_check else "off")
if len(self.modules) > 0:
modules = ""
-/* Copyright (C) 2019 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+/* Copyright (C) 2020 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
knot_rrset_free(soa, mm);
/* Bake the zone. */
- (void)zone_adjust_full(root->contents);
+ (void)zone_adjust_full(root->contents, 1);
/* Switch zone db. */
knot_zonedb_free(&server->zone_db);
-/* Copyright (C) 2019 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+/* Copyright (C) 2020 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
size_t zone_size1 = zone->contents->size;
uint32_t zone_max_ttl1 = zone->contents->max_ttl;
- ret = zone_adjust_full(zone->contents);
+ ret = zone_adjust_full(zone->contents, 2);
ok(ret == KNOT_EOK, "zone adjust full shall work");
size_t zone_size2 = zone->contents->size;
uint32_t zone_max_ttl2 = zone->contents->max_ttl;