Refactor zone dumping code to use netmgr async threadpools

Previously, dumping the zones to the files were quantized, so it doesn't
slow down network IO processing.  With the introduction of network
manager asynchronous threadpools, we can move the IO intensive work to
use that API and we don't have to quantize the work anymore as it the
file IO won't block anything except other zone dumping processes.

diff --git a/bin/named/server.c b/bin/named/server.c
index 07f853fa959e48556e08dc129dee3a52a112a90a..ba0757348eb995d2451fb5c9943d72f14f2a0f49 100644 (file)
--- a/bin/named/server.c
+++ b/bin/named/server.c
@@ -11487,7 +11487,7 @@ resume:
                                ";\n; Cache dump of view '%s' (cache %s)\n;\n",
                                dctx->view->view->name,
                                dns_cache_getname(dctx->view->view->cache));
-                       result = dns_master_dumptostreaminc(
+                       result = dns_master_dumptostreamasync(
                                dctx->mctx, dctx->cache, NULL, style, dctx->fp,
                                dctx->task, dumpdone, dctx, &dctx->mdctx);
                        if (result == DNS_R_CONTINUE) {
@@ -11547,7 +11547,7 @@ resume:
                                goto nextzone;
                        }
                        dns_db_currentversion(dctx->db, &dctx->version);
-                       result = dns_master_dumptostreaminc(
+                       result = dns_master_dumptostreamasync(
                                dctx->mctx, dctx->db, dctx->version, style,
                                dctx->fp, dctx->task, dumpdone, dctx,
                                &dctx->mdctx);

diff --git a/lib/dns/include/dns/masterdump.h b/lib/dns/include/dns/masterdump.h
index fcfc2e2e741e20405c577548a73f89eaa6802825..dbb8ec8026648182f5a29c4ce603a8f840b81639 100644 (file)
--- a/lib/dns/include/dns/masterdump.h
+++ b/lib/dns/include/dns/masterdump.h
@@ -243,11 +243,11 @@ dns_dumpctx_db(dns_dumpctx_t *dctx);
 
 /*@{*/
 isc_result_t
-dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
-                          dns_dbversion_t *         version,
-                          const dns_master_style_t *style, FILE *f,
-                          isc_task_t *task, dns_dumpdonefunc_t done,
-                          void *done_arg, dns_dumpctx_t **dctxp);
+dns_master_dumptostreamasync(isc_mem_t *mctx, dns_db_t *db,
+                            dns_dbversion_t *         version,
+                            const dns_master_style_t *style, FILE *f,
+                            isc_task_t *task, dns_dumpdonefunc_t done,
+                            void *done_arg, dns_dumpctx_t **dctxp);
 
 isc_result_t
 dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
@@ -259,11 +259,6 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
  * 'format'.  If the format is dns_masterformat_text (the RFC1035 format),
  * 'style' specifies the file style (e.g., &dns_master_style_default).
  *
- * dns_master_dumptostream() is an old form of dns_master_dumptostream3(),
- * which always specifies the dns_masterformat_text format.
- * dns_master_dumptostream2() is an old form which always specifies
- * a NULL header.
- *
  * If 'format' is dns_masterformat_raw, then 'header' can contain
  * information to be written to the file header.
  *
@@ -276,7 +271,6 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
  *
  * Returns:
  *\li  ISC_R_SUCCESS
- *\li  ISC_R_CONTINUE  dns_master_dumptostreaminc() only.
  *\li  ISC_R_NOMEMORY
  *\li  Any database or rrset iterator error.
  *\li  Any dns_rdata_totext() error code.
@@ -286,11 +280,11 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
 /*@{*/
 
 isc_result_t
-dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
-                  const dns_master_style_t *style, const char *filename,
-                  isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
-                  dns_dumpctx_t **dctxp, dns_masterformat_t format,
-                  dns_masterrawheader_t *header);
+dns_master_dumpasync(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
+                    const dns_master_style_t *style, const char *filename,
+                    isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
+                    dns_dumpctx_t **dctxp, dns_masterformat_t format,
+                    dns_masterrawheader_t *header);
 
 isc_result_t
 dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
@@ -302,11 +296,6 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
  * 'format'.  If the format is dns_masterformat_text (the RFC1035 format),
  * 'style' specifies the file style (e.g., &dns_master_style_default).
  *
- * dns_master_dumpinc() and dns_master_dump() are old forms of _dumpinc3()
- * and _dump3(), respectively, which always specify the dns_masterformat_text
- * format.  dns_master_dumpinc2() and dns_master_dump2() are old forms which
- * always specify a NULL header.
- *
  * If 'format' is dns_masterformat_raw, then 'header' can contain
  * information to be written to the file header.
  *
@@ -314,7 +303,6 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
  *
  * Returns:
  *\li  ISC_R_SUCCESS
- *\li  ISC_R_CONTINUE  dns_master_dumpinc() only.
  *\li  ISC_R_NOMEMORY
  *\li  Any database or rrset iterator error.
  *\li  Any dns_rdata_totext() error code.

diff --git a/lib/dns/masterdump.c b/lib/dns/masterdump.c
index c66eb7ccebc19f2e10ec258fe9746c35c6ffa02a..0c8a400196ed56dd8aa4a9d08596dfa33f86ac43 100644 (file)
--- a/lib/dns/masterdump.c
+++ b/lib/dns/masterdump.c
@@ -265,8 +265,8 @@ struct dns_dumpctx {
        isc_task_t *task;
        dns_dumpdonefunc_t done;
        void *done_arg;
-       unsigned int nodes;
-       /* dns_master_dumpinc() */
+       /* dns_master_dumpasync() */
+       isc_result_t result;
        char *file;
        char *tmpfile;
        dns_masterformat_t format;
@@ -1343,7 +1343,7 @@ dump_rdatasets_map(isc_mem_t *mctx, const dns_name_t *name,
 static const int initial_buffer_length = 1200;
 
 static isc_result_t
-dumptostreaminc(dns_dumpctx_t *dctx);
+dumptostream(dns_dumpctx_t *dctx);
 
 static void
 dumpctx_destroy(dns_dumpctx_t *dctx) {
@@ -1486,27 +1486,23 @@ closeandrename(FILE *f, isc_result_t result, const char *temp,
        return (result);
 }
 
+/*
+ * This will run in a libuv threadpool thread.
+ */
 static void
-dump_quantum(isc_task_t *task, isc_event_t *event) {
-       isc_result_t result;
-       isc_result_t tresult;
-       dns_dumpctx_t *dctx;
-
-       REQUIRE(event != NULL);
-       dctx = event->ev_arg;
+master_dump_cb(void *data) {
+       isc_result_t result = ISC_R_UNSET;
+       dns_dumpctx_t *dctx = data;
        REQUIRE(DNS_DCTX_VALID(dctx));
+
        if (atomic_load_acquire(&dctx->canceled)) {
                result = ISC_R_CANCELED;
        } else {
-               result = dumptostreaminc(dctx);
-       }
-       if (result == DNS_R_CONTINUE) {
-               event->ev_arg = dctx;
-               isc_task_send(task, &event);
-               return;
+               result = dumptostream(dctx);
        }
 
        if (dctx->file != NULL) {
+               isc_result_t tresult = ISC_R_UNSET;
                tresult = closeandrename(dctx->f, result, dctx->tmpfile,
                                         dctx->file);
                if (tresult != ISC_R_SUCCESS && result == ISC_R_SUCCESS) {
@@ -1515,17 +1511,51 @@ dump_quantum(isc_task_t *task, isc_event_t *event) {
        } else {
                result = flushandsync(dctx->f, result, NULL);
        }
+
+       dctx->result = result;
+}
+
+/*
+ * This will run in a network/task manager thread when the dump is complete.
+ */
+static void
+master_dump_done_cb(void *data, isc_result_t result) {
+       dns_dumpctx_t *dctx = data;
+
+       if (result == ISC_R_SUCCESS && dctx->result != ISC_R_SUCCESS) {
+               result = dctx->result;
+       }
+
        (dctx->done)(dctx->done_arg, result);
-       isc_event_free(&event);
        dns_dumpctx_detach(&dctx);
 }
 
+/*
+ * This must be run from a network/task manager thread.
+ */
+static void
+setup_dump(isc_task_t *task, isc_event_t *event) {
+       dns_dumpctx_t *dctx = NULL;
+
+       REQUIRE(isc_nm_tid() >= 0);
+       REQUIRE(event != NULL);
+
+       dctx = event->ev_arg;
+
+       REQUIRE(DNS_DCTX_VALID(dctx));
+
+       isc_nm_work_offload(isc_task_getnetmgr(task), master_dump_cb,
+                           master_dump_done_cb, dctx);
+
+       isc_event_free(&event);
+}
+
 static isc_result_t
 task_send(dns_dumpctx_t *dctx) {
        isc_event_t *event;
 
        event = isc_event_allocate(dctx->mctx, NULL, DNS_EVENT_DUMPQUANTUM,
-                                  dump_quantum, dctx, sizeof(*event));
+                                  setup_dump, dctx, sizeof(*event));
        isc_task_send(dctx->task, &event);
        return (ISC_R_SUCCESS);
 }
@@ -1548,7 +1578,6 @@ dumpctx_create(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
        dctx->done = NULL;
        dctx->done_arg = NULL;
        dctx->task = NULL;
-       dctx->nodes = 0;
        dctx->first = true;
        atomic_init(&dctx->canceled, false);
        dctx->file = NULL;
@@ -1702,13 +1731,12 @@ writeheader(dns_dumpctx_t *dctx) {
 }
 
 static isc_result_t
-dumptostreaminc(dns_dumpctx_t *dctx) {
+dumptostream(dns_dumpctx_t *dctx) {
        isc_result_t result = ISC_R_SUCCESS;
        isc_buffer_t buffer;
        char *bufmem;
        dns_name_t *name;
        dns_fixedname_t fixname;
-       unsigned int nodes;
        isc_time_t start;
 
        bufmem = isc_mem_get(dctx->mctx, initial_buffer_length);
@@ -1742,9 +1770,8 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
                result = ISC_R_SUCCESS;
        }
 
-       nodes = dctx->nodes;
        isc_time_now(&start);
-       while (result == ISC_R_SUCCESS && (dctx->nodes == 0 || nodes--)) {
+       while (result == ISC_R_SUCCESS) {
                dns_rdatasetiter_t *rdsiter = NULL;
                dns_dbnode_t *node = NULL;
 
@@ -1780,52 +1807,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
                result = dns_dbiterator_next(dctx->dbiter);
        }
 
-       /*
-        * Work out how many nodes can be written in the time between
-        * two requests to the nameserver.  Smooth the resulting number and
-        * use it as a estimate for the number of nodes to be written in the
-        * next iteration.
-        */
-       if (dctx->nodes != 0 && result == ISC_R_SUCCESS) {
-               unsigned int pps = dns_pps; /* packets per second */
-               unsigned int interval;
-               uint64_t usecs;
-               isc_time_t end;
-
-               isc_time_now(&end);
-               if (pps < 100) {
-                       pps = 100;
-               }
-               interval = 1000000 / pps; /* interval in usecs */
-               if (interval == 0) {
-                       interval = 1;
-               }
-               usecs = isc_time_microdiff(&end, &start);
-               if (usecs == 0) {
-                       dctx->nodes = dctx->nodes * 2;
-                       if (dctx->nodes > 1000) {
-                               dctx->nodes = 1000;
-                       }
-               } else {
-                       nodes = dctx->nodes * interval;
-                       nodes /= (unsigned int)usecs;
-                       if (nodes == 0) {
-                               nodes = 1;
-                       } else if (nodes > 1000) {
-                               nodes = 1000;
-                       }
-
-                       /* Smooth and assign. */
-                       dctx->nodes = (nodes + dctx->nodes * 7) / 8;
-
-                       isc_log_write(dns_lctx, ISC_LOGCATEGORY_GENERAL,
-                                     DNS_LOGMODULE_MASTERDUMP,
-                                     ISC_LOG_DEBUG(1),
-                                     "dumptostreaminc(%p) new nodes -> %d",
-                                     dctx, dctx->nodes);
-               }
-               result = DNS_R_CONTINUE;
-       } else if (result == ISC_R_NOMORE) {
+       if (result == ISC_R_NOMORE) {
                result = ISC_R_SUCCESS;
        }
 cleanup:
@@ -1835,11 +1817,11 @@ cleanup:
 }
 
 isc_result_t
-dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
-                          dns_dbversion_t *version,
-                          const dns_master_style_t *style, FILE *f,
-                          isc_task_t *task, dns_dumpdonefunc_t done,
-                          void *done_arg, dns_dumpctx_t **dctxp) {
+dns_master_dumptostreamasync(isc_mem_t *mctx, dns_db_t *db,
+                            dns_dbversion_t *version,
+                            const dns_master_style_t *style, FILE *f,
+                            isc_task_t *task, dns_dumpdonefunc_t done,
+                            void *done_arg, dns_dumpctx_t **dctxp) {
        dns_dumpctx_t *dctx = NULL;
        isc_result_t result;
 
@@ -1855,7 +1837,6 @@ dns_master_dumptostreaminc(isc_mem_t *mctx, dns_db_t *db,
        isc_task_attach(task, &dctx->task);
        dctx->done = done;
        dctx->done_arg = done_arg;
-       dctx->nodes = 100;
 
        result = task_send(dctx);
        if (result == ISC_R_SUCCESS) {
@@ -1881,7 +1862,7 @@ dns_master_dumptostream(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
                return (result);
        }
 
-       result = dumptostreaminc(dctx);
+       result = dumptostream(dctx);
        INSIST(result != DNS_R_CONTINUE);
        dns_dumpctx_detach(&dctx);
 
@@ -1927,11 +1908,11 @@ cleanup:
 }
 
 isc_result_t
-dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
-                  const dns_master_style_t *style, const char *filename,
-                  isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
-                  dns_dumpctx_t **dctxp, dns_masterformat_t format,
-                  dns_masterrawheader_t *header) {
+dns_master_dumpasync(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
+                    const dns_master_style_t *style, const char *filename,
+                    isc_task_t *task, dns_dumpdonefunc_t done, void *done_arg,
+                    dns_dumpctx_t **dctxp, dns_masterformat_t format,
+                    dns_masterrawheader_t *header) {
        FILE *f = NULL;
        isc_result_t result;
        char *tempname = NULL;
@@ -1956,7 +1937,6 @@ dns_master_dumpinc(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
        isc_task_attach(task, &dctx->task);
        dctx->done = done;
        dctx->done_arg = done_arg;
-       dctx->nodes = 100;
        dctx->file = file;
        file = NULL;
        dctx->tmpfile = tempname;
@@ -2001,7 +1981,7 @@ dns_master_dump(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
                goto cleanup;
        }
 
-       result = dumptostreaminc(dctx);
+       result = dumptostream(dctx);
        INSIST(result != DNS_R_CONTINUE);
        dns_dumpctx_detach(&dctx);
 

diff --git a/lib/dns/win32/libdns.def.in b/lib/dns/win32/libdns.def.in
index ab01b696df02351f6093479c7610d89cf48a98be..97208946ef67a2a9e8d08c6c8816bb04724979bd 100644 (file)
--- a/lib/dns/win32/libdns.def.in
+++ b/lib/dns/win32/libdns.def.in
@@ -493,11 +493,11 @@ dns_lookup_cancel
 dns_lookup_create
 dns_lookup_destroy
 dns_master_dump
-dns_master_dumpinc
+dns_master_dumpasync
 dns_master_dumpnode
 dns_master_dumpnodetostream
 dns_master_dumptostream
-dns_master_dumptostreaminc
+dns_master_dumptostreamasync
 dns_master_initrawheader
 dns_master_loadbuffer
 dns_master_loadbufferinc

diff --git a/lib/dns/zone.c b/lib/dns/zone.c
index 642434361375dd6ae94a40a18d5e753f2ea34015..c4788ea203186231127d6d06679d4dbd6c2e4d6e 100644 (file)
--- a/lib/dns/zone.c
+++ b/lib/dns/zone.c
@@ -2549,7 +2549,7 @@ zone_gotwritehandle(isc_task_t *task, isc_event_t *event) {
                } else {
                        output_style = &dns_master_style_default;
                }
-               result = dns_master_dumpinc(
+               result = dns_master_dumpasync(
                        zone->mctx, db, version, output_style, zone->masterfile,
                        zone->task, dump_done, zone, &zone->dctx,
                        zone->masterformat, &rawdata);