1911. [func] Attempt to make the amount of work performed in a

                        iteration self tuning.  The covers nodes clean from
                        the cache per iteration, nodes written to disk when
                        rewriting a master file and nodes destroyed per
                        iteration when destroying a zone or a cache.
                        [RT #14996]

diff --git a/CHANGES b/CHANGES
index 0c88cd717f6f51de5f39ec21f625ad1b790f9efd..894c380405edfc183245458a44e723e248053484 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,10 @@
+1911.  [func]          Attempt to make the amount of work performed in a
+                       iteration self tuning.  The covers nodes clean from
+                       the cache per iteration, nodes written to disk when
+                       rewriting a master file and nodes destroyed per
+                       iteration when destroying a zone or a cache.
+                       [RT #14996]
+
 1910.  [cleanup]       Don't add DNSKEY records to the additional section.
 
 1909.  [bug]           ixfr-from-differences failed to ensure that the

diff --git a/bin/named/client.c b/bin/named/client.c
index f999545beb8a0b41b303895e2d33a17055864e79..4b16228aac0172684986e36aa2ac55fd36797822 100644 (file)
--- a/bin/named/client.c
+++ b/bin/named/client.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: client.c,v 1.229 2005/07/27 02:28:58 marka Exp $ */
+/* $Id: client.c,v 1.230 2005/08/15 01:21:03 marka Exp $ */
 
 #include <config.h>
 
@@ -168,6 +168,7 @@ struct ns_clientmgr {
  * Must be greater than any valid state.
  */
 
+unsigned int ns_client_requests;
 
 static void client_read(ns_client_t *client);
 static void client_accept(ns_client_t *client);
@@ -1213,6 +1214,8 @@ client_request(isc_task_t *task, isc_event_t *event) {
               NS_CLIENTSTATE_READING :
               NS_CLIENTSTATE_READY);
 
+       ns_client_requests++;
+
        if (event->ev_type == ISC_SOCKEVENT_RECVDONE) {
                INSIST(!TCP_CLIENT(client));
                sevent = (isc_socketevent_t *)event;

diff --git a/bin/named/include/named/client.h b/bin/named/include/named/client.h
index 334c813cd3b70e13aae49bea0920894cc8abe8cf..44e25668a8fd156d2a34e77a35ae05d739ff0d60 100644 (file)
--- a/bin/named/include/named/client.h
+++ b/bin/named/include/named/client.h
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: client.h,v 1.76 2005/07/27 02:28:59 marka Exp $ */
+/* $Id: client.h,v 1.77 2005/08/15 01:21:04 marka Exp $ */
 
 #ifndef NAMED_CLIENT_H
 #define NAMED_CLIENT_H 1
@@ -160,6 +160,7 @@ struct ns_client {
 #define NS_CLIENTATTR_MULTICAST                0x08 /*%< recv'd from multicast */
 #define NS_CLIENTATTR_WANTDNSSEC       0x10 /*%< include dnssec records */
 
+extern unsigned int ns_client_requests;
 
 /***
  *** Functions

diff --git a/bin/named/include/named/server.h b/bin/named/include/named/server.h
index 9ae6d9d110de64103905be5c86ab5d6ef138ba52..fc1a0712e51e557844654f8d69317a441a94293f 100644 (file)
--- a/bin/named/include/named/server.h
+++ b/bin/named/include/named/server.h
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: server.h,v 1.78 2005/04/29 00:22:32 marka Exp $ */
+/* $Id: server.h,v 1.79 2005/08/15 01:21:05 marka Exp $ */
 
 #ifndef NAMED_SERVER_H
 #define NAMED_SERVER_H 1
@@ -82,6 +82,8 @@ struct ns_server {
 
        isc_timer_t *           interface_timer;
        isc_timer_t *           heartbeat_timer;
+       isc_timer_t *           pps_timer;
+
        isc_uint32_t            interface_interval;
        isc_uint32_t            heartbeat_interval;
 

diff --git a/bin/named/server.c b/bin/named/server.c
index 39ff745f7453e5862e0a95691c52943c0bd3427e..96435cb42dcf925428a866adceb22e6556cd1421 100644 (file)
--- a/bin/named/server.c
+++ b/bin/named/server.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: server.c,v 1.446 2005/07/27 02:28:58 marka Exp $ */
+/* $Id: server.c,v 1.447 2005/08/15 01:21:04 marka Exp $ */
 
 /*! \file */
 
@@ -51,6 +51,7 @@
 #include <dns/forward.h>
 #include <dns/journal.h>
 #include <dns/keytable.h>
+#include <dns/lib.h>
 #include <dns/master.h>
 #include <dns/masterdump.h>
 #include <dns/order.h>
@@ -2099,6 +2100,21 @@ heartbeat_timer_tick(isc_task_t *task, isc_event_t *event) {
        }
 }
 
+static void
+pps_timer_tick(isc_task_t *task, isc_event_t *event) {
+       static unsigned int oldrequests = 0;
+       unsigned int requests = ns_client_requests;
+
+       UNUSED(task);
+       isc_event_free(&event);
+
+       /*
+        * Don't worry about wrapping as the overflow result will be right.
+        */
+       dns_pps = (requests - oldrequests) / 1200;
+       oldrequests = requests;
+}
+
 /*
  * Replace the current value of '*field', a dynamically allocated
  * string or NULL, with a dynamically allocated copy of the
@@ -2204,6 +2220,7 @@ load_configuration(const char *filename, ns_server_t *server,
                   isc_boolean_t first_time)
 {
        isc_result_t result;
+       isc_interval_t interval;
        cfg_parser_t *parser = NULL;
        cfg_obj_t *config;
        cfg_obj_t *options;
@@ -2479,7 +2496,6 @@ load_configuration(const char *filename, ns_server_t *server,
                                      isc_timertype_inactive,
                                      NULL, NULL, ISC_TRUE));
        } else if (server->interface_interval != interface_interval) {
-               isc_interval_t interval;
                isc_interval_set(&interval, interface_interval, 0);
                CHECK(isc_timer_reset(server->interface_timer,
                                      isc_timertype_ticker,
@@ -2499,13 +2515,16 @@ load_configuration(const char *filename, ns_server_t *server,
                                      isc_timertype_inactive,
                                      NULL, NULL, ISC_TRUE));
        } else if (server->heartbeat_interval != heartbeat_interval) {
-               isc_interval_t interval;
                isc_interval_set(&interval, heartbeat_interval, 0);
                CHECK(isc_timer_reset(server->heartbeat_timer,
                                      isc_timertype_ticker,
                                      NULL, &interval, ISC_FALSE));
        }
        server->heartbeat_interval = heartbeat_interval;
+       
+       isc_interval_set(&interval, 1200, 0);
+       CHECK(isc_timer_reset(server->pps_timer, isc_timertype_ticker, NULL,
+                             &interval, ISC_FALSE));
 
        /*
         * Configure and freeze all explicit views.  Explicit
@@ -2938,6 +2957,12 @@ run_server(isc_task_t *task, isc_event_t *event) {
                                    server, &server->heartbeat_timer),
                   "creating heartbeat timer");
 
+       CHECKFATAL(isc_timer_create(ns_g_timermgr, isc_timertype_inactive,
+                                   NULL, NULL, server->task,
+                                   heartbeat_timer_tick,
+                                   server, &server->pps_timer),
+                  "creating pps timer");
+
        CHECKFATAL(cfg_parser_create(ns_g_mctx, NULL, &ns_g_parser),
                   "creating default configuration parser");
 
@@ -3002,6 +3027,7 @@ shutdown_server(isc_task_t *task, isc_event_t *event) {
 
        isc_timer_detach(&server->interface_timer);
        isc_timer_detach(&server->heartbeat_timer);
+       isc_timer_detach(&server->pps_timer);
 
        ns_interfacemgr_shutdown(server->interfacemgr);
        ns_interfacemgr_detach(&server->interfacemgr);
@@ -3090,6 +3116,7 @@ ns_server_create(isc_mem_t *mctx, ns_server_t **serverp) {
 
        server->interface_timer = NULL;
        server->heartbeat_timer = NULL;
+       server->pps_timer = NULL;
        
        server->interface_interval = 0;
        server->heartbeat_interval = 0;

diff --git a/lib/dns/cache.c b/lib/dns/cache.c
index 16986d883f02e80170cd16f4886a2e5950a9d2d1..0d3bd2a6b60868b222e2609313cec087a22ab60b 100644 (file)
--- a/lib/dns/cache.c
+++ b/lib/dns/cache.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: cache.c,v 1.61 2005/07/12 01:00:14 marka Exp $ */
+/* $Id: cache.c,v 1.62 2005/08/15 01:21:05 marka Exp $ */
 
 /*! \file */
 
@@ -31,6 +31,7 @@
 #include <dns/db.h>
 #include <dns/dbiterator.h>
 #include <dns/events.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/masterdump.h>
 #include <dns/rdata.h>
@@ -52,7 +53,7 @@
  * CLEANERINCREMENT is how many nodes are examined in one pass.
  * See also DNS_CACHE_MINSIZE 
  */
-#define DNS_CACHE_CLEANERINCREMENT     1000    /*%< Number of nodes. */
+#define DNS_CACHE_CLEANERINCREMENT     1000U   /*%< Number of nodes. */
 
 /***
  ***   Types
@@ -104,7 +105,7 @@ struct cache_cleaner {
        isc_event_t     *overmem_event;
 
        dns_dbiterator_t *iterator;
-       int              increment;     /*% Number of names to
+       unsigned int     increment;     /*% Number of names to
                                           clean in one increment */
        cleaner_state_t  state;         /*% Idle/Busy. */
        isc_boolean_t    overmem;       /*% The cache is in an overmem state. */
@@ -156,6 +157,64 @@ cleaner_shutdown_action(isc_task_t *task, isc_event_t *event);
 static void
 overmem_cleaning_action(isc_task_t *task, isc_event_t *event);
 
+static void
+adjust_increment(cache_cleaner_t *cleaner, unsigned int remaining,
+                isc_time_t *start)
+{
+       isc_time_t end;
+       isc_uint64_t usecs;
+       isc_uint64_t new;
+       unsigned int pps = dns_pps;
+       unsigned int interval;
+       unsigned int names;
+       
+       /*
+        * Tune for minumum of 100 pps.
+        */
+       if (pps < 100)
+               pps = 100;
+
+       isc_time_now(&end);
+
+       interval = 1000000 / pps;
+       if (interval == 0)
+               interval = 1;
+
+       INSIST(cleaner->increment >= remaining);
+       names = cleaner->increment - remaining;
+       usecs = isc_time_microdiff(&end, start);
+
+       isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
+                     ISC_LOG_INFO, "adjust_increment interval=%u "
+                     "names=%u usec=%" ISC_PLATFORM_QUADFORMAT "u",
+                     interval, names, usecs);
+       
+       if (usecs == 0) {
+               if (names == cleaner->increment) {
+                       cleaner->increment *= 2;
+                       if (cleaner->increment > DNS_CACHE_CLEANERINCREMENT)
+                               cleaner->increment = DNS_CACHE_CLEANERINCREMENT;
+                       isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE,
+                                     DNS_LOGMODULE_CACHE, ISC_LOG_INFO,
+                                     "new clear->increment = %d\n",
+                                     cleaner->increment);
+               }
+               return;
+       }
+
+       new = (names * interval);
+       new /= usecs;
+       if (new == 0)
+               new = 1;
+       else if (new > DNS_CACHE_CLEANERINCREMENT)
+               new = DNS_CACHE_CLEANERINCREMENT;
+
+       cleaner->increment = new;
+       isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
+                     ISC_LOG_INFO, "new clear->increment = %u\n",
+                     cleaner->increment);
+}
+
 static inline isc_result_t
 cache_create_db(dns_cache_t *cache, dns_db_t **db) {
        return (dns_db_create(cache->mctx, cache->db_type, dns_rootname,
@@ -717,7 +776,8 @@ static void
 incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
        cache_cleaner_t *cleaner = event->ev_arg;
        isc_result_t result;
-       int n_names;
+       unsigned int n_names;
+       isc_time_t start;
 
        UNUSED(task);
 
@@ -736,6 +796,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
 
        REQUIRE(DNS_DBITERATOR_VALID(cleaner->iterator));
 
+       isc_time_now(&start);
        while (n_names-- > 0) {
                dns_dbnode_t *node = NULL;
 
@@ -746,6 +807,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
                                 "cache cleaner: dns_dbiterator_current() "
                                 "failed: %s", dns_result_totext(result));
 
+                       adjust_increment(cleaner, n_names, &start);
                        end_cleaning(cleaner, event);
                        return;
                }
@@ -789,11 +851,14 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
                                }
                        }
 
+                       adjust_increment(cleaner, n_names, &start);
                        end_cleaning(cleaner, event);
                        return;
                }
        }
 
+       adjust_increment(cleaner, 0U, &start);
+
        /*
         * We have successfully performed a cleaning increment but have
         * not gone through the entire cache.  Free the iterator locks
@@ -804,7 +869,7 @@ incremental_cleaning_action(isc_task_t *task, isc_event_t *event) {
        RUNTIME_CHECK(result == ISC_R_SUCCESS);
 
        isc_log_write(dns_lctx, DNS_LOGCATEGORY_DATABASE, DNS_LOGMODULE_CACHE,
-                     ISC_LOG_DEBUG(1), "cache cleaner: checked %d nodes, "
+                     ISC_LOG_DEBUG(1), "cache cleaner: checked %u nodes, "
                      "mem inuse %lu, sleeping", cleaner->increment,
                      (unsigned long)isc_mem_inuse(cleaner->cache->mctx));
 

diff --git a/lib/dns/include/dns/lib.h b/lib/dns/include/dns/lib.h
index 9f2bd3a63272d8a4e76085fd8d5d4dd3eef9d136..5f384fd692369be3d0623843fdeace93453ac78c 100644 (file)
--- a/lib/dns/include/dns/lib.h
+++ b/lib/dns/include/dns/lib.h
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: lib.h,v 1.10 2005/04/29 00:22:59 marka Exp $ */
+/* $Id: lib.h,v 1.11 2005/08/15 01:21:07 marka Exp $ */
 
 #ifndef DNS_LIB_H
 #define DNS_LIB_H 1
@@ -27,6 +27,7 @@
 
 ISC_LANG_BEGINDECLS
 
+LIBDNS_EXTERNAL_DATA extern unsigned int dns_pps;
 LIBDNS_EXTERNAL_DATA extern isc_msgcat_t *dns_msgcat;
 
 void

diff --git a/lib/dns/lib.c b/lib/dns/lib.c
index db7a0f6a7f85ec00f57db0a48c5385d30c783c8b..255d9b2c399eeb1a11d2872fe602dd51c3750461 100644 (file)
--- a/lib/dns/lib.c
+++ b/lib/dns/lib.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: lib.c,v 1.13 2005/04/29 00:22:48 marka Exp $ */
+/* $Id: lib.c,v 1.14 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -33,6 +33,7 @@
  *** Globals
  ***/
 
+LIBDNS_EXTERNAL_DATA unsigned int                      dns_pps = 0U;
 LIBDNS_EXTERNAL_DATA isc_msgcat_t *                    dns_msgcat = NULL;
 
 

diff --git a/lib/dns/masterdump.c b/lib/dns/masterdump.c
index 72327d95587097bccf8e088d83bf7b1d55544a0e..0cb15f94861aaf69e78088e9d45a9215225effbc 100644 (file)
--- a/lib/dns/masterdump.c
+++ b/lib/dns/masterdump.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: masterdump.c,v 1.78 2005/06/20 01:03:53 marka Exp $ */
+/* $Id: masterdump.c,v 1.79 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -31,12 +31,14 @@
 #include <isc/stdio.h>
 #include <isc/string.h>
 #include <isc/task.h>
+#include <isc/time.h>
 #include <isc/util.h>
 
 #include <dns/db.h>
 #include <dns/dbiterator.h>
 #include <dns/events.h>
 #include <dns/fixedname.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/master.h>
 #include <dns/masterdump.h>
@@ -1261,6 +1263,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
        unsigned int nodes;
        dns_masterrawheader_t rawheader;
        isc_uint32_t now32;
+       isc_time_t start;
 
        bufmem = isc_mem_get(dctx->mctx, initial_buffer_length);
        if (bufmem == NULL)
@@ -1333,6 +1336,7 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
                result = ISC_R_SUCCESS;
 
        nodes = dctx->nodes;
+       isc_time_now(&start);
        while (result == ISC_R_SUCCESS && (dctx->nodes == 0 || nodes--)) {
                dns_rdatasetiter_t *rdsiter = NULL;
                dns_dbnode_t *node = NULL;
@@ -1367,6 +1371,30 @@ dumptostreaminc(dns_dumpctx_t *dctx) {
        }
 
        if (dctx->nodes != 0 && result == ISC_R_SUCCESS) {
+               unsigned int pps = dns_pps;
+               unsigned int interval;
+               isc_uint64_t usecs;
+               isc_time_t end;
+
+               isc_time_now(&end);
+               if (pps < 100)
+                       pps = 100;
+               interval = 1000000 / pps;
+               if (interval == 0)
+                       interval = 1;
+               usecs = isc_time_microdiff(&end, &start);
+               if (usecs == 0) {
+                       dctx->nodes = dctx->nodes * 2;
+                       if (dctx->nodes > 1000)
+                               dctx->nodes = 1000;
+               } else {
+                       dctx->nodes = dctx->nodes * interval;
+                       dctx->nodes /= usecs;
+                       if (dctx->nodes == 0)
+                               dctx->nodes = 1;
+                       else if (dctx->nodes > 1000)
+                               dctx->nodes = 1000;
+               }
                dns_dbiterator_pause(dctx->dbiter);
                result = DNS_R_CONTINUE;
        } else if (result == ISC_R_NOMORE)

diff --git a/lib/dns/rbtdb.c b/lib/dns/rbtdb.c
index f48fbd322a934eb4228df23846e39d734fc8112c..86420ec9c7aa6a6712fc41652bed622aafeb649f 100644 (file)
--- a/lib/dns/rbtdb.c
+++ b/lib/dns/rbtdb.c
@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: rbtdb.c,v 1.214 2005/07/18 05:58:58 marka Exp $ */
+/* $Id: rbtdb.c,v 1.215 2005/08/15 01:21:06 marka Exp $ */
 
 /*! \file */
 
@@ -34,6 +34,7 @@
 #include <isc/rwlock.h>
 #include <isc/string.h>
 #include <isc/task.h>
+#include <isc/time.h>
 #include <isc/util.h>
 
 #include <dns/acache.h>
@@ -41,6 +42,7 @@
 #include <dns/dbiterator.h>
 #include <dns/events.h>
 #include <dns/fixedname.h>
+#include <dns/lib.h>
 #include <dns/log.h>
 #include <dns/masterdump.h>
 #include <dns/rbt.h>
@@ -346,6 +348,7 @@ typedef struct {
 
        /* Unlocked */
        isc_mem_t **                    nodemctxs;
+       unsigned int                    quantum;
 } dns_rbtdb_t;
 
 #define RBTDB_ATTR_LOADED              0x01
@@ -548,12 +551,43 @@ free_rbtdb_callback(isc_task_t *task, isc_event_t *event) {
        free_rbtdb(rbtdb, ISC_TRUE, event);
 }
 
+static unsigned int
+adjust_quantum(unsigned int old, isc_time_t *start) {
+       unsigned int pps = dns_pps;
+       unsigned int interval;
+       isc_uint64_t usecs;
+       isc_time_t end;
+
+       if (pps < 100)
+               pps = 100;
+       isc_time_now(&end);
+
+       interval = 1000000 / pps;
+       if (interval == 0)
+               interval = 1;
+       usecs = isc_time_microdiff(&end, start);
+       if (usecs == 0) {
+               old *= 2;
+               if (old > 1000)
+                       old = 1000;
+               return (old);
+       }
+       old = old * interval;
+       old /= usecs;
+       if (old == 0)
+               old = 1;
+       else if (old > 1000)
+               old = 1000;
+       return (old);
+}
+               
 static void
 free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) {
        unsigned int i;
        isc_ondestroy_t ondest;
        isc_result_t result;
        char buf[DNS_NAME_FORMATSIZE];
+       isc_time_t start;
 
        REQUIRE(rbtdb->current_version != NULL || EMPTY(rbtdb->open_versions));
        REQUIRE(rbtdb->future_version == NULL);
@@ -569,12 +603,17 @@ free_rbtdb(dns_rbtdb_t *rbtdb, isc_boolean_t log, isc_event_t *event) {
                isc_mem_put(rbtdb->common.mctx, rbtdb->current_version,
                            sizeof(rbtdb_version_t));
        }
+       if (event == NULL)
+               rbtdb->quantum = (rbtdb->task != NULL) ? 100 : 0;
  again:
        if (rbtdb->tree != NULL) {
-               result = dns_rbt_destroy2(&rbtdb->tree,
-                                         (rbtdb->task != NULL) ? 1000 : 0);
+               isc_time_now(&start);
+               result = dns_rbt_destroy2(&rbtdb->tree, rbtdb->quantum);
                if (result == ISC_R_QUOTA) {
                        INSIST(rbtdb->task != NULL);
+                       if (rbtdb->quantum != 0)
+                               rbtdb->quantum = adjust_quantum(rbtdb->quantum,
+                                                               &start);
                        if (event == NULL)
                                event = isc_event_allocate(rbtdb->common.mctx,
                                                           NULL,