Make netmgr the authority on number of threads running

Instead of passing the "workers" variable back and forth along with
passing the single isc_nm_t instance, add isc_nm_getnworkers() function
that returns the number of netmgr threads are running.

Change the ns_interfacemgr and ns_taskmgr to utilize the newly acquired
knowledge.

diff --git a/bin/named/server.c b/bin/named/server.c
index 28bc75d89e9540dcc7a7b0d7cb6c2ab2049f831d..19590f6449654549cd45b3ea27d2f89610fac1ad 100644 (file)
--- a/bin/named/server.c
+++ b/bin/named/server.c
@@ -9934,8 +9934,8 @@ run_server(isc_task_t *task, isc_event_t *event) {
        CHECKFATAL(ns_interfacemgr_create(named_g_mctx, server->sctx,
                                          named_g_taskmgr, named_g_timermgr,
                                          named_g_netmgr, named_g_dispatchmgr,
-                                         server->task, geoip, named_g_cpus,
-                                         true, &server->interfacemgr),
+                                         server->task, geoip, true,
+                                         &server->interfacemgr),
                   "creating interface manager");
 
        isc_timer_create(named_g_timermgr, server->task, interface_timer_tick,

diff --git a/lib/isc/include/isc/netmgr.h b/lib/isc/include/isc/netmgr.h
index f9e59f769212e5cd1432b88287fc2841b1b1bec8..86ac0e9ac7ef151f6d5c8fb2bd680eab9d5cf492 100644 (file)
--- a/lib/isc/include/isc/netmgr.h
+++ b/lib/isc/include/isc/netmgr.h
@@ -648,15 +648,15 @@ isc_nm_has_encryption(const isc_nmhandle_t *handle);
  */
 
 void
-isc_nm_task_enqueue(isc_nm_t *mgr, isc_task_t *task, int threadid);
+isc_nm_task_enqueue(isc_nm_t *mgr, isc_task_t *task, int tid);
 /*%<
  * Enqueue the 'task' onto the netmgr ievents queue.
  *
  * Requires:
  * \li 'mgr' is a valid netmgr object
  * \li 'task' is a valid task
- * \li 'threadid' is either the preferred netmgr tid or -1, in which case
- *     tid will be picked randomly. The threadid is capped (by modulo) to
+ * \li 'tid' is either the preferred netmgr tid or -1, in which case
+ *     tid will be picked randomly. The tid is capped (by modulo) to
  *     maximum number of 'workers' as specifed in isc_nm_start()
  */
 
@@ -680,6 +680,12 @@ isc__nm_force_tid(int tid);
  * tests and should not be used in any production code.
  */
 
+uint32_t
+isc_nm_getnworkers(const isc_nm_t *);
+/*%<
+ * Return the number of active workers
+ */
+
 void
 isc_nmhandle_setwritetimeout(isc_nmhandle_t *handle, uint64_t write_timeout);
 

diff --git a/lib/isc/include/isc/task.h b/lib/isc/include/isc/task.h
index 4d5f0080e576f67a05186ee85e46920c14cc9e92..e5251aa4be159545e2d9c11a5d5ccb44dc208e58 100644 (file)
--- a/lib/isc/include/isc/task.h
+++ b/lib/isc/include/isc/task.h
@@ -105,9 +105,9 @@ isc_task_create(isc_taskmgr_t *manager, unsigned int quantum,
                isc_task_t **taskp);
 isc_result_t
 isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
-                     isc_task_t **taskp, int threadid);
+                     isc_task_t **taskp, int tid);
 /*%<
- * Create a task, optionally bound to a particular threadid.
+ * Create a task, optionally bound to a particular tid.
  *
  * Notes:
  *

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index c405a9b69d26d7e0f86e1db889dbdbb750776de6..fdc1e6fe4c250a1d3380dc7f3816d535f9e72e15 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -756,16 +756,10 @@ isc__nm_async_stop(isc__networker_t *worker, isc__netievent_t *ev0) {
 }
 
 void
-isc_nm_task_enqueue(isc_nm_t *nm, isc_task_t *task, int threadid) {
+isc_nm_task_enqueue(isc_nm_t *nm, isc_task_t *task, int tid) {
        isc__netievent_t *event = NULL;
-       int tid;
        isc__networker_t *worker = NULL;
-
-       if (threadid == -1) {
-               tid = (int)isc_random_uniform(nm->nworkers);
-       } else {
-               tid = threadid % nm->nworkers;
-       }
+       REQUIRE(tid >= 0 && tid < nm->nworkers);
 
        worker = &nm->workers[tid];
 
@@ -3567,6 +3561,13 @@ isc_nm_has_encryption(const isc_nmhandle_t *handle) {
        return (false);
 }
 
+uint32_t
+isc_nm_getnworkers(const isc_nm_t *netmgr) {
+       REQUIRE(VALID_NM(netmgr));
+
+       return (netmgr->nworkers);
+}
+
 #ifdef NETMGR_TRACE
 /*
  * Dump all active sockets in netmgr. We output to stderr

diff --git a/lib/isc/task.c b/lib/isc/task.c
index 6dbd018c50e3a913560befe7410c3e2b458dfb36..c89ea02fd12f96df2d8c9e2959397eaf1f1a392f 100644 (file)
--- a/lib/isc/task.c
+++ b/lib/isc/task.c
@@ -102,7 +102,7 @@ struct isc_task {
        isc_taskmgr_t *manager;
        isc_mutex_t lock;
        /* Locked by task lock. */
-       int threadid;
+       int tid;
        task_state_t state;
        isc_refcount_t references;
        isc_refcount_t running;
@@ -199,7 +199,7 @@ isc_task_create(isc_taskmgr_t *manager, unsigned int quantum,
 
 isc_result_t
 isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
-                     isc_task_t **taskp, int threadid) {
+                     isc_task_t **taskp, int tid) {
        isc_task_t *task = NULL;
        bool exiting;
 
@@ -213,21 +213,21 @@ isc_task_create_bound(isc_taskmgr_t *manager, unsigned int quantum,
 
        isc_taskmgr_attach(manager, &task->manager);
 
-       if (threadid == -1) {
+       if (tid == -1) {
                /*
-                * Task is not pinned to a queue, it's threadid will be
+                * Task is not pinned to a queue, it's tid will be
                 * chosen when first task will be sent to it - either
                 * randomly or specified by isc_task_sendto.
                 */
                task->bound = false;
-               task->threadid = -1;
+               task->tid = -1;
        } else {
                /*
                 * Task is pinned to a queue, it'll always be run
                 * by a specific thread.
                 */
                task->bound = true;
-               task->threadid = threadid;
+               task->tid = tid % isc_nm_getnworkers(task->manager->netmgr);
        }
 
        isc_mutex_init(&task->lock);
@@ -338,7 +338,11 @@ task_ready(isc_task_t *task) {
 
        isc_refcount_increment0(&task->running);
        LOCK(&task->lock);
-       isc_nm_task_enqueue(manager->netmgr, task, task->threadid);
+       if (task->tid < 0) {
+               task->tid = (int)isc_random_uniform(
+                       isc_nm_getnworkers(manager->netmgr));
+       }
+       isc_nm_task_enqueue(manager->netmgr, task, task->tid);
        UNLOCK(&task->lock);
 }
 
@@ -419,15 +423,11 @@ task_send(isc_task_t *task, isc_event_t **eventp, int c) {
 
        XTRACE("task_send");
 
-       if (task->bound) {
-               c = task->threadid;
-       } else if (c < 0) {
-               c = -1;
-       }
-
        if (task->state == task_state_idle) {
                was_idle = true;
-               task->threadid = c;
+               if (!task->bound) {
+                       task->tid = c;
+               }
                INSIST(EMPTY(task->events));
                task->state = task_state_ready;
        }
@@ -1055,7 +1055,7 @@ isc_taskmgr_setexcltask(isc_taskmgr_t *mgr, isc_task_t *task) {
        REQUIRE(VALID_TASK(task));
 
        LOCK(&task->lock);
-       REQUIRE(task->threadid == 0);
+       REQUIRE(task->tid == 0);
        UNLOCK(&task->lock);
 
        LOCK(&mgr->lock);

diff --git a/lib/ns/include/ns/interfacemgr.h b/lib/ns/include/ns/interfacemgr.h
index 5f6d97b2d07fcc17e96fe24d421dd7e05db915d9..8bd96101037d157493ee64ed112bb7dfcaefebbf 100644 (file)
--- a/lib/ns/include/ns/interfacemgr.h
+++ b/lib/ns/include/ns/interfacemgr.h
@@ -101,7 +101,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
                       isc_taskmgr_t *taskmgr, isc_timermgr_t *timermgr,
                       isc_nm_t *nm, dns_dispatchmgr_t *dispatchmgr,
                       isc_task_t *task, dns_geoip_databases_t *geoip,
-                      int ncpus, bool scan, ns_interfacemgr_t **mgrp);
+                      bool scan, ns_interfacemgr_t **mgrp);
 /*%<
  * Create a new interface manager.
  *

diff --git a/lib/ns/interfacemgr.c b/lib/ns/interfacemgr.c
index 1335f4497cfdf55adfcdd8e1ff60f52ded81ad3f..750273315e1782262f1327d9f527c53ac54f71a4 100644 (file)
--- a/lib/ns/interfacemgr.c
+++ b/lib/ns/interfacemgr.c
@@ -77,7 +77,7 @@ struct ns_interfacemgr {
        isc_task_t *excl;         /*%< Exclusive task */
        isc_timermgr_t *timermgr; /*%< Timer manager */
        isc_nm_t *nm;             /*%< Net manager */
-       int ncpus;                /*%< Number of workers */
+       uint32_t ncpus;           /*%< Number of workers */
        dns_dispatchmgr_t *dispatchmgr;
        unsigned int generation; /*%< Current generation no */
        ns_listenlist_t *listenon4;
@@ -308,7 +308,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
                       isc_taskmgr_t *taskmgr, isc_timermgr_t *timermgr,
                       isc_nm_t *nm, dns_dispatchmgr_t *dispatchmgr,
                       isc_task_t *task, dns_geoip_databases_t *geoip,
-                      int ncpus, bool scan, ns_interfacemgr_t **mgrp) {
+                      bool scan, ns_interfacemgr_t **mgrp) {
        isc_result_t result;
        ns_interfacemgr_t *mgr = NULL;
 
@@ -319,12 +319,14 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
        REQUIRE(*mgrp == NULL);
 
        mgr = isc_mem_get(mctx, sizeof(*mgr));
-       *mgr = (ns_interfacemgr_t){ .taskmgr = taskmgr,
-                                   .timermgr = timermgr,
-                                   .nm = nm,
-                                   .dispatchmgr = dispatchmgr,
-                                   .generation = 1,
-                                   .ncpus = ncpus };
+       *mgr = (ns_interfacemgr_t){
+               .taskmgr = taskmgr,
+               .timermgr = timermgr,
+               .nm = nm,
+               .dispatchmgr = dispatchmgr,
+               .generation = 1,
+               .ncpus = isc_nm_getnworkers(nm),
+       };
 
        isc_mem_attach(mctx, &mgr->mctx);
        ns_server_attach(sctx, &mgr->sctx);
@@ -375,7 +377,7 @@ ns_interfacemgr_create(isc_mem_t *mctx, ns_server_t *sctx,
 
        mgr->clientmgrs = isc_mem_get(mgr->mctx,
                                      mgr->ncpus * sizeof(mgr->clientmgrs[0]));
-       for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+       for (size_t i = 0; i < mgr->ncpus; i++) {
                result = ns_clientmgr_create(mgr->sctx, mgr->taskmgr,
                                             mgr->timermgr, mgr->aclenv, (int)i,
                                             &mgr->clientmgrs[i]);
@@ -406,7 +408,7 @@ ns_interfacemgr_destroy(ns_interfacemgr_t *mgr) {
        ns_listenlist_detach(&mgr->listenon6);
        clearlistenon(mgr);
        isc_mutex_destroy(&mgr->lock);
-       for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+       for (size_t i = 0; i < mgr->ncpus; i++) {
                ns_clientmgr_destroy(&mgr->clientmgrs[i]);
        }
        isc_mem_put(mgr->mctx, mgr->clientmgrs,
@@ -1372,7 +1374,7 @@ ns_interfacemgr_dumprecursing(FILE *f, ns_interfacemgr_t *mgr) {
        REQUIRE(NS_INTERFACEMGR_VALID(mgr));
 
        LOCK(&mgr->lock);
-       for (size_t i = 0; i < (size_t)mgr->ncpus; i++) {
+       for (size_t i = 0; i < mgr->ncpus; i++) {
                ns_client_dumprecursing(f, mgr->clientmgrs[i]);
        }
        UNLOCK(&mgr->lock);
@@ -1419,7 +1421,7 @@ ns_interfacemgr_getclientmgr(ns_interfacemgr_t *mgr) {
 
        REQUIRE(NS_INTERFACEMGR_VALID(mgr));
        REQUIRE(tid >= 0);
-       REQUIRE(tid < mgr->ncpus);
+       REQUIRE((uint32_t)tid < mgr->ncpus);
 
        return (mgr->clientmgrs[tid]);
 }

diff --git a/lib/ns/tests/nstest.c b/lib/ns/tests/nstest.c
index 84a822bdd17f8e44013c7a457c2fc5a75343fd59..690af2c64be4119fa484319b1bbc77ab61237b38 100644 (file)
--- a/lib/ns/tests/nstest.c
+++ b/lib/ns/tests/nstest.c
@@ -65,7 +65,6 @@ ns_clientmgr_t *clientmgr = NULL;
 ns_interfacemgr_t *interfacemgr = NULL;
 ns_server_t *sctx = NULL;
 bool app_running = false;
-int ncpus;
 bool debug_mem_record = true;
 static atomic_bool run_managers = false;
 
@@ -224,7 +223,7 @@ create_managers(void) {
        in_port_t port = 5300 + isc_random8();
        ns_listenlist_t *listenon = NULL;
        isc_event_t *event = NULL;
-       ncpus = isc_os_ncpus();
+       int ncpus = isc_os_ncpus();
 
        isc_managers_create(mctx, ncpus, 0, &netmgr, &taskmgr, &timermgr);
        CHECK(isc_task_create_bound(taskmgr, 0, &maintask, 0));
@@ -236,7 +235,7 @@ create_managers(void) {
        CHECK(dns_dispatchmgr_create(mctx, netmgr, &dispatchmgr));
 
        CHECK(ns_interfacemgr_create(mctx, sctx, taskmgr, timermgr, netmgr,
-                                    dispatchmgr, maintask, NULL, ncpus, false,
+                                    dispatchmgr, maintask, NULL, false,
                                     &interfacemgr));
 
        CHECK(ns_listenlist_default(mctx, port, -1, true, AF_INET, &listenon));

diff --git a/lib/ns/tests/nstest.h b/lib/ns/tests/nstest.h
index 7212e2bb3bd4bd82f5f1d01428b901a29217b7c9..58e6ebca739d1b8a83a11c1ad43f8ac9231e28e2 100644 (file)
--- a/lib/ns/tests/nstest.h
+++ b/lib/ns/tests/nstest.h
@@ -62,7 +62,6 @@ extern ns_clientmgr_t *clientmgr;
 extern ns_interfacemgr_t *interfacemgr;
 extern ns_server_t *sctx;
 extern bool app_running;
-extern int ncpus;
 extern bool debug_mem_record;
 
 #ifdef NETMGR_TRACE