Revert "Make ns_client mctxpool more thread-friendly by sharding it by netmgr threadid"

This reverts commit 3d9763b968e192ad105d0351c4d4f380a63c3d7f.

diff --git a/lib/ns/client.c b/lib/ns/client.c
index ef486430173cc6f1e0529aa0907f7e7126d0dd6f..f0d88e206f4af5832f9bed05575e1d0e5a1319db 100644 (file)
--- a/lib/ns/client.c
+++ b/lib/ns/client.c
@@ -2214,15 +2214,33 @@ ns__client_tcpconn(isc_nmhandle_t *handle, isc_result_t result, void *arg) {
 static void
 get_clientmctx(ns_clientmgr_t *manager, isc_mem_t **mctxp) {
        isc_mem_t *clientmctx;
+#if CLIENT_NMCTXS > 0
+       unsigned int nextmctx;
+#endif
        MTRACE("clientmctx");
 
-       int tid = isc_nm_tid();
-       if (tid < 0) {
-               tid = isc_random_uniform(manager->ncpus);
+#if CLIENT_NMCTXS > 0
+       LOCK(&manager->lock);
+       if (isc_nm_tid() >= 0) {
+               nextmctx = isc_nm_tid();
+       } else {
+               nextmctx = manager->nextmctx++;
+               if (manager->nextmctx == CLIENT_NMCTXS)
+                       manager->nextmctx = 0;
+
+               INSIST(nextmctx < CLIENT_NMCTXS);
        }
-       int rand = isc_random_uniform(CLIENT_NMCTXS_PERCPU);
-       int nextmctx = (rand * manager->ncpus) + tid;
+
        clientmctx = manager->mctxpool[nextmctx];
+       if (clientmctx == NULL) {
+               isc_mem_create(&clientmctx);
+               isc_mem_setname(clientmctx, "client", NULL);
+               manager->mctxpool[nextmctx] = clientmctx;
+       }
+       UNLOCK(&manager->lock);
+#else
+       clientmctx = manager->mctx;
+#endif
 
        isc_mem_attach(clientmctx, mctxp);
 }
@@ -2236,7 +2254,7 @@ get_clienttask(ns_clientmgr_t *manager, isc_task_t **taskp) {
                tid = isc_random_uniform(manager->ncpus);
        }
 
-       int rand = isc_random_uniform(CLIENT_NMCTXS_PERCPU);
+       int rand = isc_random_uniform(CLIENT_NTASKS_PERCPU);
        int nexttask = (rand * manager->ncpus) + tid;
        isc_task_attach(manager->taskpool[nexttask], taskp);
 }
@@ -2392,18 +2410,21 @@ clientmgr_detach(ns_clientmgr_t **mp) {
 
 static void
 clientmgr_destroy(ns_clientmgr_t *manager) {
+#if CLIENT_NMCTXS > 0
        int i;
+#endif
 
        MTRACE("clientmgr_destroy");
 
        isc_refcount_destroy(&manager->references);
        manager->magic = 0;
 
-       for (i = 0; i < manager->ncpus * CLIENT_NMCTXS_PERCPU; i++) {
-               isc_mem_detach(&manager->mctxpool[i]);
+#if CLIENT_NMCTXS > 0
+       for (i = 0; i < CLIENT_NMCTXS; i++) {
+               if (manager->mctxpool[i] != NULL)
+                       isc_mem_detach(&manager->mctxpool[i]);
        }
-       isc_mem_put(manager->mctx, manager->mctxpool,
-                   manager->ncpus * CLIENT_NMCTXS_PERCPU * sizeof(isc_mem_t*));
+#endif
 
        if (manager->interface != NULL) {
                ns_interface_detach(&manager->interface);
@@ -2435,8 +2456,9 @@ ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
 {
        ns_clientmgr_t *manager;
        isc_result_t result;
+#if CLIENT_NMCTXS > 0
        int i;
-       int npools;
+#endif
 
        manager = isc_mem_get(mctx, sizeof(*manager));
        *manager = (ns_clientmgr_t) { .magic = 0 };
@@ -2472,16 +2494,11 @@ ns_clientmgr_create(isc_mem_t *mctx, ns_server_t *sctx, isc_taskmgr_t *taskmgr,
        ns_server_attach(sctx, &manager->sctx);
 
        ISC_LIST_INIT(manager->recursing);
-
-       npools = CLIENT_NMCTXS_PERCPU * manager->ncpus;
-       manager->mctxpool = isc_mem_get(manager->mctx,
-                                       npools * sizeof(isc_mem_t*));
-       for (i = 0; i < npools; i++) {
-               manager->mctxpool[i] = NULL;
-               isc_mem_create(&manager->mctxpool[i]);
-               isc_mem_setname(manager->mctxpool[i], "client", NULL);
-       }
-
+#if CLIENT_NMCTXS > 0
+       manager->nextmctx = 0;
+       for (i = 0; i < CLIENT_NMCTXS; i++)
+               manager->mctxpool[i] = NULL; /* will be created on-demand */
+#endif
        manager->magic = MANAGER_MAGIC;
 
        MTRACE("create");

diff --git a/lib/ns/include/ns/client.h b/lib/ns/include/ns/client.h
index 5e4622a0927834c33710ff81ff036bd0164dd96f..cd77e3ea70a148fc334bbdc53aa26a437414b8ae 100644 (file)
--- a/lib/ns/include/ns/client.h
+++ b/lib/ns/include/ns/client.h
@@ -85,7 +85,7 @@
 #define NS_CLIENT_SEND_BUFFER_SIZE             4096
 #define NS_CLIENT_RECV_BUFFER_SIZE             4096
 
-#define CLIENT_NMCTXS_PERCPU                   8
+#define CLIENT_NMCTXS                          100
 /*%<
  * Number of 'mctx pools' for clients. (Should this be configurable?)
  * When enabling threads, we use a pool of memory contexts shared by
@@ -181,8 +181,11 @@ struct ns_clientmgr {
        isc_mutex_t             reclock;
        client_list_t           recursing;    /*%< Recursing clients */
 
+#if CLIENT_NMCTXS > 0
        /*%< mctx pool for clients. */
-       isc_mem_t **            mctxpool;
+       unsigned int            nextmctx;
+       isc_mem_t *             mctxpool[CLIENT_NMCTXS];
+#endif
 };
 
 /*% nameserver client structure */