Replace the tcp_buffers memory pool with static per-loop buffer

As a single thread can process only one TCP send at the time, we don't
really need a memory pool for the TCP buffers, but it's enough to have
a single per-loop (client manager) static buffer that's being used to
assemble the DNS message and then it gets copied into own sending
buffer.

In the future, this should get optimized by exposing the uv_try API
from the network manager, and first try to send the message directly
and allocate the sending buffer only if we need to send the data
asynchronously.

diff --git a/lib/ns/client.c b/lib/ns/client.c
index 411a03cb3c7ceb5b9e5dc4b882f6a3b49ce02596..ec9de757b378cc3a9f1358f6ed072847d12817d6 100644 (file)
--- a/lib/ns/client.c
+++ b/lib/ns/client.c
@@ -376,7 +376,7 @@ static void
 client_setup_tcp_buffer(ns_client_t *client) {
        REQUIRE(client->tcpbuf == NULL);
 
-       client->tcpbuf = isc_mempool_get(client->manager->tcp_buffers);
+       client->tcpbuf = client->manager->tcp_buffer;
        client->tcpbuf_size = NS_CLIENT_TCP_BUFFER_SIZE;
 }
 
@@ -386,13 +386,12 @@ client_put_tcp_buffer(ns_client_t *client) {
                return;
        }
 
-       if (client->tcpbuf_size == NS_CLIENT_TCP_BUFFER_SIZE) {
-               isc_mempool_put(client->manager->tcp_buffers, client->tcpbuf);
-       } else {
+       if (client->tcpbuf != client->manager->tcp_buffer) {
                isc_mem_put(client->manager->send_mctx, client->tcpbuf,
                            client->tcpbuf_size);
        }
 
+       client->tcpbuf = NULL;
        client->tcpbuf_size = 0;
 }
 
@@ -440,21 +439,14 @@ client_sendpkg(ns_client_t *client, isc_buffer_t *buffer) {
 
        if (isc_buffer_base(buffer) == client->tcpbuf) {
                size_t used = isc_buffer_usedlength(buffer);
-               const size_t threshold = (3 * NS_CLIENT_TCP_BUFFER_SIZE) / 4;
-
                INSIST(client->tcpbuf_size == NS_CLIENT_TCP_BUFFER_SIZE);
+
                /*
                 * Copy the data into a smaller buffer before sending,
                 * and keep the original big TCP send buffer for reuse
                 * by other clients.
                 */
-               if (used > threshold) {
-                       /*
-                        * The data in the buffer is very large, so there is
-                        * no point in using a smaller buffer.
-                        */
-                       r.base = buffer->base;
-               } else if (used > NS_CLIENT_SEND_BUFFER_SIZE) {
+               if (used > NS_CLIENT_SEND_BUFFER_SIZE) {
                        /*
                         * We can save space by allocating a new buffer with a
                         * correct size and freeing the big buffer.
@@ -1728,8 +1720,6 @@ ns__client_put_cb(void *client0) {
 
        client->magic = 0;
 
-       isc_mem_put(manager->send_mctx, client->sendbuf,
-                   NS_CLIENT_SEND_BUFFER_SIZE);
        if (client->opt != NULL) {
                INSIST(dns_rdataset_isassociated(client->opt));
                dns_rdataset_disassociate(client->opt);
@@ -2437,9 +2427,6 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
                                   client->manager->rdspool,
                                   DNS_MESSAGE_INTENTPARSE, &client->message);
 
-               client->sendbuf = isc_mem_get(client->manager->send_mctx,
-                                             NS_CLIENT_SEND_BUFFER_SIZE);
-
                /*
                 * Set magic earlier than usual because ns_query_init()
                 * and the functions it calls will require it.
@@ -2460,7 +2447,6 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
                *client = (ns_client_t){
                        .magic = 0,
                        .manager = client->manager,
-                       .sendbuf = client->sendbuf,
                        .message = client->message,
                        .query = client->query,
                };
@@ -2485,8 +2471,6 @@ ns__client_setup(ns_client_t *client, ns_clientmgr_t *mgr, bool new) {
        return (ISC_R_SUCCESS);
 
 cleanup:
-       isc_mem_put(client->manager->send_mctx, client->sendbuf,
-                   NS_CLIENT_SEND_BUFFER_SIZE);
        dns_message_detach(&client->message);
        ns_clientmgr_detach(&client->manager);
 
@@ -2514,8 +2498,6 @@ clientmgr_destroy_cb(void *arg) {
 
        dns_message_destroypools(&manager->rdspool, &manager->namepool);
 
-       isc_mempool_destroy(&manager->tcp_buffers);
-
        isc_mem_detach(&manager->send_mctx);
 
        isc_mem_putanddetach(&manager->mctx, manager, sizeof(*manager));
@@ -2607,13 +2589,6 @@ ns_clientmgr_create(ns_server_t *sctx, isc_loopmgr_t *loopmgr,
         */
        (void)isc_mem_arena_set_muzzy_decay_ms(manager->send_mctx, 0);
 
-       isc_mempool_create(manager->send_mctx,
-                          (size_t)NS_CLIENT_TCP_BUFFER_SIZE,
-                          &manager->tcp_buffers);
-       isc_mempool_setfillcount(manager->tcp_buffers, TCPBUFFERS_FILLCOUNT);
-       isc_mempool_setfreemax(manager->tcp_buffers, TCPBUFFERS_FREEMAX);
-       isc_mempool_setname(manager->tcp_buffers, "ns_clientmgr_tcp");
-
        manager->magic = MANAGER_MAGIC;
 
        MTRACE("create");

diff --git a/lib/ns/include/ns/client.h b/lib/ns/include/ns/client.h
index 753cbe677ee7872c285cd42ba9d0afeae2bf9201..a3b7ccf3383b345df0598820a50ab3ae5be5cd68 100644 (file)
--- a/lib/ns/include/ns/client.h
+++ b/lib/ns/include/ns/client.h
@@ -159,7 +159,7 @@ struct ns_clientmgr {
        isc_mutex_t   reclock;
        client_list_t recursing; /*%< Recursing clients */
 
-       isc_mempool_t *tcp_buffers;
+       uint8_t tcp_buffer[NS_CLIENT_TCP_BUFFER_SIZE];
 };
 
 /*% nameserver client structure */
@@ -180,7 +180,6 @@ struct ns_client {
        unsigned char  *tcpbuf;
        size_t          tcpbuf_size;
        dns_message_t  *message;
-       unsigned char  *sendbuf;
        dns_rdataset_t *opt;
        dns_ednsopt_t  *ede;
        uint16_t        udpsize;
@@ -230,6 +229,8 @@ struct ns_client {
         * bits will be used as the rcode in the response message.
         */
        int32_t rcode_override;
+
+       uint8_t sendbuf[NS_CLIENT_SEND_BUFFER_SIZE];
 };
 
 #define NS_CLIENT_MAGIC           ISC_MAGIC('N', 'S', 'C', 'c')