Convert isc_astack usage in netmgr to mempool and ISC_LIST

Change the per-socket inactive uvreq cache (implemented as isc_astack)
to per-worker memory pool.

Change the per-socket inactive nmhandle cache (implemented as
isc_astack) to unlocked per-socket ISC_LIST.

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index f4bb0f0871d1554b77b7f0664df7c2671c4f7d2f..4a722130fa32253d4efa90cbb303010aee8a9ac2 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -18,7 +18,6 @@
 #include <openssl/err.h>
 #include <openssl/ssl.h>
 
-#include <isc/astack.h>
 #include <isc/atomic.h>
 #include <isc/barrier.h>
 #include <isc/buffer.h>
@@ -201,6 +200,7 @@ typedef struct isc__networker {
 
        ISC_LIST(isc_nmsocket_t) active_sockets;
 
+       isc_mempool_t *uvreq_pool;
 } isc__networker_t;
 
 ISC_REFCOUNT_DECL(isc__networker);
@@ -245,6 +245,7 @@ struct isc_nmhandle {
        int backtrace_size;
 #endif
        LINK(isc_nmhandle_t) active_link;
+       LINK(isc_nmhandle_t) inactive_link;
        void *opaque;
 };
 
@@ -323,6 +324,7 @@ struct isc__nm_uvreq {
                uv_fs_t fs;
        } uv_req;
        ISC_LINK(isc__nm_uvreq_t) link;
+       ISC_LINK(isc__nm_uvreq_t) inactive_link;
 };
 
 void *
@@ -987,8 +989,7 @@ struct isc_nmsocket {
         * 'spare' handles for that can be reused to avoid allocations,
         * for UDP.
         */
-       isc_astack_t *inactivehandles;
-       isc_astack_t *inactivereqs;
+       ISC_LIST(isc_nmhandle_t) inactive_handles;
 
        /*%
         * Used to pass a result back from listen or connect events.

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index 613112e8169bed8c34a588a5f4affdb635b3ae42..d85742fe9e0a8b1a916e6f3bceee2e5009663dbe 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -248,6 +248,11 @@ isc_netmgr_create(isc_mem_t *mctx, isc_loopmgr_t *loopmgr, isc_nm_t **netmgrp) {
 
                isc_mem_attach(loop->mctx, &worker->mctx);
 
+               isc_mempool_create(worker->mctx, sizeof(isc__nm_uvreq_t),
+                                  &worker->uvreq_pool);
+               isc_mempool_setfreemax(worker->uvreq_pool,
+                                      ISC_NM_REQS_STACK_SIZE);
+
                isc_loop_attach(loop, &worker->loop);
                isc_loop_teardown(loop, networker_teardown, worker);
                isc_refcount_init(&worker->references, 1);
@@ -596,7 +601,6 @@ nmsocket_cleanup(isc_nmsocket_t *sock) {
        REQUIRE(!isc__nmsocket_active(sock));
 
        isc_nmhandle_t *handle = NULL;
-       isc__nm_uvreq_t *uvreq = NULL;
        isc__networker_t *worker = sock->worker;
 
        isc_refcount_destroy(&sock->references);
@@ -635,7 +639,8 @@ nmsocket_cleanup(isc_nmsocket_t *sock) {
                isc___nmsocket_detach(&sock->outer FLARG_PASS);
        }
 
-       while ((handle = isc_astack_pop(sock->inactivehandles)) != NULL) {
+       while ((handle = ISC_LIST_HEAD(sock->inactive_handles)) != NULL) {
+               ISC_LIST_DEQUEUE(sock->inactive_handles, handle, inactive_link);
                nmhandle_free(sock, handle);
        }
 
@@ -645,14 +650,6 @@ nmsocket_cleanup(isc_nmsocket_t *sock) {
 
        sock->pquota = NULL;
 
-       isc_astack_destroy(sock->inactivehandles);
-
-       while ((uvreq = isc_astack_pop(sock->inactivereqs)) != NULL) {
-               isc_mem_put(sock->worker->mctx, uvreq, sizeof(*uvreq));
-       }
-
-       isc_astack_destroy(sock->inactivereqs);
-
        isc__nm_tls_cleanup_data(sock);
 #if HAVE_LIBNGHTTP2
        isc__nm_http_cleanup_data(sock);
@@ -855,10 +852,7 @@ isc___nmsocket_init(isc_nmsocket_t *sock, isc__networker_t *worker,
                .type = type,
                .tid = worker->loop->tid,
                .fd = -1,
-               .inactivehandles = isc_astack_new(worker->mctx,
-                                                 ISC_NM_HANDLES_STACK_SIZE),
-               .inactivereqs = isc_astack_new(worker->mctx,
-                                              ISC_NM_REQS_STACK_SIZE),
+               .inactive_handles = ISC_LIST_INITIALIZER,
                .result = ISC_R_UNSET,
                .active_handles = ISC_LIST_INITIALIZER,
                .active_link = ISC_LINK_INITIALIZER,
@@ -988,27 +982,38 @@ alloc_handle(isc_nmsocket_t *sock) {
        *handle = (isc_nmhandle_t){
                .magic = NMHANDLE_MAGIC,
                .active_link = ISC_LINK_INITIALIZER,
+               .inactive_link = ISC_LINK_INITIALIZER,
        };
        isc_refcount_init(&handle->references, 1);
 
        return (handle);
 }
 
+static isc_nmhandle_t *
+dequeue_handle(isc_nmsocket_t *sock) {
+#if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__
+       isc_nmhandle_t *handle = ISC_LIST_HEAD(sock->inactive_handles);
+       if (handle != NULL) {
+               ISC_LIST_DEQUEUE(sock->inactive_handles, handle, inactive_link);
+
+               isc_refcount_init(&handle->references, 1);
+               INSIST(VALID_NMHANDLE(handle));
+               return (handle);
+       }
+#else
+       UNUSED(sock);
+#endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */
+       return (NULL);
+}
+
 isc_nmhandle_t *
 isc___nmhandle_get(isc_nmsocket_t *sock, isc_sockaddr_t const *peer,
                   isc_sockaddr_t const *local FLARG) {
-       isc_nmhandle_t *handle = NULL;
-
        REQUIRE(VALID_NMSOCK(sock));
 
-       handle = isc_astack_pop(sock->inactivehandles);
-
+       isc_nmhandle_t *handle = dequeue_handle(sock);
        if (handle == NULL) {
                handle = alloc_handle(sock);
-       } else {
-               isc_refcount_init(&handle->references, 1);
-               INSIST(VALID_NMHANDLE(handle));
-               ISC_LINK_INIT(handle, active_link);
        }
 
        NETMGR_TRACE_LOG(
@@ -1107,25 +1112,17 @@ nmhandle_free(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
 
 static void
 nmhandle_deactivate(isc_nmsocket_t *sock, isc_nmhandle_t *handle) {
-       bool reuse = false;
-       uint_fast32_t ah;
+       uint_fast32_t ah = atomic_fetch_sub(&sock->ah, 1);
+       INSIST(ah > 0);
 
-       /*
-        * We do all of this under lock to avoid races with socket
-        * destruction.  We have to do this now, because at this point the
-        * socket is either unused or still attached to event->sock.
-        */
        ISC_LIST_UNLINK(sock->active_handles, handle, active_link);
 
-       ah = atomic_fetch_sub(&sock->ah, 1);
-       INSIST(ah > 0);
-
 #if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__
        if (atomic_load(&sock->active)) {
-               reuse = isc_astack_trypush(sock->inactivehandles, handle);
-       }
+               ISC_LIST_APPEND(sock->inactive_handles, handle, inactive_link);
+       } else
 #endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */
-       if (!reuse) {
+       {
                nmhandle_free(sock, handle);
        }
 }
@@ -1196,6 +1193,11 @@ nmhandle_detach_cb(isc_nmhandle_t **handlep FLARG) {
        }
 #endif
 
+       if (handle == sock->statichandle) {
+               /* statichandle is assigned, not attached. */
+               sock->statichandle = NULL;
+       }
+
        nmhandle_deactivate(sock, handle);
 
        /*
@@ -1207,11 +1209,6 @@ nmhandle_detach_cb(isc_nmhandle_t **handlep FLARG) {
                sock->closehandle_cb(sock);
        }
 
-       if (handle == sock->statichandle) {
-               /* statichandle is assigned, not attached. */
-               sock->statichandle = NULL;
-       }
-
        isc___nmsocket_detach(&sock FLARG_PASS);
 }
 
@@ -1761,35 +1758,31 @@ isc___nm_uvreq_get(isc__networker_t *worker, isc_nmsocket_t *sock FLARG) {
 
        REQUIRE(worker != NULL);
        REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->tid == isc_tid());
 
-       if (sock != NULL && isc__nmsocket_active(sock)) {
-               /* Try to reuse one */
-               req = isc_astack_pop(sock->inactivereqs);
-       }
-
-       if (req == NULL) {
-               req = isc_mem_get(worker->mctx, sizeof(*req));
-       }
-
+       req = isc_mempool_get(worker->uvreq_pool);
        *req = (isc__nm_uvreq_t){
-               .magic = 0,
                .connect_tries = 3,
+               .link = ISC_LINK_INITIALIZER,
+               .inactive_link = ISC_LINK_INITIALIZER,
+               .uv_req.req.data = req,
+               .magic = UVREQ_MAGIC,
        };
-       ISC_LINK_INIT(req, link);
-       req->uv_req.req.data = req;
        isc___nmsocket_attach(sock, &req->sock FLARG_PASS);
-       req->magic = UVREQ_MAGIC;
 
        return (req);
 }
 
 void
 isc___nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *sock FLARG) {
-       isc__nm_uvreq_t *req = NULL;
-       isc_nmhandle_t *handle = NULL;
-
        REQUIRE(req0 != NULL);
        REQUIRE(VALID_UVREQ(*req0));
+       REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->tid == isc_tid());
+
+       isc__nm_uvreq_t *req = NULL;
+       isc_nmhandle_t *handle = NULL;
+       isc__networker_t *worker = sock->worker;
 
        req = *req0;
        *req0 = NULL;
@@ -1802,18 +1795,9 @@ isc___nm_uvreq_put(isc__nm_uvreq_t **req0, isc_nmsocket_t *sock FLARG) {
         * We need to save this first to make sure that handle,
         * sock, and the netmgr won't all disappear.
         */
-       handle = req->handle;
-       req->handle = NULL;
+       ISC_SWAP(handle, req->handle);
 
-#if !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__
-       if (!isc__nmsocket_active(sock) ||
-           !isc_astack_trypush(sock->inactivereqs, req))
-       {
-               isc_mem_put(sock->worker->mctx, req, sizeof(*req));
-       }
-#else  /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */
-       isc_mem_put(sock->worker->mctx, req, sizeof(*req));
-#endif /* !__SANITIZE_ADDRESS__ && !__SANITIZE_THREAD__ */
+       isc_mempool_put(worker->uvreq_pool, req);
 
        if (handle != NULL) {
                isc__nmhandle_detach(&handle FLARG_PASS);
@@ -2633,6 +2617,8 @@ isc__networker_destroy(isc__networker_t *worker) {
 
        isc_loop_detach(&worker->loop);
 
+       isc_mempool_destroy(&worker->uvreq_pool);
+
        isc_mem_put(worker->mctx, worker->sendbuf, ISC_NETMGR_SENDBUF_SIZE);
        isc_mem_putanddetach(&worker->mctx, worker->recvbuf,
                             ISC_NETMGR_RECVBUF_SIZE);