Use different allocators for UDP and TCP

Each worker has a receive buffer with space for 20 DNS messages of up
to 2^16 bytes each, and the allocator function passed to uv_read_start()
or uv_udp_recv_start() will reserve a portion of it for use by sockets.
UDP can use recvmmsg() and so it needs that entire space, but TCP reads
one message at a time.

This commit introduces separate allocator functions for TCP and UDP
setting different buffer size limits, so that libuv will provide the
correct buffer sizes to each of them.

(cherry picked from commit 38264b6a4d272dbbe948cf66eef9634041d14f98)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index d2cd5836e9154f6c9edc3c3e1e83ef01ce76fb95..ba37d643bf7fbe88cffdc0074a1d60074dd66c81 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -559,16 +559,6 @@ isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event);
  * way to use an isc__networker_t from another thread.)
  */
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf);
-/*%<
- * Allocator for recv operations.
- *
- * Note that as currently implemented, this doesn't actually
- * allocate anything, it just assigns the the isc__networker's UDP
- * receive buffer to a socket, and marks it as "in use".
- */
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf);
 /*%<

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index c2f95ccbf9582707e3d23dead715ccf54e86da75..29adc61a2acf7768bc28e6197a1089f8d8056c29 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -1003,23 +1003,6 @@ isc__nmsocket_clearcb(isc_nmsocket_t *sock) {
        sock->accept_cbarg = NULL;
 }
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
-       isc_nmsocket_t *sock = uv_handle_get_data(handle);
-       isc__networker_t *worker = NULL;
-
-       REQUIRE(VALID_NMSOCK(sock));
-       REQUIRE(isc__nm_in_netthread());
-       REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
-
-       worker = &sock->mgr->workers[sock->tid];
-       INSIST(!worker->recvbuf_inuse);
-
-       buf->base = worker->recvbuf;
-       worker->recvbuf_inuse = true;
-       buf->len = ISC_NETMGR_RECVBUF_SIZE;
-}
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
        isc__networker_t *worker = NULL;
@@ -1032,7 +1015,7 @@ isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
        worker = &sock->mgr->workers[sock->tid];
 
        REQUIRE(worker->recvbuf_inuse);
-       if (buf->base > worker->recvbuf &&
+       if (sock->type == isc_nm_udpsocket && buf->base > worker->recvbuf &&
            buf->base <= worker->recvbuf + ISC_NETMGR_RECVBUF_SIZE)
        {
                /* Can happen in case of out-of-order recvmmsg in libuv1.36 */

diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c
index 9542bd16dae63a4a66db352f67b10243b573465d..75cfd82128cd2820bdb5438250157583173490d0 100644 (file)
--- a/lib/isc/netmgr/tcp.c
+++ b/lib/isc/netmgr/tcp.c
@@ -637,6 +637,30 @@ isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
        return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for TCP read operations. Limited to size 2^16.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+tcp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+       isc_nmsocket_t *sock = uv_handle_get_data(handle);
+       isc__networker_t *worker = NULL;
+
+       REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->type == isc_nm_tcpsocket);
+       REQUIRE(isc__nm_in_netthread());
+       REQUIRE(size <= 65536);
+
+       worker = &sock->mgr->workers[sock->tid];
+       INSIST(!worker->recvbuf_inuse);
+
+       buf->base = worker->recvbuf;
+       buf->len = size;
+       worker->recvbuf_inuse = true;
+}
+
 void
 isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
        isc__netievent_startread_t *ievent = (isc__netievent_startread_t *)ev0;
@@ -654,7 +678,7 @@ isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
                               0);
        }
 
-       r = uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, read_cb);
+       r = uv_read_start(&sock->uv_handle.stream, tcp_alloc_cb, read_cb);
        if (r != 0) {
                isc__nm_incstats(sock->mgr, sock->statsindex[STATID_RECVFAIL]);
        }

diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index 851e1f72b658b13321148c2d5c0bbde518c3b321..2e83ff1d21c32dac1da175ae1f263c92d8fb107a 100644 (file)
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -132,6 +132,32 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
        return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for UDP recv operations. Limited to size 20 * (2^16 + 2),
+ * which allows enough space for recvmmsg() to get multiple messages at
+ * a time.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+udp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+       isc_nmsocket_t *sock = uv_handle_get_data(handle);
+       isc__networker_t *worker = NULL;
+
+       REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->type == isc_nm_udpsocket);
+       REQUIRE(isc__nm_in_netthread());
+       REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
+
+       worker = &sock->mgr->workers[sock->tid];
+       INSIST(!worker->recvbuf_inuse);
+
+       buf->base = worker->recvbuf;
+       buf->len = ISC_NETMGR_RECVBUF_SIZE;
+       worker->recvbuf_inuse = true;
+}
+
 /*
  * handle 'udplisten' async call - start listening on a socket.
  */
@@ -193,7 +219,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
        uv_send_buffer_size(&sock->uv_handle.handle,
                            &(int){ ISC_SEND_BUFFER_SIZE });
 #endif
-       uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb, udp_recv_cb);
+       uv_udp_recv_start(&sock->uv_handle.udp, udp_alloc_cb, udp_recv_cb);
 }
 
 static void