Use different allocators for UDP and TCP

Each worker has a receive buffer with space for 20 DNS messages of up
to 2^16 bytes each, and the allocator function passed to uv_read_start()
or uv_udp_recv_start() will reserve a portion of it for use by sockets.
UDP can use recvmmsg() and so it needs that entire space, but TCP reads
one message at a time.

This commit introduces separate allocator functions for TCP and UDP
setting different buffer size limits, so that libuv will provide the
correct buffer sizes to each of them.

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index 5876bba328c9bbe9e36270b9af035839ccda26e0..149445a533c206e3983ec91fd1395028c20a7f26 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -560,16 +560,6 @@ isc__nm_enqueue_ievent(isc__networker_t *worker, isc__netievent_t *event);
  * way to use an isc__networker_t from another thread.)
  */
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf);
-/*%<
- * Allocator for recv operations.
- *
- * Note that as currently implemented, this doesn't actually
- * allocate anything, it just assigns the the isc__networker's UDP
- * receive buffer to a socket, and marks it as "in use".
- */
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf);
 /*%<

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index a40af49ba682f07b451657ac20a2c17eae9779ff..a6a03e6b3849750fbda9daf9ed4bf10b340779e0 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -976,23 +976,6 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, isc_nmsocket_type type,
        sock->magic = NMSOCK_MAGIC;
 }
 
-void
-isc__nm_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
-       isc_nmsocket_t *sock = uv_handle_get_data(handle);
-       isc__networker_t *worker = NULL;
-
-       REQUIRE(VALID_NMSOCK(sock));
-       REQUIRE(isc__nm_in_netthread());
-       REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
-
-       worker = &sock->mgr->workers[sock->tid];
-       INSIST(!worker->recvbuf_inuse);
-
-       buf->base = worker->recvbuf;
-       worker->recvbuf_inuse = true;
-       buf->len = ISC_NETMGR_RECVBUF_SIZE;
-}
-
 void
 isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
        isc__networker_t *worker = NULL;
@@ -1005,7 +988,7 @@ isc__nm_free_uvbuf(isc_nmsocket_t *sock, const uv_buf_t *buf) {
        worker = &sock->mgr->workers[sock->tid];
 
        REQUIRE(worker->recvbuf_inuse);
-       if (buf->base > worker->recvbuf &&
+       if (sock->type == isc_nm_udpsocket && buf->base > worker->recvbuf &&
            buf->base <= worker->recvbuf + ISC_NETMGR_RECVBUF_SIZE)
        {
                /* Can happen in case of out-of-order recvmmsg in libuv1.36 */

diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c
index 7e9644e4ff61f09da8edd79bf899fa1d7c17c4b7..355d0c35ac0455eeb2b4a602ee05949d6f48eeba 100644 (file)
--- a/lib/isc/netmgr/tcp.c
+++ b/lib/isc/netmgr/tcp.c
@@ -532,6 +532,30 @@ isc__nm_tcp_read(isc_nmhandle_t *handle, isc_nm_recv_cb_t cb, void *cbarg) {
        return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for TCP read operations. Limited to size 2^16.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+tcp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+       isc_nmsocket_t *sock = uv_handle_get_data(handle);
+       isc__networker_t *worker = NULL;
+
+       REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->type == isc_nm_tcpsocket);
+       REQUIRE(isc__nm_in_netthread());
+       REQUIRE(size <= 65536);
+
+       worker = &sock->mgr->workers[sock->tid];
+       INSIST(!worker->recvbuf_inuse);
+
+       buf->base = worker->recvbuf;
+       buf->len = size;
+       worker->recvbuf_inuse = true;
+}
+
 void
 isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
        isc__netievent_startread_t *ievent = (isc__netievent_startread_t *)ev0;
@@ -549,7 +573,7 @@ isc__nm_async_tcp_startread(isc__networker_t *worker, isc__netievent_t *ev0) {
                               0);
        }
 
-       r = uv_read_start(&sock->uv_handle.stream, isc__nm_alloc_cb, read_cb);
+       r = uv_read_start(&sock->uv_handle.stream, tcp_alloc_cb, read_cb);
        if (r != 0) {
                isc__nm_incstats(sock->mgr, sock->statsindex[STATID_RECVFAIL]);
        }

diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index 0d130453ff2f95f1683fc17c05a7de6ef6431eb4..25afd22b8a1ba51df5ecb572cddec49f4e03a2c3 100644 (file)
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -132,6 +132,32 @@ isc_nm_listenudp(isc_nm_t *mgr, isc_nmiface_t *iface, isc_nm_recv_cb_t cb,
        return (ISC_R_SUCCESS);
 }
 
+/*%<
+ * Allocator for UDP recv operations. Limited to size 20 * (2^16 + 2),
+ * which allows enough space for recvmmsg() to get multiple messages at
+ * a time.
+ *
+ * Note this doesn't actually allocate anything, it just assigns the
+ * worker's receive buffer to a socket, and marks it as "in use".
+ */
+static void
+udp_alloc_cb(uv_handle_t *handle, size_t size, uv_buf_t *buf) {
+       isc_nmsocket_t *sock = uv_handle_get_data(handle);
+       isc__networker_t *worker = NULL;
+
+       REQUIRE(VALID_NMSOCK(sock));
+       REQUIRE(sock->type == isc_nm_udpsocket);
+       REQUIRE(isc__nm_in_netthread());
+       REQUIRE(size <= ISC_NETMGR_RECVBUF_SIZE);
+
+       worker = &sock->mgr->workers[sock->tid];
+       INSIST(!worker->recvbuf_inuse);
+
+       buf->base = worker->recvbuf;
+       buf->len = ISC_NETMGR_RECVBUF_SIZE;
+       worker->recvbuf_inuse = true;
+}
+
 /*
  * handle 'udplisten' async call - start listening on a socket.
  */
@@ -192,7 +218,7 @@ isc__nm_async_udplisten(isc__networker_t *worker, isc__netievent_t *ev0) {
        uv_send_buffer_size(&sock->uv_handle.handle,
                            &(int){ ISC_SEND_BUFFER_SIZE });
 #endif
-       uv_udp_recv_start(&sock->uv_handle.udp, isc__nm_alloc_cb, udp_recv_cb);
+       uv_udp_recv_start(&sock->uv_handle.udp, udp_alloc_cb, udp_recv_cb);
 }
 
 static void