From: Witold Kręcicki Date: Tue, 7 Jan 2020 11:02:41 +0000 (+0100) Subject: Use isc_uv_export() to pass bound TCP listening socket to child listeners. X-Git-Tag: v9.15.8~14^2~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=67c1ca9a790b7fb9eb18181b754592582a20c68a;p=thirdparty%2Fbind9.git Use isc_uv_export() to pass bound TCP listening socket to child listeners. For multithreaded TCP listening we need to pass a bound socket to all listening threads. Instead of using uv_pipe handle passing method which is quite complex (lots of callbacks, each of them with its own error handling) we now use isc_uv_export() to export the socket, pass it as a member of the isc__netievent_tcpchildlisten_t structure, and then isc_uv_import() it in the child thread, simplifying the process significantly. --- diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h index 235fbd2288f..68f3f399b85 100644 --- a/lib/isc/netmgr/netmgr-int.h +++ b/lib/isc/netmgr/netmgr-int.h @@ -29,6 +29,7 @@ #include #include #include +#include "uv-compat.h" #define ISC_NETMGR_TID_UNKNOWN -1 @@ -208,9 +209,17 @@ typedef struct isc__netievent__socket_req { typedef isc__netievent__socket_req_t isc__netievent_tcpconnect_t; typedef isc__netievent__socket_req_t isc__netievent_tcplisten_t; -typedef isc__netievent__socket_req_t isc__netievent_tcpchildlisten_t; typedef isc__netievent__socket_req_t isc__netievent_tcpsend_t; +typedef struct isc__netievent__socket_streaminfo { + isc__netievent_type type; + isc_nmsocket_t *sock; + isc_uv_stream_info_t streaminfo; +} isc__netievent__socket_streaminfo_t; + +typedef isc__netievent__socket_streaminfo_t isc__netievent_tcpchildlisten_t; + + typedef struct isc__netievent__socket_handle { isc__netievent_type type; isc_nmsocket_t *sock; @@ -348,11 +357,6 @@ struct isc_nmsocket { isc_nmiface_t *iface; isc_nmhandle_t *tcphandle; - /*% Used to transfer listening TCP sockets to children */ - uv_pipe_t ipc; - char ipc_pipe_name[64]; - atomic_int_fast32_t schildren; - /*% Extra data allocated at the end of each isc_nmhandle_t */ size_t extrahandlesize; diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c index fe1f898cb22..90a2015bc4e 100644 --- a/lib/isc/netmgr/netmgr.c +++ b/lib/isc/netmgr/netmgr.c @@ -43,12 +43,6 @@ ISC_THREAD_LOCAL int isc__nm_tid_v = ISC_NETMGR_TID_UNKNOWN; -#ifdef WIN32 -#define NAMED_PIPE_PATTERN "\\\\.\\pipe\\named-%d-%u.pipe" -#else -#define NAMED_PIPE_PATTERN "/tmp/named-%d-%u.pipe" -#endif - static void nmsocket_maybe_destroy(isc_nmsocket_t *sock); static void @@ -856,16 +850,6 @@ isc__nmsocket_init(isc_nmsocket_t *sock, isc_nm_t *mgr, sock->ah_handles[i] = NULL; } - /* - * Use a random number in the named pipe name. Also add getpid() - * to the name to make sure we don't get a conflict between - * different unit tests running at the same time, where the PRNG - * is initialized to a constant seed. - */ - snprintf(sock->ipc_pipe_name, sizeof(sock->ipc_pipe_name), - NAMED_PIPE_PATTERN, getpid(), isc_random32()); - sock->ipc_pipe_name[sizeof(sock->ipc_pipe_name) - 1] = '\0'; - isc_mutex_init(&sock->lock); isc_condition_init(&sock->cond); isc_refcount_init(&sock->references, 1); diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c index d6ba3f4a999..46c612ab832 100644 --- a/lib/isc/netmgr/tcp.c +++ b/lib/isc/netmgr/tcp.c @@ -52,16 +52,6 @@ read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf); static void tcp_close_cb(uv_handle_t *uvhandle); -static void -ipc_connection_cb(uv_stream_t *stream, int status); -static void -ipc_write_cb(uv_write_t *uvreq, int status); -static void -ipc_close_cb(uv_handle_t *handle); -static void -childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status); -static void -childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf); static void stoplistening(isc_nmsocket_t *sock); static void @@ -206,25 +196,6 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface, } } -#ifndef WIN32 -/* - * Run fsync() on the directory containing a socket's IPC pipe, to - * ensure that all threads can see the pipe. - */ -static void -syncdir(const isc_nmsocket_t *sock) { - char *pipe = isc_mem_strdup(sock->mgr->mctx, sock->ipc_pipe_name); - int fd = open(dirname(pipe), O_RDONLY); - - RUNTIME_CHECK(fd >= 0); - - isc_mem_free(sock->mgr->mctx, pipe); - - fsync(fd); - close(fd); -} -#endif - /* * For multi-threaded TCP listening, we create a single "parent" socket, * bind to it, and then pass its uv_handle to a set of child sockets, one @@ -241,7 +212,8 @@ isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) { isc__netievent_tcplisten_t *ievent = (isc__netievent_tcplisten_t *) ev0; isc_nmsocket_t *sock = ievent->sock; - int r; + struct sockaddr_storage sname; + int r, snamelen = sizeof(sname); REQUIRE(isc__nm_in_netthread()); REQUIRE(sock->type == isc_nm_tcplistener); @@ -278,45 +250,25 @@ isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) { sock->result = isc__nm_uverr2result(r); goto done; } - uv_handle_set_data(&sock->uv_handle.handle, sock); /* - * uv_pipe_init() is incorrectly documented in libuv, and the - * example in benchmark-multi-accept.c is also wrong. - * - * The third parameter ('ipc') indicates that the pipe will be - * used to *send* a handle to other threads. Therefore, it must - * must be set to 0 for a 'listening' IPC socket, and 1 - * only for sockets that are really passing FDs between - * threads. + * By doing this now, we can find out immediately whether bind() + * failed, and quit if so. (uv_bind() uses a delayed error, + * initially returning success even if bind() fails, and this + * could cause a deadlock later if we didn't check first.) */ - r = uv_pipe_init(&worker->loop, &sock->ipc, 0); - RUNTIME_CHECK(r == 0); - - uv_handle_set_data((uv_handle_t *)&sock->ipc, sock); - r = uv_pipe_bind(&sock->ipc, sock->ipc_pipe_name); - RUNTIME_CHECK(r == 0); - - r = uv_listen((uv_stream_t *) &sock->ipc, sock->nchildren, - ipc_connection_cb); - RUNTIME_CHECK(r == 0); + r = uv_tcp_getsockname(&sock->uv_handle.tcp, &sname, &snamelen); + if (r != 0) { + uv_close(&sock->uv_handle.handle, tcp_close_cb); + sock->result = isc__nm_uverr2result(r); + goto done; + } -#ifndef WIN32 - /* - * On Unices a child thread might not see the pipe yet; - * that happened quite often in unit tests on FreeBSD. - * Syncing the directory ensures that the pipe is visible - * to everyone. - * This isn't done on Windows because named pipes exist - * within a different namespace, not on VFS. - */ - syncdir(sock); -#endif + uv_handle_set_data(&sock->uv_handle.handle, sock); /* - * For each worker we send a 'tcpchildlisten' event. The child - * listener will then receive its version of the socket - * uv_handle via IPC in isc__nm_async_tcpchildlisten(). + * For each worker, we send a 'tcpchildlisten' event with + * the exported socket. */ for (int i = 0; i < sock->nchildren; i++) { isc_nmsocket_t *csock = &sock->children[i]; @@ -324,6 +276,7 @@ isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) { event = isc__nm_get_ievent(csock->mgr, netievent_tcpchildlisten); + isc_uv_export(&sock->uv_handle.stream, &event->streaminfo); event->sock = csock; if (csock->tid == isc_nm_tid()) { isc__nm_async_tcpchildlisten(&sock->mgr->workers[i], @@ -344,67 +297,6 @@ isc__nm_async_tcplisten(isc__networker_t *worker, isc__netievent_t *ev0) { return; } -/* - * The parent received an IPC connection from a child and can now send - * the uv_handle to the child for listening. - */ -static void -ipc_connection_cb(uv_stream_t *stream, int status) { - isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream); - isc__networker_t *worker = &sock->mgr->workers[isc_nm_tid()]; - isc__nm_uvreq_t *nreq = isc__nm_uvreq_get(sock->mgr, sock); - int r; - - REQUIRE(status == 0); - - /* - * The buffer can be anything, it will be ignored, but it has to - * be something that won't disappear. - */ - nreq->uvbuf = uv_buf_init((char *)nreq, 1); - uv_pipe_init(&worker->loop, &nreq->ipc, 1); - uv_handle_set_data((uv_handle_t *)&nreq->ipc, nreq); - - r = uv_accept((uv_stream_t *) &sock->ipc, (uv_stream_t *) &nreq->ipc); - RUNTIME_CHECK(r == 0); - - r = uv_write2(&nreq->uv_req.write, - (uv_stream_t *) &nreq->ipc, &nreq->uvbuf, 1, - (uv_stream_t *) &sock->uv_handle.stream, ipc_write_cb); - RUNTIME_CHECK(r == 0); -} - -/* - * The call to send a socket uv_handle is complete; we may be able - * to close the IPC connection. - */ -static void -ipc_write_cb(uv_write_t *uvreq, int status) { - isc__nm_uvreq_t *req = uvreq->data; - - UNUSED(status); - - /* - * We want all children to get the socket. Once that's done, - * we can stop listening on the IPC socket. - */ - if (atomic_fetch_add(&req->sock->schildren, 1) == - req->sock->nchildren - 1) - { - uv_close((uv_handle_t *) &req->sock->ipc, NULL); - } - uv_close((uv_handle_t *) &req->ipc, ipc_close_cb); -} - -/* - * The IPC socket is closed: free its resources. - */ -static void -ipc_close_cb(uv_handle_t *handle) { - isc__nm_uvreq_t *req = uv_handle_get_data(handle); - isc__nm_uvreq_put(&req, req->sock); -} - /* * Connect to the parent socket and be ready to receive the uv_handle * for the socket we'll be listening on. @@ -414,73 +306,24 @@ isc__nm_async_tcpchildlisten(isc__networker_t *worker, isc__netievent_t *ev0) { isc__netievent_tcpchildlisten_t *ievent = (isc__netievent_tcpchildlisten_t *) ev0; isc_nmsocket_t *sock = ievent->sock; - isc__nm_uvreq_t *req = NULL; int r; REQUIRE(isc__nm_in_netthread()); REQUIRE(sock->type == isc_nm_tcpchildlistener); - r = uv_pipe_init(&worker->loop, &sock->ipc, 1); - RUNTIME_CHECK(r == 0); - - uv_handle_set_data((uv_handle_t *) &sock->ipc, sock); - - req = isc__nm_uvreq_get(sock->mgr, sock); - uv_pipe_connect(&req->uv_req.connect, &sock->ipc, - sock->parent->ipc_pipe_name, - childlisten_ipc_connect_cb); -} - -/* Child is now connected to parent via IPC and can begin reading. */ -static void -childlisten_ipc_connect_cb(uv_connect_t *uvreq, int status) { - isc__nm_uvreq_t *req = uvreq->data; - isc_nmsocket_t *sock = req->sock; - int r; - - UNUSED(status); - - isc__nm_uvreq_put(&req, sock); - - r = uv_read_start((uv_stream_t *) &sock->ipc, isc__nm_alloc_cb, - childlisten_read_cb); - RUNTIME_CHECK(r == 0); -} - -/* - * Child has received the socket uv_handle via IPC, and can now begin - * listening for connections and can close the IPC socket. - */ -static void -childlisten_read_cb(uv_stream_t *stream, ssize_t nread, const uv_buf_t *buf) { - isc_nmsocket_t *sock = uv_handle_get_data((uv_handle_t *) stream); - isc__networker_t *worker = NULL; - uv_pipe_t *ipc = NULL; - uv_handle_type type; - int r; - - UNUSED(nread); - - REQUIRE(VALID_NMSOCK(sock)); - REQUIRE(buf != NULL); - - ipc = (uv_pipe_t *) stream; - type = uv_pipe_pending_type(ipc); - INSIST(type == UV_TCP); - - isc__nm_free_uvbuf(sock, buf); worker = &sock->mgr->workers[isc_nm_tid()]; + uv_tcp_init(&worker->loop, (uv_tcp_t *) &sock->uv_handle.tcp); uv_handle_set_data(&sock->uv_handle.handle, sock); + isc_uv_import(&sock->uv_handle.stream, &ievent->streaminfo); - uv_accept(stream, &sock->uv_handle.stream); r = uv_listen((uv_stream_t *) &sock->uv_handle.tcp, sock->backlog, tcp_connection_cb); - uv_close((uv_handle_t *) ipc, NULL); + if (r != 0) { isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_NETMGR, ISC_LOG_ERROR, - "IPC socket close failed: %s", + "uv_listen failed: %s", isc_result_totext(isc__nm_uverr2result(r))); return; }