Handle the transient TCP connect() failures on FreeBSD

On FreeBSD (and perhaps other *BSD) systems, the TCP connect() call (via
uv_tcp_connect()) can fail with transient UV_EADDRINUSE error.  The UDP
code already handles this by trying three times (is a charm) before
giving up.  Add a code for the TCP, TCPDNS and TLSDNS layers to also try
three times before giving up by calling uv_tcp_connect() from the
callback two more time on UV_EADDRINUSE error.

Additionally, stop the timer only if we succeed or on hard error via
isc__nm_failed_connect_cb().

(cherry picked from commit b21f507c0ac5b5d2d2c27fd2d71e27e8605dd5fc)

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index a7338d475e059f5b8f7bf8f906e170e22e002a8e..1417d7c68bf6e6b4b213d5bdeea5b411d88d4ef3 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -384,6 +384,7 @@ struct isc__nm_uvreq {
        isc__nm_cb_t cb;       /* callback */
        void *cbarg;           /* callback argument */
        isc_nm_timer_t *timer; /* TCP write timer */
+       int connect_tries;     /* connect retries */
 
        union {
                uv_handle_t handle;

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index e1ab339d71b06c6dcf18284c0f5f3d34aed4bc84..d6188d989972ea8465139b311f37e41b7d0d69bf 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -2518,7 +2518,10 @@ isc___nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock FLARG) {
                req = isc_mem_get(mgr->mctx, sizeof(*req));
        }
 
-       *req = (isc__nm_uvreq_t){ .magic = 0 };
+       *req = (isc__nm_uvreq_t){
+               .magic = 0,
+               .connect_tries = 3,
+       };
        ISC_LINK_INIT(req, link);
        req->uv_req.req.data = req;
        isc___nmsocket_attach(sock, &req->sock FLARG_PASS);

diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c
index 040bebc0aaf1a7f0f32d936111441a1da56f7826..d52e84c270f107345401eb570543cb8b35e559b8 100644 (file)
--- a/lib/isc/netmgr/tcp.c
+++ b/lib/isc/netmgr/tcp.c
@@ -229,9 +229,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        REQUIRE(VALID_NMSOCK(sock));
        REQUIRE(sock->tid == isc_nm_tid());
 
-       isc__nmsocket_timer_stop(sock);
-       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
-
        req = uv_handle_get_data((uv_handle_t *)uvreq);
 
        REQUIRE(VALID_UVREQ(req));
@@ -240,9 +237,7 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (!atomic_load(&sock->connecting)) {
+       } else if (!atomic_load(&sock->connecting)) {
                /*
                 * The connect was cancelled from timeout; just clean up
                 * the req.
@@ -261,11 +256,33 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(&req->uv_req.connect,
+                                          &sock->uv_handle.tcp,
+                                          &req->peer.type.sa, tcp_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc__nm_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc__nm_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc__nm_uverr2result(status);
                goto error;
        }
 
+       isc__nmsocket_timer_stop(sock);
+       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
+
        isc__nm_incstats(sock, STATID_CONNECT);
        r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
                               &(int){ sizeof(ss) });
@@ -282,7 +299,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
 
        return;
-
 error:
        isc__nm_failed_connect_cb(sock, req, result, false);
 }

diff --git a/lib/isc/netmgr/tcpdns.c b/lib/isc/netmgr/tcpdns.c
index ba6f6656998f7a2396918ebfbaa12d14265b3608..bafae418b1ab595a8d548a5bca661fbdb889427c 100644 (file)
--- a/lib/isc/netmgr/tcpdns.c
+++ b/lib/isc/netmgr/tcpdns.c
@@ -199,9 +199,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        REQUIRE(VALID_NMSOCK(sock));
        REQUIRE(sock->tid == isc_nm_tid());
 
-       isc__nmsocket_timer_stop(sock);
-       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
-
        req = uv_handle_get_data((uv_handle_t *)uvreq);
 
        REQUIRE(VALID_UVREQ(req));
@@ -210,9 +207,7 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (isc__nm_closing(sock)) {
+       } else if (isc__nm_closing(sock)) {
                /* Network manager shutting down */
                result = ISC_R_SHUTTINGDOWN;
                goto error;
@@ -224,11 +219,33 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(
+                               &req->uv_req.connect, &sock->uv_handle.tcp,
+                               &req->peer.type.sa, tcpdns_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc__nm_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc__nm_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc__nm_uverr2result(status);
                goto error;
        }
 
+       isc__nmsocket_timer_stop(sock);
+       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
+
        isc__nm_incstats(sock, STATID_CONNECT);
        r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
                               &(int){ sizeof(ss) });
@@ -245,7 +262,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
 
        return;
-
 error:
        isc__nm_failed_connect_cb(sock, req, result, false);
 }

diff --git a/lib/isc/netmgr/tlsdns.c b/lib/isc/netmgr/tlsdns.c
index a6831763f85bba55b4efdf4f82a4305cd0200dbe..79b92d5d8366ede0a2d5f5bb4a3630b98c4e765a 100644 (file)
--- a/lib/isc/netmgr/tlsdns.c
+++ b/lib/isc/netmgr/tlsdns.c
@@ -247,9 +247,7 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (isc__nm_closing(sock)) {
+       } else if (isc__nm_closing(sock)) {
                /* Network manager shutting down */
                result = ISC_R_SHUTTINGDOWN;
                goto error;
@@ -261,6 +259,25 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(
+                               &req->uv_req.connect, &sock->uv_handle.tcp,
+                               &req->peer.type.sa, tlsdns_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc__nm_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc__nm_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc__nm_uverr2result(status);
                goto error;

diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index c5449892d01a0fb34830f1aa38e300c094a2ca65..10065d2a1aacc80e3aa98d6f2f2e455df472f2e8 100644 (file)
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -850,7 +850,6 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
        isc__networker_t *worker = NULL;
        int uv_bind_flags = UV_UDP_REUSEADDR;
        isc_result_t result = ISC_R_UNSET;
-       int tries = 3;
        int r;
 
        REQUIRE(isc__nm_in_netthread());
@@ -901,7 +900,7 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
        do {
                r = isc_uv_udp_connect(&sock->uv_handle.udp,
                                       &req->peer.type.sa);
-       } while (r == UV_EADDRINUSE && --tries > 0);
+       } while (r == UV_EADDRINUSE && --req->connect_tries > 0);
        if (r != 0) {
                isc__nm_incstats(sock, STATID_CONNECTFAIL);
                goto done;