Handle the transient TCP connect() failures on FreeBSD

On FreeBSD (and perhaps other *BSD) systems, the TCP connect() call (via
uv_tcp_connect()) can fail with transient UV_EADDRINUSE error.  The UDP
code already handles this by trying three times (is a charm) before
giving up.  Add a code for the TCP, TCPDNS and TLSDNS layers to also try
three times before giving up by calling uv_tcp_connect() from the
callback two more time on UV_EADDRINUSE error.

Additionally, stop the timer only if we succeed or on hard error via
isc__nm_failed_connect_cb().

diff --git a/lib/isc/netmgr/netmgr-int.h b/lib/isc/netmgr/netmgr-int.h
index ebb57a339065673fd421878306e2fa65269c2825..96f863fa4d08d77fa42e84b603d34b4bbda61527 100644 (file)
--- a/lib/isc/netmgr/netmgr-int.h
+++ b/lib/isc/netmgr/netmgr-int.h
@@ -377,6 +377,7 @@ struct isc__nm_uvreq {
        isc__nm_cb_t cb;       /* callback */
        void *cbarg;           /* callback argument */
        isc_nm_timer_t *timer; /* TCP write timer */
+       int connect_tries;     /* connect retries */
 
        union {
                uv_handle_t handle;

diff --git a/lib/isc/netmgr/netmgr.c b/lib/isc/netmgr/netmgr.c
index 4b416fae0676532fbe88ed89a90a7e5b2eec9453..350117d6f7c5494761c2fa187c29b840078a1d4f 100644 (file)
--- a/lib/isc/netmgr/netmgr.c
+++ b/lib/isc/netmgr/netmgr.c
@@ -2465,7 +2465,10 @@ isc___nm_uvreq_get(isc_nm_t *mgr, isc_nmsocket_t *sock FLARG) {
                req = isc_mem_get(mgr->mctx, sizeof(*req));
        }
 
-       *req = (isc__nm_uvreq_t){ .magic = 0 };
+       *req = (isc__nm_uvreq_t){
+               .magic = 0,
+               .connect_tries = 3,
+       };
        ISC_LINK_INIT(req, link);
        req->uv_req.req.data = req;
        isc___nmsocket_attach(sock, &req->sock FLARG_PASS);

diff --git a/lib/isc/netmgr/tcp.c b/lib/isc/netmgr/tcp.c
index f7af009081bf6c242974143cb3a72ec0dc05cc2d..3d139b0dc19ecfb420a98953942da024bf986f32 100644 (file)
--- a/lib/isc/netmgr/tcp.c
+++ b/lib/isc/netmgr/tcp.c
@@ -228,9 +228,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        REQUIRE(VALID_NMSOCK(sock));
        REQUIRE(sock->tid == isc_nm_tid());
 
-       isc__nmsocket_timer_stop(sock);
-       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
-
        req = uv_handle_get_data((uv_handle_t *)uvreq);
 
        REQUIRE(VALID_UVREQ(req));
@@ -239,9 +236,7 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (!atomic_load(&sock->connecting)) {
+       } else if (!atomic_load(&sock->connecting)) {
                /*
                 * The connect was cancelled from timeout; just clean up
                 * the req.
@@ -260,11 +255,33 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(&req->uv_req.connect,
+                                          &sock->uv_handle.tcp,
+                                          &req->peer.type.sa, tcp_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc_uverr2result(status);
                goto error;
        }
 
+       isc__nmsocket_timer_stop(sock);
+       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
+
        isc__nm_incstats(sock, STATID_CONNECT);
        r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
                               &(int){ sizeof(ss) });
@@ -281,7 +298,6 @@ tcp_connect_cb(uv_connect_t *uvreq, int status) {
        isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
 
        return;
-
 error:
        isc__nm_failed_connect_cb(sock, req, result, false);
 }

diff --git a/lib/isc/netmgr/tcpdns.c b/lib/isc/netmgr/tcpdns.c
index e0da11b43f97b597c3ab9e5a39106a17fb5aaeb7..685268eed96a49ead015ba1d214691d4be98f77a 100644 (file)
--- a/lib/isc/netmgr/tcpdns.c
+++ b/lib/isc/netmgr/tcpdns.c
@@ -198,9 +198,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        REQUIRE(VALID_NMSOCK(sock));
        REQUIRE(sock->tid == isc_nm_tid());
 
-       isc__nmsocket_timer_stop(sock);
-       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
-
        req = uv_handle_get_data((uv_handle_t *)uvreq);
 
        REQUIRE(VALID_UVREQ(req));
@@ -209,9 +206,7 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (isc__nm_closing(sock)) {
+       } else if (isc__nm_closing(sock)) {
                /* Network manager shutting down */
                result = ISC_R_SHUTTINGDOWN;
                goto error;
@@ -223,11 +218,33 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(
+                               &req->uv_req.connect, &sock->uv_handle.tcp,
+                               &req->peer.type.sa, tcpdns_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc_uverr2result(status);
                goto error;
        }
 
+       isc__nmsocket_timer_stop(sock);
+       uv_handle_set_data((uv_handle_t *)&sock->read_timer, sock);
+
        isc__nm_incstats(sock, STATID_CONNECT);
        r = uv_tcp_getpeername(&sock->uv_handle.tcp, (struct sockaddr *)&ss,
                               &(int){ sizeof(ss) });
@@ -244,7 +261,6 @@ tcpdns_connect_cb(uv_connect_t *uvreq, int status) {
        isc__nm_connectcb(sock, req, ISC_R_SUCCESS, false);
 
        return;
-
 error:
        isc__nm_failed_connect_cb(sock, req, result, false);
 }

diff --git a/lib/isc/netmgr/tlsdns.c b/lib/isc/netmgr/tlsdns.c
index 42f47cb4238b799363fcf1775107d7472a16883e..44c546836b503f4173fc57f1c07d5ffa20aab11c 100644 (file)
--- a/lib/isc/netmgr/tlsdns.c
+++ b/lib/isc/netmgr/tlsdns.c
@@ -246,9 +246,7 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
        if (atomic_load(&sock->timedout)) {
                result = ISC_R_TIMEDOUT;
                goto error;
-       }
-
-       if (isc__nm_closing(sock)) {
+       } else if (isc__nm_closing(sock)) {
                /* Network manager shutting down */
                result = ISC_R_SHUTTINGDOWN;
                goto error;
@@ -260,6 +258,25 @@ tlsdns_connect_cb(uv_connect_t *uvreq, int status) {
                /* Timeout status code here indicates hard error */
                result = ISC_R_TIMEDOUT;
                goto error;
+       } else if (status == UV_EADDRINUSE) {
+               /*
+                * On FreeBSD the TCP connect() call sometimes results in a
+                * spurious transient EADDRINUSE. Try a few more times before
+                * giving up.
+                */
+               if (--req->connect_tries > 0) {
+                       r = uv_tcp_connect(
+                               &req->uv_req.connect, &sock->uv_handle.tcp,
+                               &req->peer.type.sa, tlsdns_connect_cb);
+                       if (r != 0) {
+                               isc__nm_incstats(sock, STATID_CONNECTFAIL);
+                               result = isc_uverr2result(r);
+                               goto error;
+                       }
+                       return;
+               }
+               result = isc_uverr2result(status);
+               goto error;
        } else if (status != 0) {
                result = isc_uverr2result(status);
                goto error;

diff --git a/lib/isc/netmgr/udp.c b/lib/isc/netmgr/udp.c
index 37a3f0480a9f9030fa072d7a3951d2a71c79eac5..b4aa2ea7633aeecc2d5bd0e8e0d56609334b95b7 100644 (file)
--- a/lib/isc/netmgr/udp.c
+++ b/lib/isc/netmgr/udp.c
@@ -846,7 +846,6 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
        isc__networker_t *worker = NULL;
        int uv_bind_flags = UV_UDP_REUSEADDR;
        isc_result_t result = ISC_R_UNSET;
-       int tries = 3;
        int r;
 
        REQUIRE(isc__nm_in_netthread());
@@ -901,7 +900,7 @@ udp_connect_direct(isc_nmsocket_t *sock, isc__nm_uvreq_t *req) {
        do {
                r = isc_uv_udp_connect(&sock->uv_handle.udp,
                                       &req->peer.type.sa);
-       } while (r == UV_EADDRINUSE && --tries > 0);
+       } while (r == UV_EADDRINUSE && --req->connect_tries > 0);
        if (r != 0) {
                isc__nm_incstats(sock, STATID_CONNECTFAIL);
                goto done;