dnsdist: Detect stale backend socket, reconnect() them

diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc
index 150028ce079dbf5c6cbd6f745a330de576978f30..300f13241e47e03b1574f2606f65a16903ab2d1e 100644 (file)
--- a/pdns/dnsdist.cc
+++ b/pdns/dnsdist.cc
@@ -464,8 +464,15 @@ void* responderThread(std::shared_ptr<DownstreamState> state)
   return 0;
 }
 
-DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_): remote(remote_), sourceAddr(sourceAddr_), sourceItf(sourceItf_)
+void DownstreamState::reconnect()
 {
+  connected = false;
+  if (fd != -1) {
+    /* shutdown() is needed to wake up recv() in the responderThread */
+    shutdown(fd, SHUT_RDWR);
+    close(fd);
+    fd = -1;
+  }
   if (!IsAnyAddress(remote)) {
     fd = SSocket(remote.sin4.sin_family, SOCK_DGRAM, 0);
     if (!IsAnyAddress(sourceAddr)) {
@@ -479,6 +486,13 @@ DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress
     catch(const std::runtime_error& error) {
       infolog("Error connecting to new server with address %s: %s", remote.toStringWithPort(), error.what());
     }
+  }
+}
+
+DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_): remote(remote_), sourceAddr(sourceAddr_), sourceItf(sourceItf_)
+{
+  if (!IsAnyAddress(remote)) {
+    reconnect();
     idStates.resize(g_maxOutstanding);
     sw.start();
     infolog("Added downstream server %s", remote.toStringWithPort());
@@ -843,16 +857,32 @@ bool processResponse(LocalStateHolder<vector<pair<std::shared_ptr<DNSRule>, std:
 
 static ssize_t udpClientSendRequestToBackend(DownstreamState* ss, const int sd, const char* request, const size_t requestLen)
 {
+  ssize_t result;
+
   if (ss->sourceItf == 0) {
-    return send(sd, request, requestLen, 0);
+    result = send(sd, request, requestLen, 0);
+  }
+  else {
+    struct msghdr msgh;
+    struct iovec iov;
+    char cbuf[256];
+    fillMSGHdr(&msgh, &iov, cbuf, sizeof(cbuf), const_cast<char*>(request), requestLen, &ss->remote);
+    addCMsgSrcAddr(&msgh, cbuf, &ss->sourceAddr, ss->sourceItf);
+    result = sendmsg(sd, &msgh, 0);
   }
 
-  struct msghdr msgh;
-  struct iovec iov;
-  char cbuf[256];
-  fillMSGHdr(&msgh, &iov, cbuf, sizeof(cbuf), const_cast<char*>(request), requestLen, &ss->remote);
-  addCMsgSrcAddr(&msgh, cbuf, &ss->sourceAddr, ss->sourceItf);
-  return sendmsg(sd, &msgh, 0);
+  if (result == -1) {
+    int savederrno = errno;
+    vinfolog("Error sending request to backend %s: %d", ss->remote.toStringWithPort(), savederrno);
+
+    /* This might sound silly, but on Linux send() might fail with EINVAL
+       if the interface the socket was bound to doesn't exist anymore. */
+    if (savederrno == EINVAL) {
+      ss->reconnect();
+    }
+  }
+
+  return result;
 }
 
 // listens to incoming queries, sends out to downstream servers, noting the intended return path 
@@ -1286,6 +1316,8 @@ void* healthChecksThread()
             }
             catch(const std::runtime_error& error) {
               infolog("Error connecting to new server with address %s: %s", dss->remote.toStringWithPort(), error.what());
+              newState = false;
+              dss->connected = false;
             }
           }
 

diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 8398c140a07548a500df87eaa0f5be2eb15399f5..eb1d955cb94f924bb86ad64b6f0f0ab9b426fcee 100644 (file)
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -380,7 +380,7 @@ struct DownstreamState
     }
     return name + " (" + remote.toStringWithPort()+ ")";
   }
-
+  void reconnect();
 };
 using servers_t =vector<std::shared_ptr<DownstreamState>>;