dnsdist: Add an option to limit the number of queued TCP connections

By default, if all TCP threads are busy we will queue incoming TCP
connections until the communication pipes are full.
In some setup, we might run out of file descriptors quickly,
causing new TCP connections but also health checks, web requests,
console connections to fail.
This new `setMaxTCPQueuedConnections()` option limits the number of
queued connections, dropping new ones when the threshold is
reached.

diff --git a/pdns/README-dnsdist.md b/pdns/README-dnsdist.md
index 6a9d4a03dbb8aa231e27b31402a27077b330a383..f04f430d02a933e03ee79de219a490ed6391766a 100644 (file)
--- a/pdns/README-dnsdist.md
+++ b/pdns/README-dnsdist.md
@@ -788,6 +788,11 @@ First, a few words about `dnsdist` architecture:
 The maximum number of threads in the TCP pool is controlled by the
 `setMaxTCPClientThreads()` directive, and defaults to 10. This number can be
 increased to handle a large number of simultaneous TCP connections.
+If all the TCP threads are busy, new TCP connections are queued while
+they wait to be picked up. The maximum number of queued connections
+can be configured with `setMaxTCPQueuedConnections()`, and any value other
+than 0 (the default) will cause new connections to be dropped if there
+are already too many queued.
 
 When dispatching UDP queries to backend servers, `dnsdist` keeps track of at
 most `n` outstanding queries for each backend. This number `n` can be tuned by
@@ -1137,6 +1142,7 @@ instantiate a server with additional parameters
     * `setTCPRecvTimeout(n)`: set the read timeout on TCP connections from the client, in seconds
     * `setTCPSendTimeout(n)`: set the write timeout on TCP connections from the client, in seconds
     * `setMaxTCPClientThreads(n)`: set the maximum of TCP client threads, handling TCP connections
+    * `setMaxTCPQueuedConnections(n)`: set the maximum number of TCP connections queued (waiting to be picked up by a client thread)
     * `setMaxUDPOutstanding(n)`: set the maximum number of outstanding UDP queries to a given backend server. This can only be set at configuration time and defaults to 10240
     * `setCacheCleaningDelay(n)`: set the interval in seconds between two runs of the cache cleaning algorithm, removing expired entries
     * `setStaleCacheEntriesTTL(n)`: allows using cache entries expired for at most `n` seconds when no backend available to answer for a query

diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc
index 4c5f39232e76018cc8f09fe9414e241e30480384..8e9dc591462d5e960c8cad355ffc380afaf685d9 100644 (file)
--- a/pdns/dnsdist-console.cc
+++ b/pdns/dnsdist-console.cc
@@ -101,6 +101,7 @@ void doClient(ComboAddress server, const std::string& command)
       cout<<endl;
     }
   }
+  close(fd);
 }
 
 void doConsole()
@@ -208,7 +209,8 @@ char* my_generator(const char* text, int state)
       "QTypeRule(",
       "setACL(", "setDNSSECPool(", "setECSOverride(",
       "setECSSourcePrefixV4(", "setECSSourcePrefixV6(", "setKey(", "setLocal(",
-      "setMaxTCPClientThreads(", "setMaxUDPOutstanding(", "setServerPolicy(", "setServerPolicyLua(",
+      "setMaxTCPClientThreads(", "setMaxTCPQueuedConnections(", "setMaxUDPOutstanding(", "setServerPolicy(",
+      "setServerPolicyLua(",
       "setTCPRecvTimeout(", "setTCPSendTimeout(", "setVerboseHealthChecks(", "show(", "showACL()",
       "showDNSCryptBinds()", "showDynBlocks()", "showResponseLatency()", "showRules()",
       "showServerPolicy()", "showServers()", "shutdown()", "SpoofAction(",

diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc
index d13d5a1ce507d2d9146ef39eead10724905f8f1c..677708337742bf6db6a64ed76a0f992557fb9391 100644 (file)
--- a/pdns/dnsdist-lua.cc
+++ b/pdns/dnsdist-lua.cc
@@ -1303,7 +1303,21 @@ vector<std::function<void(void)>> setupLua(bool client, const std::string& confi
   g_lua.registerMember<bool (DNSQuestion::*)>("tcp", [](const DNSQuestion& dq) -> bool { return dq.tcp; }, [](DNSQuestion& dq, bool newTcp) { (void) newTcp; });
   g_lua.registerMember<bool (DNSQuestion::*)>("skipCache", [](const DNSQuestion& dq) -> bool { return dq.skipCache; }, [](DNSQuestion& dq, bool newSkipCache) { dq.skipCache = newSkipCache; });
 
-  g_lua.writeFunction("setMaxTCPClientThreads", [](uint64_t max) { g_maxTCPClientThreads = max; });
+  g_lua.writeFunction("setMaxTCPClientThreads", [](uint64_t max) {
+      if (!g_configurationDone) {
+        g_maxTCPClientThreads = max;
+      } else {
+        g_outputBuffer="Maximum TCP client threads count cannot be altered at runtime!\n";
+      }
+    });
+
+  g_lua.writeFunction("setMaxTCPQueuedConnections", [](uint64_t max) {
+      if (!g_configurationDone) {
+        g_maxTCPQueuedConnections = max;
+      } else {
+        g_outputBuffer="The maximum number of queued TCP connections cannot be altered at runtime!\n";
+      }
+    });
 
   g_lua.writeFunction("setCacheCleaningDelay", [](uint32_t delay) { g_cacheCleaningDelay = delay; });
 

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 520b04954af1a0808ca7935884fb92f15d98f817..2cb75474b76eaaef29e294a278d5e40a763ae589 100644 (file)
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -65,13 +65,14 @@ struct ConnectionInfo
   ClientState* cs;
 };
 
+uint64_t g_maxTCPQueuedConnections{0};
 void* tcpClientThread(int pipefd);
 
 // Should not be called simultaneously!
 void TCPClientCollection::addTCPClientThread()
 {
   if (d_numthreads >= d_tcpclientthreads.capacity()) {
-    warnlog("Adding a new TCP client thread would exceed the vector capacity, skipping");
+    warnlog("Adding a new TCP client thread would exceed the vector capacity (%d/%d), skipping", d_numthreads.load(), d_tcpclientthreads.capacity());
     return;
   }
 
@@ -81,13 +82,16 @@ void TCPClientCollection::addTCPClientThread()
   if(pipe(pipefds) < 0)
     unixDie("Creating pipe");
 
-  if (!setNonBlocking(pipefds[1]))
+  if (!setNonBlocking(pipefds[1])) {
+    close(pipefds[0]);
+    close(pipefds[1]);
     unixDie("Setting pipe non-blocking");
+  }
 
   d_tcpclientthreads.push_back(pipefds[1]);
+  ++d_numthreads;
   thread t1(tcpClientThread, pipefds[0]);
   t1.detach();
-  ++d_numthreads;
 }
 
 static bool getNonBlockingMsgLen(int fd, uint16_t* len, int timeout)
@@ -585,6 +589,14 @@ void* tcpAcceptorThread(void* p)
        continue;
       }
 
+      if(g_maxTCPQueuedConnections > 0 && g_tcpclientthreads->d_queued >= g_maxTCPQueuedConnections) {
+        close(ci->fd);
+        delete ci;
+        ci=nullptr;
+        vinfolog("Dropping TCP connection from %s because we have too many queued already", remote.toStringWithPort());
+        continue;
+      }
+
       vinfolog("Got TCP connection from %s", remote.toStringWithPort());
       
       ci->remote = remote;
@@ -593,6 +605,7 @@ void* tcpAcceptorThread(void* p)
         writen2WithTimeout(pipe, &ci, sizeof(ci), 0);
       }
       else {
+        --g_tcpclientthreads->d_queued;
         close(ci->fd);
         delete ci;
       }

diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc
index 688626928bbd7e78ae1a51d21abc55fb070a1138..0fce07e9b37edc5852ebf6c4b7b6c9380305fe84 100644 (file)
--- a/pdns/dnsdist.cc
+++ b/pdns/dnsdist.cc
@@ -1025,7 +1025,7 @@ catch(...)
   return false;
 }
 
-std::atomic<uint64_t> g_maxTCPClientThreads{10};
+uint64_t g_maxTCPClientThreads{10};
 std::atomic<uint16_t> g_cacheCleaningDelay{60};
 
 void* maintThread()
@@ -1071,7 +1071,7 @@ void* healthChecksThread()
   for(;;) {
     sleep(interval);
 
-    if(g_tcpclientthreads->d_queued > 1 && g_tcpclientthreads->d_numthreads < g_maxTCPClientThreads)
+    if(g_tcpclientthreads->d_queued > 1 && g_tcpclientthreads->d_numthreads < g_tcpclientthreads->d_maxthreads)
       g_tcpclientthreads->addTCPClientThread();
 
     for(auto& dss : g_dstates.getCopy()) { // this points to the actual shared_ptrs!

diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 2a4e8b4b73347f4a5a4624d1d3e8a96e3586f1e7..446a78b3ba8e00d2d92f3af30a6a2a9c23d98375 100644 (file)
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -273,15 +273,17 @@ class TCPClientCollection {
   std::atomic<uint64_t> d_pos{0};
 public:
   std::atomic<uint64_t> d_queued{0}, d_numthreads{0};
+  uint64_t d_maxthreads{0};
 
   TCPClientCollection(size_t maxThreads)
   {
+    d_maxthreads = maxThreads;
     d_tcpclientthreads.reserve(maxThreads);
   }
 
   int getThread()
   {
-    int pos = d_pos++;
+    uint64_t pos = d_pos++;
     ++d_queued;
     return d_tcpclientthreads[pos % d_numthreads];
   }
@@ -466,7 +468,8 @@ extern int g_tcpRecvTimeout;
 extern int g_tcpSendTimeout;
 extern uint16_t g_maxOutstanding;
 extern std::atomic<bool> g_configurationDone;
-extern std::atomic<uint64_t> g_maxTCPClientThreads;
+extern uint64_t g_maxTCPClientThreads;
+extern uint64_t g_maxTCPQueuedConnections;
 extern std::atomic<uint16_t> g_cacheCleaningDelay;
 extern uint16_t g_ECSSourcePrefixV4;
 extern uint16_t g_ECSSourcePrefixV6;