rec: better estimate of number of file descriptors required

diff --git a/pdns/recursordist/pdns_recursor.cc b/pdns/recursordist/pdns_recursor.cc
index 4ab069e149dcdfb014b3e9ed1e444d273dcc9760..df12afa2c4417b93316bb512f42c4eb7a4d6a3cf 100644 (file)
--- a/pdns/recursordist/pdns_recursor.cc
+++ b/pdns/recursordist/pdns_recursor.cc
@@ -2677,7 +2677,7 @@ static void handleNewUDPQuestion(int fileDesc, FDMultiplexer::funcparam_t& /* va
   t_Counters.updateSnap(g_regressionTestMode);
 }
 
-void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log)
+unsigned int makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log)
 {
   int one = 1;
   vector<string> localAddresses;
@@ -2769,6 +2769,7 @@ void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log)
     SLOG(g_log << Logger::Info << "Listening for UDP queries on " << address.toStringWithPort() << endl,
          log->info(Logr::Info, "Listening for queries", "proto", Logging::Loggable("UDP"), "address", Logging::Loggable(address)));
   }
+  return localAddresses.size();
 }
 
 static bool trySendingQueryToWorker(unsigned int target, ThreadMSG* tmsg)

diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc
index 39a1d2c6971927153ba58cfdf667a57a2818c64e..a9b0e595cdefef826dc5dc6120c752d102644c88 100644 (file)
--- a/pdns/recursordist/rec-main.cc
+++ b/pdns/recursordist/rec-main.cc
@@ -997,23 +997,47 @@ static void checkOrFixLinuxMapCountLimits([[maybe_unused]] Logr::log_t log)
 #endif
 }
 
-static void checkOrFixFDS(Logr::log_t log)
-{
-  unsigned int availFDs = getFilenumLimit();
-  unsigned int wantFDs = g_maxMThreads * (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers()) + 25; // even healthier margin than before
-  wantFDs += (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers()) * TCPOutConnectionManager::s_maxIdlePerThread;
+static void checkOrFixFDS(unsigned int listeningSockets, Logr::log_t log)
+{
+  const auto availFDs = getFilenumLimit();
+  // Posix threads
+  const auto threads = RecThreadInfo::numRecursorThreads();
+  // We do not count the handler and task threads, they do not spawn many mthreads at once
+  const auto workers = RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers();
+
+  // Static part: the FDs from the start, pipes, controlsocket, web socket, listen sockets
+  unsigned int staticPart = 25; // general  allowance, including control socket, web, snmp
+  // Handler thread gets one pipe, the others all of them
+  staticPart += 2 + (threads - 1) * (sizeof(RecThreadInfo::ThreadPipeSet) / sizeof(int)); // number of fd's in ThreadPipeSet
+  // listen sockets
+  staticPart += listeningSockets;
+  // Another fd per thread for poll/kqueue
+  staticPart += threads;
+  // Incoming TCP, connections are shared by threads and are kept open for a while
+  staticPart += g_maxTCPClients;
+
+  // Dynamic parts per worker
+  // Each mthread uses one fd for either outgoing UDP or outgoing TCP (but not simultaneously)
+  unsigned int perWorker = g_maxMThreads;
+  // plus each worker thread can have a number of idle outgoing TCP connections
+  perWorker += TCPOutConnectionManager::s_maxIdlePerThread;
+
+  auto wantFDs = staticPart + workers * perWorker;
 
   if (wantFDs > availFDs) {
     unsigned int hardlimit = getFilenumLimit(true);
+    if (staticPart >= hardlimit) {
+      log->info(Logr::Critical, "Number of available filedescriptors is lower than the minimum needed",
+                "hardlimit", Logging::Loggable(hardlimit), "minimum", Logging::Loggable(staticPart));
+      _exit(1);
+    }
     if (hardlimit >= wantFDs) {
       setFilenumLimit(wantFDs);
-      SLOG(g_log << Logger::Warning << "Raised soft limit on number of filedescriptors to " << wantFDs << " to match max-mthreads and threads settings" << endl,
-           log->info(Logr::Warning, "Raised soft limit on number of filedescriptors to match max-mthreads and threads settings", "limit", Logging::Loggable(wantFDs)));
+      log->info(Logr::Warning, "Raised soft limit on number of filedescriptors to match max-mthreads and threads settings", "limit", Logging::Loggable(wantFDs));
     }
     else {
-      auto newval = (hardlimit - 25 - TCPOutConnectionManager::s_maxIdlePerThread) / (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers());
-      SLOG(g_log << Logger::Warning << "Insufficient number of filedescriptors available for max-mthreads*threads setting! (" << hardlimit << " < " << wantFDs << "), reducing max-mthreads to " << newval << endl,
-           log->info(Logr::Warning, "Insufficient number of filedescriptors available for max-mthreads*threads setting! Reducing max-mthreads", "hardlimit", Logging::Loggable(hardlimit), "want", Logging::Loggable(wantFDs), "max-mthreads", Logging::Loggable(newval)));
+      auto newval = (hardlimit - staticPart) / workers;
+      log->info(Logr::Warning, "Insufficient number of filedescriptors available for max-mthreads*threads setting! Reducing max-mthreads", "hardlimit", Logging::Loggable(hardlimit), "want", Logging::Loggable(wantFDs), "max-mthreads", Logging::Loggable(newval));
       g_maxMThreads = newval;
       setFilenumLimit(hardlimit);
     }
@@ -1841,8 +1865,9 @@ static int initSyncRes(Logr::log_t log)
   return 0;
 }
 
-static void initDistribution(Logr::log_t log)
+static unsigned int initDistribution(Logr::log_t log)
 {
+  unsigned int count = 0;
   g_balancingFactor = ::arg().asDouble("distribution-load-factor");
   if (g_balancingFactor != 0.0 && g_balancingFactor < 1.0) {
     g_balancingFactor = 0.0;
@@ -1863,7 +1888,7 @@ static void initDistribution(Logr::log_t log)
       for (unsigned int i = 0; i < RecThreadInfo::numDistributors(); i++, threadNum++) {
         auto& info = RecThreadInfo::info(threadNum);
         auto& deferredAdds = info.getDeferredAdds();
-        makeUDPServerSockets(deferredAdds, log);
+        count += makeUDPServerSockets(deferredAdds, log);
       }
     }
     else {
@@ -1871,7 +1896,7 @@ static void initDistribution(Logr::log_t log)
       for (unsigned int i = 0; i < RecThreadInfo::numUDPWorkers(); i++, threadNum++) {
         auto& info = RecThreadInfo::info(threadNum);
         auto& deferredAdds = info.getDeferredAdds();
-        makeUDPServerSockets(deferredAdds, log);
+        count += makeUDPServerSockets(deferredAdds, log);
       }
     }
     threadNum = 1 + RecThreadInfo::numDistributors() + RecThreadInfo::numUDPWorkers();
@@ -1879,15 +1904,15 @@ static void initDistribution(Logr::log_t log)
       auto& info = RecThreadInfo::info(threadNum);
       auto& deferredAdds = info.getDeferredAdds();
       auto& tcpSockets = info.getTCPSockets();
-      makeTCPServerSockets(deferredAdds, tcpSockets, log);
+      count += makeTCPServerSockets(deferredAdds, tcpSockets, log);
     }
   }
   else {
     std::set<int> tcpSockets;
     /* we don't have reuseport so we can only open one socket per
        listening addr:port and everyone will listen on it */
-    makeUDPServerSockets(s_deferredUDPadds, log);
-    makeTCPServerSockets(s_deferredTCPadds, tcpSockets, log);
+    count += makeUDPServerSockets(s_deferredUDPadds, log);
+    count += makeTCPServerSockets(s_deferredTCPadds, tcpSockets, log);
 
     // TCP queries are handled by TCP workers
     for (unsigned int i = 0; i < RecThreadInfo::numTCPWorkers(); i++) {
@@ -1895,6 +1920,7 @@ static void initDistribution(Logr::log_t log)
       info.setTCPSockets(tcpSockets);
     }
   }
+  return count;
 }
 
 static int initForks(Logr::log_t log)
@@ -2280,7 +2306,7 @@ static int serviceMain(Logr::log_t log)
 
   initSuffixMatchNodes(log);
   initCarbon();
-  initDistribution(log);
+  auto listeningSockets = initDistribution(log);
 
 #ifdef NOD_ENABLED
   // Setup newly observed domain globals
@@ -2289,7 +2315,16 @@ static int serviceMain(Logr::log_t log)
 
   auto forks = initForks(log);
 
-  checkOrFixFDS(log);
+  g_tcpTimeout = ::arg().asNum("client-tcp-timeout");
+  g_maxTCPClients = ::arg().asNum("max-tcp-clients");
+  g_maxTCPPerClient = ::arg().asNum("max-tcp-per-client");
+  g_tcpMaxQueriesPerConn = ::arg().asNum("max-tcp-queries-per-connection");
+  g_maxUDPQueriesPerRound = ::arg().asNum("max-udp-queries-per-round");
+
+  g_useKernelTimestamp = ::arg().mustDo("protobuf-use-kernel-timestamp");
+  g_maxChainLength = ::arg().asNum("max-chain-length");
+
+  checkOrFixFDS(listeningSockets, log);
   checkOrFixLinuxMapCountLimits(log);
 
 #ifdef HAVE_LIBSODIUM
@@ -2326,15 +2361,6 @@ static int serviceMain(Logr::log_t log)
 
   RecThreadInfo::makeThreadPipes(log);
 
-  g_tcpTimeout = ::arg().asNum("client-tcp-timeout");
-  g_maxTCPClients = ::arg().asNum("max-tcp-clients");
-  g_maxTCPPerClient = ::arg().asNum("max-tcp-per-client");
-  g_tcpMaxQueriesPerConn = ::arg().asNum("max-tcp-queries-per-connection");
-  g_maxUDPQueriesPerRound = ::arg().asNum("max-udp-queries-per-round");
-
-  g_useKernelTimestamp = ::arg().mustDo("protobuf-use-kernel-timestamp");
-  g_maxChainLength = ::arg().asNum("max-chain-length");
-
   disableStats(StatComponent::API, ::arg()["stats-api-blacklist"]);
   disableStats(StatComponent::Carbon, ::arg()["stats-carbon-blacklist"]);
   disableStats(StatComponent::RecControl, ::arg()["stats-rec-control-blacklist"]);

diff --git a/pdns/recursordist/rec-main.hh b/pdns/recursordist/rec-main.hh
index 2654f8f5d521633c89c64e40d95fac4326b3fd3f..08f60145e42837f49ec404a8a35d8694d7b5a324 100644 (file)
--- a/pdns/recursordist/rec-main.hh
+++ b/pdns/recursordist/rec-main.hh
@@ -615,10 +615,10 @@ bool expectProxyProtocol(const ComboAddress& from, const ComboAddress& listenAdd
 void finishTCPReply(std::unique_ptr<DNSComboWriter>&, bool hadError, bool updateInFlight);
 void checkFastOpenSysctl(bool active, Logr::log_t);
 void checkTFOconnect(Logr::log_t);
-void makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set<int>& tcpSockets, Logr::log_t);
+unsigned int makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set<int>& tcpSockets, Logr::log_t);
 void handleNewTCPQuestion(int fileDesc, FDMultiplexer::funcparam_t&);
 
-void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t);
+unsigned int makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t);
 string doTraceRegex(FDWrapper file, vector<string>::const_iterator begin, vector<string>::const_iterator end);
 extern bool g_luaSettingsInYAML;
 void startLuaConfigDelayedThreads(const LuaConfigItems& luaConfig, uint64_t generation);

diff --git a/pdns/recursordist/rec-tcp.cc b/pdns/recursordist/rec-tcp.cc
index 3cd714e1c67229e0c9e3cbb3fd691f0e12f4e344..ae0505c6eff51648c30085e50a5a9792cbd1be70 100644 (file)
--- a/pdns/recursordist/rec-tcp.cc
+++ b/pdns/recursordist/rec-tcp.cc
@@ -1085,7 +1085,7 @@ LWResult::Result arecvtcp(PacketBuffer& data, const size_t len, shared_ptr<TCPIO
   return LWResult::Result::Success;
 }
 
-void makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set<int>& tcpSockets, Logr::log_t log)
+unsigned int makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set<int>& tcpSockets, Logr::log_t log)
 {
   vector<string> localAddresses;
   stringtok(localAddresses, ::arg()["local-address"], " ,");
@@ -1192,4 +1192,5 @@ void makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set<int>& tcpSockets
     first = false;
 #endif
   }
+  return localAddresses.size();
 }