From: Otto Moerbeek Date: Wed, 6 Nov 2024 13:29:42 +0000 (+0100) Subject: rec: better estimate of number of file descriptors required X-Git-Tag: rec-5.2.0-beta1~27^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=618d561ba64531794d6a43a0bf4026a9b53c05f9;p=thirdparty%2Fpdns.git rec: better estimate of number of file descriptors required --- diff --git a/pdns/recursordist/pdns_recursor.cc b/pdns/recursordist/pdns_recursor.cc index 4ab069e149..df12afa2c4 100644 --- a/pdns/recursordist/pdns_recursor.cc +++ b/pdns/recursordist/pdns_recursor.cc @@ -2677,7 +2677,7 @@ static void handleNewUDPQuestion(int fileDesc, FDMultiplexer::funcparam_t& /* va t_Counters.updateSnap(g_regressionTestMode); } -void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log) +unsigned int makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log) { int one = 1; vector localAddresses; @@ -2769,6 +2769,7 @@ void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t log) SLOG(g_log << Logger::Info << "Listening for UDP queries on " << address.toStringWithPort() << endl, log->info(Logr::Info, "Listening for queries", "proto", Logging::Loggable("UDP"), "address", Logging::Loggable(address))); } + return localAddresses.size(); } static bool trySendingQueryToWorker(unsigned int target, ThreadMSG* tmsg) diff --git a/pdns/recursordist/rec-main.cc b/pdns/recursordist/rec-main.cc index 39a1d2c697..a9b0e595cd 100644 --- a/pdns/recursordist/rec-main.cc +++ b/pdns/recursordist/rec-main.cc @@ -997,23 +997,47 @@ static void checkOrFixLinuxMapCountLimits([[maybe_unused]] Logr::log_t log) #endif } -static void checkOrFixFDS(Logr::log_t log) -{ - unsigned int availFDs = getFilenumLimit(); - unsigned int wantFDs = g_maxMThreads * (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers()) + 25; // even healthier margin than before - wantFDs += (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers()) * TCPOutConnectionManager::s_maxIdlePerThread; +static void checkOrFixFDS(unsigned int listeningSockets, Logr::log_t log) +{ + const auto availFDs = getFilenumLimit(); + // Posix threads + const auto threads = RecThreadInfo::numRecursorThreads(); + // We do not count the handler and task threads, they do not spawn many mthreads at once + const auto workers = RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers(); + + // Static part: the FDs from the start, pipes, controlsocket, web socket, listen sockets + unsigned int staticPart = 25; // general allowance, including control socket, web, snmp + // Handler thread gets one pipe, the others all of them + staticPart += 2 + (threads - 1) * (sizeof(RecThreadInfo::ThreadPipeSet) / sizeof(int)); // number of fd's in ThreadPipeSet + // listen sockets + staticPart += listeningSockets; + // Another fd per thread for poll/kqueue + staticPart += threads; + // Incoming TCP, connections are shared by threads and are kept open for a while + staticPart += g_maxTCPClients; + + // Dynamic parts per worker + // Each mthread uses one fd for either outgoing UDP or outgoing TCP (but not simultaneously) + unsigned int perWorker = g_maxMThreads; + // plus each worker thread can have a number of idle outgoing TCP connections + perWorker += TCPOutConnectionManager::s_maxIdlePerThread; + + auto wantFDs = staticPart + workers * perWorker; if (wantFDs > availFDs) { unsigned int hardlimit = getFilenumLimit(true); + if (staticPart >= hardlimit) { + log->info(Logr::Critical, "Number of available filedescriptors is lower than the minimum needed", + "hardlimit", Logging::Loggable(hardlimit), "minimum", Logging::Loggable(staticPart)); + _exit(1); + } if (hardlimit >= wantFDs) { setFilenumLimit(wantFDs); - SLOG(g_log << Logger::Warning << "Raised soft limit on number of filedescriptors to " << wantFDs << " to match max-mthreads and threads settings" << endl, - log->info(Logr::Warning, "Raised soft limit on number of filedescriptors to match max-mthreads and threads settings", "limit", Logging::Loggable(wantFDs))); + log->info(Logr::Warning, "Raised soft limit on number of filedescriptors to match max-mthreads and threads settings", "limit", Logging::Loggable(wantFDs)); } else { - auto newval = (hardlimit - 25 - TCPOutConnectionManager::s_maxIdlePerThread) / (RecThreadInfo::numUDPWorkers() + RecThreadInfo::numTCPWorkers()); - SLOG(g_log << Logger::Warning << "Insufficient number of filedescriptors available for max-mthreads*threads setting! (" << hardlimit << " < " << wantFDs << "), reducing max-mthreads to " << newval << endl, - log->info(Logr::Warning, "Insufficient number of filedescriptors available for max-mthreads*threads setting! Reducing max-mthreads", "hardlimit", Logging::Loggable(hardlimit), "want", Logging::Loggable(wantFDs), "max-mthreads", Logging::Loggable(newval))); + auto newval = (hardlimit - staticPart) / workers; + log->info(Logr::Warning, "Insufficient number of filedescriptors available for max-mthreads*threads setting! Reducing max-mthreads", "hardlimit", Logging::Loggable(hardlimit), "want", Logging::Loggable(wantFDs), "max-mthreads", Logging::Loggable(newval)); g_maxMThreads = newval; setFilenumLimit(hardlimit); } @@ -1841,8 +1865,9 @@ static int initSyncRes(Logr::log_t log) return 0; } -static void initDistribution(Logr::log_t log) +static unsigned int initDistribution(Logr::log_t log) { + unsigned int count = 0; g_balancingFactor = ::arg().asDouble("distribution-load-factor"); if (g_balancingFactor != 0.0 && g_balancingFactor < 1.0) { g_balancingFactor = 0.0; @@ -1863,7 +1888,7 @@ static void initDistribution(Logr::log_t log) for (unsigned int i = 0; i < RecThreadInfo::numDistributors(); i++, threadNum++) { auto& info = RecThreadInfo::info(threadNum); auto& deferredAdds = info.getDeferredAdds(); - makeUDPServerSockets(deferredAdds, log); + count += makeUDPServerSockets(deferredAdds, log); } } else { @@ -1871,7 +1896,7 @@ static void initDistribution(Logr::log_t log) for (unsigned int i = 0; i < RecThreadInfo::numUDPWorkers(); i++, threadNum++) { auto& info = RecThreadInfo::info(threadNum); auto& deferredAdds = info.getDeferredAdds(); - makeUDPServerSockets(deferredAdds, log); + count += makeUDPServerSockets(deferredAdds, log); } } threadNum = 1 + RecThreadInfo::numDistributors() + RecThreadInfo::numUDPWorkers(); @@ -1879,15 +1904,15 @@ static void initDistribution(Logr::log_t log) auto& info = RecThreadInfo::info(threadNum); auto& deferredAdds = info.getDeferredAdds(); auto& tcpSockets = info.getTCPSockets(); - makeTCPServerSockets(deferredAdds, tcpSockets, log); + count += makeTCPServerSockets(deferredAdds, tcpSockets, log); } } else { std::set tcpSockets; /* we don't have reuseport so we can only open one socket per listening addr:port and everyone will listen on it */ - makeUDPServerSockets(s_deferredUDPadds, log); - makeTCPServerSockets(s_deferredTCPadds, tcpSockets, log); + count += makeUDPServerSockets(s_deferredUDPadds, log); + count += makeTCPServerSockets(s_deferredTCPadds, tcpSockets, log); // TCP queries are handled by TCP workers for (unsigned int i = 0; i < RecThreadInfo::numTCPWorkers(); i++) { @@ -1895,6 +1920,7 @@ static void initDistribution(Logr::log_t log) info.setTCPSockets(tcpSockets); } } + return count; } static int initForks(Logr::log_t log) @@ -2280,7 +2306,7 @@ static int serviceMain(Logr::log_t log) initSuffixMatchNodes(log); initCarbon(); - initDistribution(log); + auto listeningSockets = initDistribution(log); #ifdef NOD_ENABLED // Setup newly observed domain globals @@ -2289,7 +2315,16 @@ static int serviceMain(Logr::log_t log) auto forks = initForks(log); - checkOrFixFDS(log); + g_tcpTimeout = ::arg().asNum("client-tcp-timeout"); + g_maxTCPClients = ::arg().asNum("max-tcp-clients"); + g_maxTCPPerClient = ::arg().asNum("max-tcp-per-client"); + g_tcpMaxQueriesPerConn = ::arg().asNum("max-tcp-queries-per-connection"); + g_maxUDPQueriesPerRound = ::arg().asNum("max-udp-queries-per-round"); + + g_useKernelTimestamp = ::arg().mustDo("protobuf-use-kernel-timestamp"); + g_maxChainLength = ::arg().asNum("max-chain-length"); + + checkOrFixFDS(listeningSockets, log); checkOrFixLinuxMapCountLimits(log); #ifdef HAVE_LIBSODIUM @@ -2326,15 +2361,6 @@ static int serviceMain(Logr::log_t log) RecThreadInfo::makeThreadPipes(log); - g_tcpTimeout = ::arg().asNum("client-tcp-timeout"); - g_maxTCPClients = ::arg().asNum("max-tcp-clients"); - g_maxTCPPerClient = ::arg().asNum("max-tcp-per-client"); - g_tcpMaxQueriesPerConn = ::arg().asNum("max-tcp-queries-per-connection"); - g_maxUDPQueriesPerRound = ::arg().asNum("max-udp-queries-per-round"); - - g_useKernelTimestamp = ::arg().mustDo("protobuf-use-kernel-timestamp"); - g_maxChainLength = ::arg().asNum("max-chain-length"); - disableStats(StatComponent::API, ::arg()["stats-api-blacklist"]); disableStats(StatComponent::Carbon, ::arg()["stats-carbon-blacklist"]); disableStats(StatComponent::RecControl, ::arg()["stats-rec-control-blacklist"]); diff --git a/pdns/recursordist/rec-main.hh b/pdns/recursordist/rec-main.hh index 2654f8f5d5..08f60145e4 100644 --- a/pdns/recursordist/rec-main.hh +++ b/pdns/recursordist/rec-main.hh @@ -615,10 +615,10 @@ bool expectProxyProtocol(const ComboAddress& from, const ComboAddress& listenAdd void finishTCPReply(std::unique_ptr&, bool hadError, bool updateInFlight); void checkFastOpenSysctl(bool active, Logr::log_t); void checkTFOconnect(Logr::log_t); -void makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set& tcpSockets, Logr::log_t); +unsigned int makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set& tcpSockets, Logr::log_t); void handleNewTCPQuestion(int fileDesc, FDMultiplexer::funcparam_t&); -void makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t); +unsigned int makeUDPServerSockets(deferredAdd_t& deferredAdds, Logr::log_t); string doTraceRegex(FDWrapper file, vector::const_iterator begin, vector::const_iterator end); extern bool g_luaSettingsInYAML; void startLuaConfigDelayedThreads(const LuaConfigItems& luaConfig, uint64_t generation); diff --git a/pdns/recursordist/rec-tcp.cc b/pdns/recursordist/rec-tcp.cc index 3cd714e1c6..ae0505c6ef 100644 --- a/pdns/recursordist/rec-tcp.cc +++ b/pdns/recursordist/rec-tcp.cc @@ -1085,7 +1085,7 @@ LWResult::Result arecvtcp(PacketBuffer& data, const size_t len, shared_ptr& tcpSockets, Logr::log_t log) +unsigned int makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set& tcpSockets, Logr::log_t log) { vector localAddresses; stringtok(localAddresses, ::arg()["local-address"], " ,"); @@ -1192,4 +1192,5 @@ void makeTCPServerSockets(deferredAdd_t& deferredAdds, std::set& tcpSockets first = false; #endif } + return localAddresses.size(); }