ostringstream ret;
boost::format fmt("%-12d %-12d %-12d %-12d");
ret << (fmt % "Workers" % "Max Workers" % "Queued" % "Max Queued") << endl;
- ret << (fmt % g_tcpclientthreads->getThreadsCount() % g_maxTCPClientThreads % g_tcpclientthreads->getQueuedCount() % g_maxTCPQueuedConnections) << endl;
+ ret << (fmt % g_tcpclientthreads->getThreadsCount() % (g_maxTCPClientThreads ? *g_maxTCPClientThreads : 0) % g_tcpclientthreads->getQueuedCount() % g_maxTCPQueuedConnections) << endl;
ret << endl;
ret << "Query distribution mode is: " << std::string(g_useTCPSinglePipe ? "single queue" : "per-thread queues") << endl;
{
std::lock_guard<std::mutex> lock(d_mutex);
- if (d_numthreads >= d_tcpclientthreads.capacity()) {
- warnlog("Adding a new TCP client thread would exceed the vector capacity (%d/%d), skipping", d_numthreads.load(), d_tcpclientthreads.capacity());
+ if (d_numthreads >= d_tcpclientthreads.size()) {
+ vinfolog("Adding a new TCP client thread would exceed the vector capacity (%d/%d), skipping", d_numthreads.load(), d_tcpclientthreads.size());
if (!d_useSinglePipe) {
close(pipefds[0]);
close(pipefds[1]);
return;
}
- d_tcpclientthreads.push_back(pipefds[1]);
+ d_tcpclientthreads.at(d_numthreads) = pipefds[1];
++d_numthreads;
}
}
ComboAddress remote;
remote.sin4.sin_family = cs->local.sin4.sin_family;
- if(!g_tcpclientthreads->hasReachedMaxThreads()) {
- g_tcpclientthreads->addTCPClientThread();
- }
-
auto acl = g_ACL.getLocal();
for(;;) {
bool queuedCounterIncremented = false;
#endif
}
-uint64_t g_maxTCPClientThreads{10};
+boost::optional<uint64_t> g_maxTCPClientThreads{boost::none};
std::atomic<uint16_t> g_cacheCleaningDelay{60};
std::atomic<uint16_t> g_cacheCleaningPercentage{100};
for(;;) {
sleep(interval);
- if(g_tcpclientthreads->getQueuedCount() > 1 && !g_tcpclientthreads->hasReachedMaxThreads()) {
- g_tcpclientthreads->addTCPClientThread();
- }
-
auto mplexer = std::shared_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent());
auto states = g_dstates.getLocal(); // this points to the actual shared_ptrs!
for(auto& dss : *states) {
}
requiredFDsCount += backendUDPSocketsCount;
/* TCP sockets to backends */
- requiredFDsCount += (backends->size() * g_maxTCPClientThreads);
+ if (g_maxTCPClientThreads) {
+ requiredFDsCount += (backends->size() * (*g_maxTCPClientThreads));
+ }
/* listening sockets */
requiredFDsCount += udpBindsCount;
requiredFDsCount += tcpBindsCount;
- /* max TCP connections currently served */
- requiredFDsCount += g_maxTCPClientThreads;
- /* max pipes for communicating between TCP acceptors and client threads */
- requiredFDsCount += (g_maxTCPClientThreads * 2);
+ /* number of TCP connections currently served, assuming 1 connection per worker thread which is of course not right */
+ if (g_maxTCPClientThreads) {
+ requiredFDsCount += *g_maxTCPClientThreads;
+ /* max pipes for communicating between TCP acceptors and client threads */
+ requiredFDsCount += (*g_maxTCPClientThreads * 2);
+ }
/* max TCP queued connections */
requiredFDsCount += g_maxTCPQueuedConnections;
/* DelayPipe pipe */
g_snmpAgent->run();
}
- g_tcpclientthreads = std::unique_ptr<TCPClientCollection>(new TCPClientCollection(g_maxTCPClientThreads, g_useTCPSinglePipe));
+ if (!g_maxTCPClientThreads) {
+ g_maxTCPClientThreads = std::max(tcpBindsCount, static_cast<size_t>(10));
+ }
+ else if (*g_maxTCPClientThreads == 0 && tcpBindsCount > 0) {
+ warnlog("setMaxTCPClientThreads() has been set to 0 while we are accepting TCP connections, raising to 1");
+ g_maxTCPClientThreads = 1;
+ }
+
+ g_tcpclientthreads = std::unique_ptr<TCPClientCollection>(new TCPClientCollection(*g_maxTCPClientThreads, g_useTCPSinglePipe));
- for(auto& t : todo)
+ for (auto& t : todo) {
t();
+ }
localPools = g_pools.getCopy();
/* create the default pool no matter what */
}
}
+ while (!g_tcpclientthreads->hasReachedMaxThreads()) {
+ g_tcpclientthreads->addTCPClientThread();
+ }
+
thread carbonthread(carbonDumpThread);
carbonthread.detach();
const bool d_useSinglePipe;
public:
- TCPClientCollection(size_t maxThreads, bool useSinglePipe=false): d_maxthreads(maxThreads), d_singlePipe{-1,-1}, d_useSinglePipe(useSinglePipe)
+ TCPClientCollection(size_t maxThreads, bool useSinglePipe=false): d_tcpclientthreads(maxThreads), d_maxthreads(maxThreads), d_singlePipe{-1,-1}, d_useSinglePipe(useSinglePipe)
{
- d_tcpclientthreads.reserve(maxThreads);
-
if (d_useSinglePipe) {
if (pipe(d_singlePipe) < 0) {
int err = errno;
{
uint64_t pos = d_pos++;
++d_queued;
- return d_tcpclientthreads[pos % d_numthreads];
+ return d_tcpclientthreads.at(pos % d_numthreads);
}
bool hasReachedMaxThreads() const
{
extern int g_udpTimeout;
extern uint16_t g_maxOutstanding;
extern std::atomic<bool> g_configurationDone;
-extern uint64_t g_maxTCPClientThreads;
+extern boost::optional<uint64_t> g_maxTCPClientThreads;
extern uint64_t g_maxTCPQueuedConnections;
extern size_t g_maxTCPQueriesPerConn;
extern size_t g_maxTCPConnectionDuration;
* Each local bind has its own thread listening for incoming UDP queries
* and its own thread listening for incoming TCP connections, dispatching them right away to a pool of threads
- * Each backend has its own thread listening for UDP responses
+ * Each backend has its own thread listening for UDP responses, including the ones triggered by DoH queries, if any
* A maintenance thread calls the maintenance() Lua function every second if any, and is responsible for cleaning the cache
* A health check thread checks the backends availability
* A control thread handles console connections
* A carbon thread exports statistics to carbon servers if needed
* One or more webserver threads handle queries to the internal webserver
-The maximum number of threads in the TCP pool is controlled by the :func:`setMaxTCPClientThreads` directive, and defaults to 10.
-This number can be increased to handle a large number of simultaneous TCP connections.
+TCP and DNS over TLS
+--------------------
+
+The maximum number of threads in the TCP / DNS over TLS pool is controlled by the :func:`setMaxTCPClientThreads` directive, and defaults to 10.
+This number can be increased to handle a large number of simultaneous TCP / DNS over TLS connections.
If all the TCP threads are busy, new TCP connections are queued while they wait to be picked up.
Before 1.4.0, a TCP thread could only handle a single incoming connection at a time. Starting with 1.4.0 the handling of TCP connections is now event-based, so a single TCP worker can handle a large number of TCP incoming connections simultaneously.
+Note that before 1.6.0 the TCP worker threads were created at runtime, adding a new thread when the existing ones seemed to struggle with the load, until the maximum number of threads had been reached. Starting with 1.6.0 the configured number of worker threads are immediately created at startup.
The maximum number of queued connections can be configured with :func:`setMaxTCPQueuedConnections` and defaults to 1000.
Any value larger than 0 will cause new connections to be dropped if there are already too many queued.
The experimental :func:`setTCPUseSinglePipe` directive can be used so that all the incoming TCP connections are put into a single queue and handled by the first TCP worker available.
-When dispatching UDP queries to backend servers, dnsdist keeps track of at most **n** outstanding queries for each backend.
-This number **n** can be tuned by the :func:`setMaxUDPOutstanding` directive, defaulting to 10240 (65535 since 1.4.0), with a maximum value of 65535.
-Large installations are advised to increase the default value at the cost of a slightly increased memory usage.
+Rules and Lua
+-------------
Most of the query processing is done in C++ for maximum performance, but some operations are executed in Lua for maximum flexibility:
While Lua is fast, its use should be restricted to the strict necessary in order to achieve maximum performance, it might be worth considering using LuaJIT instead of Lua.
When Lua inspection is needed, the best course of action is to restrict the queries sent to Lua inspection by using :func:`addLuaAction` with a selector.
-:program:`dnsdist` design choices mean that the processing of UDP queries is done by only one thread per local bind.
+UDP and DNS over HTTPS
+-----------------------
+
+:program:`dnsdist` design choices mean that the processing of UDP and DNS over HTTPS queries is done by only one thread per local bind.
This is great to keep lock contention to a low level, but might not be optimal for setups using a lot of processing power, caused for example by a large number of complicated rules.
To be able to use more CPU cores for UDP queries processing, it is possible to use the ``reusePort`` parameter of the :func:`addLocal` and :func:`setLocal` directives to be able to add several identical local binds to dnsdist::
:program:`dnsdist` will then add four identical local binds as if they were different IPs or ports, start four threads to handle incoming queries and let the kernel load balance those randomly to the threads, thus using four CPU cores for rules processing.
Note that this require ``SO_REUSEPORT`` support in the underlying operating system (added for example in Linux 3.9).
-Please also be aware that doing so will increase lock contention and might not therefore scale linearly.
-This is especially true for Lua-intensive setups, because Lua processing in dnsdist is serialized by a unique lock for all threads.
+Please also be aware that doing so will increase lock contention and might not therefore scale linearly, as discussed below.
Another possibility is to use the reuseport option to run several dnsdist processes in parallel on the same host, thus avoiding the lock contention issue at the cost of having to deal with the fact that the different processes will not share informations, like statistics or DDoS offenders.
newServer({address="192.0.2.127:53", name="Backend2"})
newServer({address="192.0.2.127:53", name="Backend3"})
newServer({address="192.0.2.127:53", name="Backend4"})
+
+For DNS over HTTPS, every :func:`addDOHLocal` directive adds a new thread dealing with incoming connections, so it might be useful to add more than one directive, as indicated above.
+
+When dealing with a large traffic load, it might happen that the internal pipe used to pass queries between the threads handling the incoming connections and the one getting a response from the backend become full too quickly, degrading performance and causing timeouts. This can be prevented by increasing the size of the internal pipe buffer, via the `internalPipeBufferSize` option of :func:`addDOHLocal`. Setting a value of `1048576` is known to yield good results on Linux.
+
+When dispatching UDP queries to backend servers, dnsdist keeps track of at most **n** outstanding queries for each backend.
+This number **n** can be tuned by the :func:`setMaxUDPOutstanding` directive, defaulting to 10240 (65535 since 1.4.0), with a maximum value of 65535.
+Large installations are advised to increase the default value at the cost of a slightly increased memory usage.
+
+Lock contention and sharding
+----------------------------
+
+Adding more threads makes it possible to use more CPU cores to deal with the load, but at the cost of possibly increasing lock contention between threads. This is especially true for Lua-intensive setups, because Lua processing in dnsdist is serialized by a unique lock for all threads.
+For other components, like the packet cache and the in-memory ring buffers, it is possible to reduce the amount of contention by using sharding. Sharding divides the memory into several pieces, every one of these having its own separate lock, reducing the amount of times two threads or more will need to access the same data.
+Sharding is disabled by default and can be enabled via the `newPacketCache` option to :func:`newPacketCache` and :func:`setRingBuffersSize`.
.. function:: setMaxTCPClientThreads(num)
+ .. versionchanged:: 1.6.0
+ Before 1.6.0 the default value was 10.
+
Set the maximum of TCP client threads, handling TCP connections. Before 1.4.0 a TCP thread could only handle a single incoming TCP connection at a time, while after 1.4.0 it can handle a larger number of them simultaneously.
+ Since 1.6.0, the default value is at least 10 TCP workers, but might be more if there is more than 10 TCP listeners (added via :func:`addDNSCryptBind`, :func:`addLocal`, or :func:`addTLSLocal`). In that last case there will be as many TCP workers as TCP listeners.
+ Note that before 1.6.0 the TCP worker threads were created at runtime, adding a new thread when the existing ones seemed to struggle with the load, until the maximum number of threads had been reached. Starting with 1.6.0 the configured number of worker threads are immediately created at startup.
:param int num:
projects / thirdparty / pdns.git / commitdiff
