g_outputBuffer="setRingBuffersSize() cannot be used at runtime!\n";
return;
}
- g_rings.setCapacity(capacity, numberOfShards ? *numberOfShards : 1);
+ g_rings.setCapacity(capacity, numberOfShards ? *numberOfShards : 10);
});
luaCtx.writeFunction("setRingBuffersLockRetries", [](size_t retries) {
std::mutex respLock;
};
- Rings(size_t capacity=10000, size_t numberOfShards=1, size_t nbLockTries=5, bool keepLockingStats=false): d_blockingQueryInserts(0), d_blockingResponseInserts(0), d_deferredQueryInserts(0), d_deferredResponseInserts(0), d_nbQueryEntries(0), d_nbResponseEntries(0), d_currentShardId(0), d_numberOfShards(numberOfShards), d_nbLockTries(nbLockTries), d_keepLockingStats(keepLockingStats)
+ Rings(size_t capacity=10000, size_t numberOfShards=10, size_t nbLockTries=5, bool keepLockingStats=false): d_blockingQueryInserts(0), d_blockingResponseInserts(0), d_deferredQueryInserts(0), d_deferredResponseInserts(0), d_nbQueryEntries(0), d_nbResponseEntries(0), d_currentShardId(0), d_numberOfShards(numberOfShards), d_nbLockTries(nbLockTries), d_keepLockingStats(keepLockingStats)
{
setCapacity(capacity, numberOfShards);
- if (numberOfShards <= 1) {
- d_nbLockTries = 0;
- }
}
+
std::unordered_map<int, vector<boost::variant<string,double> > > getTopBandwidth(unsigned int numentries);
size_t numDistinctRequestors();
/* This function should only be called at configuration time before any query or response has been inserted */
void setCapacity(size_t newCapacity, size_t numberOfShards)
{
- if (numberOfShards < d_numberOfShards) {
- throw std::runtime_error("Decreasing the number of shards in the query and response rings is not supported");
+ if (numberOfShards <= 1) {
+ d_nbLockTries = 0;
}
d_shards.resize(numberOfShards);
static std::mutex s_tcpClientsCountMutex;
static std::map<ComboAddress,size_t,ComboAddress::addressOnlyLessThan> s_tcpClientsCount;
-uint64_t g_maxTCPQueuedConnections{1000};
size_t g_maxTCPQueriesPerConn{0};
size_t g_maxTCPConnectionDuration{0};
size_t g_maxTCPConnectionsPerClient{0};
+#ifdef __linux__
+// On Linux this gives us 128k pending queries (default is 8192 queries),
+// which should be enough to deal with huge spikes
+size_t g_tcpInternalPipeBufferSize{1024*1024};
+uint64_t g_maxTCPQueuedConnections{10000};
+#else
size_t g_tcpInternalPipeBufferSize{0};
+uint64_t g_maxTCPQueuedConnections{1000};
+#endif
uint16_t g_downstreamTCPCleanupInterval{60};
int g_tcpRecvTimeout{2};
int g_tcpSendTimeout{2};
auto mplexer = std::shared_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent());
auto states = g_dstates.getLocal(); // this points to the actual shared_ptrs!
for(auto& dss : *states) {
- if(++dss->lastCheck < dss->checkInterval) {
+ if (++dss->lastCheck < dss->checkInterval) {
continue;
}
size_t tempFailTTL = 60;
size_t maxNegativeTTL = 3600;
size_t staleTTL = 60;
- size_t numberOfShards = 1;
+ size_t numberOfShards = 20;
bool dontAge = false;
bool deferrableInsertLock = true;
bool ecsParsing = false;
class TCPConnectionToBackend
{
public:
- TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, const struct timeval& now): d_responseBuffer(s_maxPacketCacheEntrySize), d_ds(ds), d_connectionStartTime(now), d_enableFastOpen(ds->tcpFastOpen)
+ TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, const struct timeval& now): d_responseBuffer(s_maxPacketCacheEntrySize), d_ds(ds), d_connectionStartTime(now), d_lastDataReceivedTime(now), d_enableFastOpen(ds->tcpFastOpen)
{
reconnect();
}
Before 1.4.0, a TCP thread could only handle a single incoming connection at a time. Starting with 1.4.0 the handling of TCP connections is now event-based, so a single TCP worker can handle a large number of TCP incoming connections simultaneously.
Note that before 1.6.0 the TCP worker threads were created at runtime, adding a new thread when the existing ones seemed to struggle with the load, until the maximum number of threads had been reached. Starting with 1.6.0 the configured number of worker threads are immediately created at startup.
-The maximum number of queued connections can be configured with :func:`setMaxTCPQueuedConnections` and defaults to 1000. Note that the size of the internal pipe used to distribute queries might need to be increased as well, using :func:`setTCPInternalPipeBufferSize`.
+The maximum number of queued connections can be configured with :func:`setMaxTCPQueuedConnections` and defaults to 1000 (10000 on Linux since 1.6.0). Note that the size of the internal pipe used to distribute queries might need to be increased as well, using :func:`setTCPInternalPipeBufferSize`.
Any value larger than 0 will cause new connections to be dropped if there are already too many queued.
By default, every TCP worker thread has its own queue, and the incoming TCP connections are dispatched to TCP workers on a round-robin basis.
This might cause issues if some connections are taking a very long time, since incoming ones will be waiting until the TCP worker they have been assigned to has finished handling its current query, while other TCP workers might be available.
Adding more threads makes it possible to use more CPU cores to deal with the load, but at the cost of possibly increasing lock contention between threads. This is especially true for Lua-intensive setups, because Lua processing in dnsdist is serialized by a unique lock for all threads.
For other components, like the packet cache and the in-memory ring buffers, it is possible to reduce the amount of contention by using sharding. Sharding divides the memory into several pieces, every one of these having its own separate lock, reducing the amount of times two threads or more will need to access the same data.
-Sharding is disabled by default and can be enabled via the `numberOfShards` option to :func:`newPacketCache` and :func:`setRingBuffersSize`.
+Sharding was disabled by default before 1.6.0 and could be enabled via the `numberOfShards` option to :func:`newPacketCache` and :func:`setRingBuffersSize`. It might still make sense to increment the number of shards when dealing with a lot of threads.
.. versionchanged:: 1.6.0
``exactPathMatching`` and ``releaseBuffers`` options added.
+ ``internalPipeBufferSize`` now defaults to 1048576 on Linux.
Listen on the specified address and TCP port for incoming DNS over HTTPS connections, presenting the specified X.509 certificate.
If no certificate (or key) files are specified, listen for incoming DNS over HTTP connections instead.
* ``sendCacheControlHeaders``: bool - Whether to parse the response to find the lowest TTL and set a HTTP Cache-Control header accordingly. Default is true.
* ``trustForwardedForHeader``: bool - Whether to parse any existing X-Forwarded-For header in the HTTP query and use the right-most value as the client source address and port, for ACL checks, rules, logging and so on. Default is false.
* ``tcpListenQueueSize=SOMAXCONN``: int - Set the size of the listen queue. Default is ``SOMAXCONN``.
- * ``internalPipeBufferSize=0``: int - Set the size in bytes of the internal buffer of the pipes used internally to pass queries and responses between threads. Requires support for ``F_SETPIPE_SZ`` which is present in Linux since 2.6.35. The actual size might be rounded up to a multiple of a page size. 0 means that the OS default size is used.
+ * ``internalPipeBufferSize=0``: int - Set the size in bytes of the internal buffer of the pipes used internally to pass queries and responses between threads. Requires support for ``F_SETPIPE_SZ`` which is present in Linux since 2.6.35. The actual size might be rounded up to a multiple of a page size. 0 means that the OS default size is used. The default value is 0, except on Linux where it is 1048576 since 1.6.0.
* ``exactPathMatching=true``: bool - Whether to do exact path matching of the query path against the paths configured in ``urls`` (true, the default since 1.5.0) or to accepts sub-paths (false, and was the default before 1.5.0).
* ``releaseBuffers=true``: bool - Whether OpenSSL should release its I/O buffers when a connection goes idle, saving roughly 35 kB of memory per connection.
.. function:: setRingBuffersSize(num [, numberOfShards])
+ .. versionchanged:: 1.6.0
+ ``numberOfShards`` defaults to 10.
+
Set the capacity of the ringbuffers used for live traffic inspection to ``num``, and the number of shards to ``numberOfShards`` if specified.
:param int num: The maximum amount of queries to keep in the ringbuffer. Defaults to 10000
- :param int numberOfShards: the number of shards to use to limit lock contention. Defaults to 1
+ :param int numberOfShards: the number of shards to use to limit lock contention. Default is 10, used to be 1 before 1.6.0
Servers
-------
.. versionchanged:: 1.6.0
``cookieHashing`` parameter added.
+ ``numberOfShards`` now defaults to 20.
Creates a new :class:`PacketCache` with the settings specified.
* ``maxNegativeTTL=3600``: int - Cache a NXDomain or NoData answer from the backend for at most this amount of seconds, even if the TTL of the SOA record is higher.
* ``maxTTL=86400``: int - Cap the TTL for records to his number.
* ``minTTL=0``: int - Don't cache entries with a TTL lower than this.
- * ``numberOfShards=1``: int - Number of shards to divide the cache into, to reduce lock contention.
+ * ``numberOfShards=20``: int - Number of shards to divide the cache into, to reduce lock contention. Used to be 1 (no shards) before 1.6.0, and is now 20.
* ``parseECS=false``: bool - Whether any EDNS Client Subnet option present in the query should be extracted and stored to be able to detect hash collisions involving queries with the same qname, qtype and qclass but a different incoming ECS value. Enabling this option adds a parsing cost and only makes sense if at least one backend might send different responses based on the ECS value, so it's disabled by default. Enabling this option is required for the 'zero scope' option to work
* ``staleTTL=60``: int - When the backend servers are not reachable, and global configuration ``setStaleCacheEntriesTTL`` is set appropriately, TTL that will be used when a stale cache entry is returned.
* ``temporaryFailureTTL=60``: int - On a SERVFAIL or REFUSED from the backend, cache for this amount of seconds..
.. function:: setMaxTCPQueuedConnections(num)
- Set the maximum number of TCP connections queued (waiting to be picked up by a client thread), defaults to 1000. 0 means unlimited
+ .. versionchanged:: 1.6.0
+ Before 1.6.0 the default value was 1000 on all systems.
+
+ Set the maximum number of TCP connections queued (waiting to be picked up by a client thread), defaults to 1000 (10000 on Linux since 1.6.0). 0 means unlimited
:param int num:
.. versionadded:: 1.6.0
- Set the size in bytes of the internal buffer of the pipes used internally to distribute connections to TCP (and DoT) workers threads. Requires support for ``F_SETPIPE_SZ`` which is present in Linux since 2.6.35. The actual size might be rounded up to a multiple of a page size. 0 means that the OS default size is used.
+ Set the size in bytes of the internal buffer of the pipes used internally to distribute connections to TCP (and DoT) workers threads. Requires support for ``F_SETPIPE_SZ`` which is present in Linux since 2.6.35. The actual size might be rounded up to a multiple of a page size. 0 means that the OS default size is used. The default value is 0, except on Linux where it is 1048576 since 1.6.0.
:param int size: The size in bytes.
HTTPVersionStats d_http1Stats;
HTTPVersionStats d_http2Stats;
+#ifdef __linux__
+ // On Linux this gives us 128k pending queries (default is 8192 queries),
+ // which should be enough to deal with huge spikes
+ uint32_t d_internalPipeBufferSize{1024*1024};
+#else
uint32_t d_internalPipeBufferSize{0};
+#endif
bool d_sendCacheControlHeaders{true};
bool d_trustForwardedForHeader{false};
/* whether we require tue query path to exactly match one of configured ones,
projects / thirdparty / pdns.git / commitdiff
