From: Remi Gacogne Date: Fri, 7 Oct 2022 15:43:21 +0000 (+0200) Subject: dnsdist: Implement exponential back-off for the 'lazy' mode X-Git-Tag: dnsdist-1.8.0-rc1~271^2~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=93b395a5dd9c18b124df4c135493627262d3bd92;p=thirdparty%2Fpdns.git dnsdist: Implement exponential back-off for the 'lazy' mode --- diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index 873cf704a2..cb6f5b2edd 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -550,6 +550,16 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) config.d_lazyHealthChecksFailedInterval = value; } + if (vars.count("lazyHealthCheckUseExponentialBackOff")) { + config.d_lazyHealthChecksUseExponentialBackOff = boost::get(vars.at("lazyHealthCheckUseExponentialBackOff")); + } + + if (vars.count("lazyHealthCheckMaxBackOff")) { + auto value = std::stoi(boost::get(vars.at("lazyHealthCheckMaxBackOff"))); + checkParameterBound("lazyHealthCheckMaxBackOff", value); + config.d_lazyHealthChecksMaxBackOff = value; + } + if (vars.count("lazyHealthCheckMode")) { auto mode = boost::get(vars.at("lazyHealthCheckMode")); if (pdns_iequals(mode, "TimeoutOnly")) { diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc index f9d990e7a6..d0b3fbec93 100644 --- a/pdns/dnsdist.cc +++ b/pdns/dnsdist.cc @@ -2762,8 +2762,11 @@ int main(int argc, char** argv) auto states = g_dstates.getCopy(); // it is a copy, but the internal shared_ptrs are the real deal auto mplexer = std::unique_ptr(FDMultiplexer::getMultiplexerSilent(states.size())); for (auto& dss : states) { - if (dss->d_config.availability == DownstreamState::Availability::Auto) { - dss->d_nextCheck = dss->d_config.checkInterval; + if (dss->d_config.availability == DownstreamState::Availability::Auto || dss->d_config.availability == DownstreamState::Availability::Lazy) { + if (dss->d_config.availability == DownstreamState::Availability::Auto) { + dss->d_nextCheck = dss->d_config.checkInterval; + } + if (!queueHealthCheck(mplexer, dss, true)) { dss->setUpStatus(false); warnlog("Marking downstream %s as 'down'", dss->getNameWithAddr()); diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 91a50e1776..4f0e3c66de 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -810,6 +810,7 @@ struct DownstreamState: public std::enable_shared_from_this uint16_t d_lazyHealthChecksSampleSize{100}; uint16_t d_lazyHealthChecksMinSampleCount{1}; uint16_t d_lazyHealthChecksFailedInterval{30}; + uint16_t d_lazyHealthChecksMaxBackOff{3600}; uint8_t d_lazyHealthChecksThreshold{20}; LazyHealthCheckMode d_lazyHealthChecksMode{LazyHealthCheckMode::TimeoutOrServFail}; uint8_t maxCheckFailures{1}; @@ -827,6 +828,7 @@ struct DownstreamState: public std::enable_shared_from_this bool d_tcpCheck{false}; bool d_tcpOnly{false}; bool d_addXForwardedHeaders{false}; // for DoH backends + bool d_lazyHealthChecksUseExponentialBackOff{false}; }; DownstreamState(DownstreamState::Config&& config, std::shared_ptr tlsCtx, bool connect); @@ -894,7 +896,7 @@ public: double latencyUsec{0.0}; double latencyUsecTCP{0.0}; unsigned int d_nextCheck{0}; - uint8_t currentCheckFailures{0}; + uint16_t currentCheckFailures{0}; uint8_t consecutiveSuccessfulChecks{0}; std::atomic hashesComputed{false}; std::atomic connected{false}; @@ -1055,6 +1057,7 @@ public: static bool s_randomizeIDs; private: void handleUDPTimeout(IDState& ids); + void updateNextLazyHealthCheck(LazyHealthCheckStats& stats); }; using servers_t = vector>; diff --git a/pdns/dnsdistdist/dnsdist-backend.cc b/pdns/dnsdistdist/dnsdist-backend.cc index eb2f940646..a93d36d01d 100644 --- a/pdns/dnsdistdist/dnsdist-backend.cc +++ b/pdns/dnsdistdist/dnsdist-backend.cc @@ -510,7 +510,10 @@ bool DownstreamState::healthCheckRequired() if (stats->d_status == LazyHealthCheckStats::LazyStatus::Failed) { auto now = time(nullptr); if (stats->d_nextCheck <= now) { - stats->d_nextCheck = now + d_config.d_lazyHealthChecksFailedInterval; + /* we update the next check time here because the check might time out, + and we do not want to send a second check during that time unless + the timer is actually very short */ + updateNextLazyHealthCheck(*stats); vinfolog("Sending health-check query for %s which is still in the Failed state", getNameWithAddr()); return true; } @@ -536,8 +539,10 @@ bool DownstreamState::healthCheckRequired() lastResults.clear(); vinfolog("Backend %s reached the lazy health-check threshold (%f out of %f, looking at sample of %d items with %d failures), moving to Potential Failure state", getNameWithAddr(), current, maxFailureRate, totalCount, failures); stats->d_status = LazyHealthCheckStats::LazyStatus::PotentialFailure; - auto now = time(nullptr); - stats->d_nextCheck = now; + /* we update the next check time here because the check might time out, + and we do not want to send a second check during that time unless + the timer is actually very short */ + updateNextLazyHealthCheck(*stats); return true; } } @@ -564,19 +569,55 @@ time_t DownstreamState::getNextLazyHealthCheck() return stats->d_nextCheck; } -void DownstreamState::submitHealthCheckResult(bool initial, bool newState) +void DownstreamState::updateNextLazyHealthCheck(LazyHealthCheckStats& stats) +{ + auto now = time(nullptr); + if (d_config.d_lazyHealthChecksUseExponentialBackOff) { + if (stats.d_status == DownstreamState::LazyHealthCheckStats::LazyStatus::PotentialFailure) { + /* we are still in the "up" state, we need to send the next query quickly to + determine if the backend is really down */ + stats.d_nextCheck = now + d_config.d_lazyHealthChecksFailedInterval; + } + else if (consecutiveSuccessfulChecks > 0) { + /* we are in 'Failed' state, but just had one (or more) successful check, + so we want the next one to happen quite quickly as the backend might + be available again. */ + stats.d_nextCheck = now + d_config.d_lazyHealthChecksFailedInterval; + } + else { + const uint16_t failedTests = currentCheckFailures; + size_t backOffCoeff = std::pow(2U, failedTests); + time_t backOff = d_config.d_lazyHealthChecksMaxBackOff; + if ((std::numeric_limits::max() / d_config.d_lazyHealthChecksFailedInterval) >= backOffCoeff) { + backOff = d_config.d_lazyHealthChecksFailedInterval * backOffCoeff; + if (backOff > d_config.d_lazyHealthChecksMaxBackOff || (std::numeric_limits::max() - now) <= backOff) { + backOff = d_config.d_lazyHealthChecksMaxBackOff; + } + } + + stats.d_nextCheck = now + backOff; + } + } + else { + stats.d_nextCheck = now + d_config.d_lazyHealthChecksFailedInterval; + } +} + +void DownstreamState::submitHealthCheckResult(bool initial, bool newResult) { if (initial) { /* if this is the initial health-check, at startup, we do not care about the minimum number of failed/successful health-checks */ if (!IsAnyAddress(d_config.remote)) { - infolog("Marking downstream %s as '%s'", getNameWithAddr(), newState ? "up" : "down"); + infolog("Marking downstream %s as '%s'", getNameWithAddr(), newResult ? "up" : "down"); } - setUpStatus(newState); + setUpStatus(newResult); return; } - if (newState) { + bool newState = newResult; + + if (newResult) { /* check succeeded */ currentCheckFailures = 0; @@ -589,6 +630,11 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newState) /* we need more than one successful check to rise and we didn't reach the threshold yet, let's stay down */ newState = false; + + if (d_config.availability == DownstreamState::Availability::Lazy) { + auto stats = d_lazyHealthCheckStats.lock(); + updateNextLazyHealthCheck(*stats); + } } } @@ -596,6 +642,7 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newState) if (d_config.availability == DownstreamState::Availability::Lazy) { auto stats = d_lazyHealthCheckStats.lock(); stats->d_status = LazyHealthCheckStats::LazyStatus::Healthy; + stats->d_lastResults.clear(); } } } @@ -603,11 +650,12 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newState) /* check failed */ consecutiveSuccessfulChecks = 0; + currentCheckFailures++; + if (upStatus) { /* we were previously marked as "up" and failed a health-check, let's see if this is enough to move to the "down" state or if need more failed checks for that */ - currentCheckFailures++; if (currentCheckFailures < d_config.maxCheckFailures) { /* we need more than one failure to be marked as down, and we did not reach the threshold yet, let's stay up */ @@ -616,8 +664,8 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newState) else if (d_config.availability == DownstreamState::Availability::Lazy) { auto stats = d_lazyHealthCheckStats.lock(); stats->d_status = LazyHealthCheckStats::LazyStatus::Failed; - auto now = time(nullptr); - stats->d_nextCheck = now + d_config.d_lazyHealthChecksFailedInterval; + currentCheckFailures = 0; + updateNextLazyHealthCheck(*stats); } } } @@ -634,8 +682,6 @@ void DownstreamState::submitHealthCheckResult(bool initial, bool newState) } setUpStatus(newState); - currentCheckFailures = 0; - consecutiveSuccessfulChecks = 0; if (g_snmpAgent && g_snmpTrapsEnabled) { g_snmpAgent->sendBackendStatusChangeTrap(*this); } diff --git a/pdns/dnsdistdist/dnsdist-nghttp2.cc b/pdns/dnsdistdist/dnsdist-nghttp2.cc index 87065b9a77..361fc087c5 100644 --- a/pdns/dnsdistdist/dnsdist-nghttp2.cc +++ b/pdns/dnsdistdist/dnsdist-nghttp2.cc @@ -158,7 +158,9 @@ void DoHConnectionToBackend::handleResponse(PendingRequest&& request) void DoHConnectionToBackend::handleResponseError(PendingRequest&& request, const struct timeval& now) { try { - d_ds->reportTimeoutOrError(); + if (!d_healthCheckQuery) { + d_ds->reportTimeoutOrError(); + } request.d_sender->notifyIOError(std::move(request.d_query.d_idstate), now); } diff --git a/pdns/dnsdistdist/docs/guides/downstreams.rst b/pdns/dnsdistdist/docs/guides/downstreams.rst index 8f1323402c..ac720d1d6a 100644 --- a/pdns/dnsdistdist/docs/guides/downstreams.rst +++ b/pdns/dnsdistdist/docs/guides/downstreams.rst @@ -86,6 +86,8 @@ So for example, if we set ``healthCheckMode`` to ``lazy``, ``lazyHealthCheckSamp newServer({address="192.0.2.1", healthCheckMode='lazy', checkInterval=1, lazyHealthCheckFailedInterval=30, rise=2, maxCheckFailures=3, lazyHealthCheckThreshold=30, lazyHealthCheckSampleSize=100, lazyHealthCheckMinSampleCount=10, lazyHealthCheckMode='TimeoutOnly'}) +The 'lazy' mode also supports using an exponential back-off time between health-check queries, once a backend has been moved to the 'down' state. This can be enabled by setting the ``lazyHealthCheckUseExponentialBackOff`` parameter to 'true'. Once the backend has been marked as 'down', the first query will be sent after ``lazyHealthCheckFailedInterval`` seconds, the second one after 2 times ``lazyHealthCheckFailedInterval`` seconds, the third after 4 times ``lazyHealthCheckFailedInterval`` seconds, and so on and so forth, until ``lazyHealthCheckMaxBackOff`` has been reached. Then probes will be sent every ``lazyHealthCheckMaxBackOff`` seconds (default is 3600 so one hour) until the backend comes 'up' again. + Source address selection ------------------------ diff --git a/pdns/dnsdistdist/docs/reference/config.rst b/pdns/dnsdistdist/docs/reference/config.rst index e0441e690d..9b82ce55d9 100644 --- a/pdns/dnsdistdist/docs/reference/config.rst +++ b/pdns/dnsdistdist/docs/reference/config.rst @@ -561,7 +561,7 @@ Servers Added ``addXForwardedHeaders``, ``caStore``, ``checkTCP``, ``ciphers``, ``ciphers13``, ``dohPath``, ``enableRenegotiation``, ``releaseBuffers``, ``subjectName``, ``tcpOnly``, ``tls`` and ``validateCertificates`` to server_table. .. versionchanged:: 1.8.0 - Added ``autoUpgrade``, ``autoUpgradeDoHKey``, ``autoUpgradeInterval``, ``autoUpgradeKeep``, ``autoUpgradePool``, ``maxConcurrentTCPConnections``, ``subjectAddr``, ``lazyHealthCheckSampleSize``, ``lazyHealthCheckMinSampleCount``, ``lazyHealthCheckThreshold``, ``lazyHealthCheckFailedInterval``, ``lazyHealthCheckMode`` and ``healthCheckMode`` to server_table. + Added ``autoUpgrade``, ``autoUpgradeDoHKey``, ``autoUpgradeInterval``, ``autoUpgradeKeep``, ``autoUpgradePool``, ``maxConcurrentTCPConnections``, ``subjectAddr``, ``lazyHealthCheckSampleSize``, ``lazyHealthCheckMinSampleCount``, ``lazyHealthCheckThreshold``, ``lazyHealthCheckFailedInterval``, ``lazyHealthCheckMode``, ``lazyHealthCheckUseExponentialBackOff``, ``lazyHealthCheckMaxBackOff`` and ``healthCheckMode`` to server_table. Add a new backend server. Call this function with either a string:: @@ -572,68 +572,70 @@ Servers or a table:: newServer({ - address="IP:PORT", -- IP and PORT of the backend server (mandatory) - id=STRING, -- Use a pre-defined UUID instead of a random one - qps=NUM, -- Limit the number of queries per second to NUM, when using the `firstAvailable` policy - order=NUM, -- The order of this server, used by the `leastOutstanding` and `firstAvailable` policies - weight=NUM, -- The weight of this server, used by the `wrandom`, `whashed` and `chashed` policies, default: 1 - -- Supported values are a minimum of 1, and a maximum of 2147483647. - pool=STRING|{STRING}, -- The pools this server belongs to (unset or empty string means default pool) as a string or table of strings - retries=NUM, -- The number of TCP connection attempts to the backend, for a given query - tcpConnectTimeout=NUM, -- The timeout (in seconds) of a TCP connection attempt - tcpSendTimeout=NUM, -- The timeout (in seconds) of a TCP write attempt - tcpRecvTimeout=NUM, -- The timeout (in seconds) of a TCP read attempt - tcpFastOpen=BOOL, -- Whether to enable TCP Fast Open - ipBindAddrNoPort=BOOL, -- Whether to enable IP_BIND_ADDRESS_NO_PORT if available, default: true - name=STRING, -- The name associated to this backend, for display purpose - checkClass=NUM, -- Use NUM as QCLASS in the health-check query, default: DNSClass.IN - checkName=STRING, -- Use STRING as QNAME in the health-check query, default: "a.root-servers.net." - checkType=STRING, -- Use STRING as QTYPE in the health-check query, default: "A" - checkFunction=FUNCTION, -- Use this function to dynamically set the QNAME, QTYPE and QCLASS to use in the health-check query (see :ref:`Healthcheck`) - checkTimeout=NUM, -- The timeout (in milliseconds) of a health-check query, default: 1000 (1s) - setCD=BOOL, -- Set the CD (Checking Disabled) flag in the health-check query, default: false - maxCheckFailures=NUM, -- Allow NUM check failures before declaring the backend down, default: 1 - checkInterval=NUM -- The time in seconds between health checks - mustResolve=BOOL, -- Set to true when the health check MUST return a RCODE different from NXDomain, ServFail and Refused. Default is false, meaning that every RCODE except ServFail is considered valid - useClientSubnet=BOOL, -- Add the client's IP address in the EDNS Client Subnet option when forwarding the query to this backend - source=STRING, -- The source address or interface to use for queries to this backend, by default this is left to the kernel's address selection - -- The following formats are supported: - -- "address", e.g. "192.0.2.2" - -- "interface name", e.g. "eth0" - -- "address@interface", e.g. "192.0.2.2@eth0" - addXPF=NUM, -- Add the client's IP address and port to the query, along with the original destination address and port, - -- using the experimental XPF record from `draft-bellis-dnsop-xpf `_ and the specified option code. Default is disabled (0). This is a deprecated feature that will be removed in the near future. - sockets=NUM, -- Number of UDP sockets (and thus source ports) used toward the backend server, defaults to a single one. Note that for backends which are multithreaded, this setting will have an effect on the number of cores that will be used to process traffic from dnsdist. For example you may want to set 'sockets' to a number somewhat higher than the number of worker threads configured in the backend, particularly if the Linux kernel is being used to distribute traffic to multiple threads listening on the same socket (via `reuseport`). - disableZeroScope=BOOL, -- Disable the EDNS Client Subnet 'zero scope' feature, which does a cache lookup for an answer valid for all subnets (ECS scope of 0) before adding ECS information to the query and doing the regular lookup. This requires the ``parseECS`` option of the corresponding cache to be set to true - rise=NUM, -- Require NUM consecutive successful checks before declaring the backend up, default: 1 - useProxyProtocol=BOOL, -- Add a proxy protocol header to the query, passing along the client's IP address and port along with the original destination address and port. Default is disabled. - reconnectOnUp=BOOL, -- Close and reopen the sockets when a server transits from Down to Up. This helps when an interface is missing when dnsdist is started. Default is disabled. - maxInFlight=NUM, -- Maximum number of in-flight queries. The default is 0, which disables out-of-order processing. It should only be enabled if the backend does support out-of-order processing. As of 1.6.0, out-of-order processing needs to be enabled on the frontend as well, via :func:`addLocal` and/or :func:`addTLSLocal`. Note that out-of-order is always enabled on DoH frontends. - tcpOnly=BOOL, -- Always forward queries to that backend over TCP, never over UDP. Always enabled for TLS backends. Default is false. - checkTCP=BOOL, -- Whether to do healthcheck queries over TCP, instead of UDP. Always enabled for DNS over TLS backend. Default is false. - tls=STRING, -- Enable DNS over TLS communications for this backend, or DNS over HTTPS if ``dohPath`` is set, using the TLS provider ("openssl" or "gnutls") passed in parameter. Default is an empty string, which means this backend is used for plain UDP and TCP. - caStore=STRING, -- Specifies the path to the CA certificate file, in PEM format, to use to check the certificate presented by the backend. Default is an empty string, which means to use the system CA store. Note that this directive is only used if ``validateCertificates`` is set. - ciphers=STRING, -- The TLS ciphers to use. The exact format depends on the provider used. When the OpenSSL provider is used, ciphers for TLS 1.3 must be specified via ``ciphersTLS13``. - ciphersTLS13=STRING, -- The ciphers to use for TLS 1.3, when the OpenSSL provider is used. When the GnuTLS provider is used, ``ciphers`` applies regardless of the TLS protocol and this setting is not used. - subjectName=STRING, -- The subject name passed in the SNI value of the TLS handshake, and against which to validate the certificate presented by the backend. Default is empty. If set this value supersedes any ``subjectAddr`` one. - subjectAddr=STRING, -- The subject IP address passed in the SNI value of the TLS handshake, and against which to validate the certificate presented by the backend. Default is empty. - validateCertificates=BOOL, -- Whether the certificate presented by the backend should be validated against the CA store (see ``caStore``). Default is true. - dohPath=STRING, -- Enable DNS over HTTPS communication for this backend, using POST queries to the HTTP host supplied as ``subjectName`` and the HTTP path supplied in this parameter. - addXForwardedHeaders=BOOL, -- Whether to add X-Forwarded-For, X-Forwarded-Port and X-Forwarded-Proto headers to a DNS over HTTPS backend. - releaseBuffers=BOOL, -- Whether OpenSSL should release its I/O buffers when a connection goes idle, saving roughly 35 kB of memory per connection. Default to true. - enableRenegotiation=BOOL, -- Whether secure TLS renegotiation should be enabled. Disabled by default since it increases the attack surface and is seldom used for DNS. - autoUpgrade=BOOL, -- Whether to use the 'Discovery of Designated Resolvers' mechanism to automatically upgrade a Do53 backend to DoT or DoH, depending on the priorities present in the SVCB record returned by the backend. Default to false. - autoUpgradeInterval=NUM, -- If ``autoUpgrade`` is set, how often to check if an upgrade is available, in seconds. Default is 3600 seconds. - autoUpgradeKeep=BOOL, -- If ``autoUpgrade`` is set, whether to keep the existing Do53 backend around after an upgrade. Default is false which means the Do53 backend will be replaced by the upgraded one. - autoUpgradePool=STRING, -- If ``autoUpgrade`` is set, in which pool to place the newly upgraded backend. Default is empty which means the backend is placed in the default pool. - autoUpgradeDoHKey=NUM, -- If ``autoUpgrade`` is set, the value to use for the SVC key corresponding to the DoH path. Default is 7. - maxConcurrentTCPConnections=NUM, -- Maximum number of TCP connections to that backend. When that limit is reached, queries routed to that backend that cannot be forwarded over an existing connection will be dropped. Default is 0 which means no limit. - healthCheckMode=STRING -- The health-check mode to use: 'auto' which sends health-check queries every ``checkInterval`` seconds, 'up' which considers that the backend is always available, 'down' that it is always not available, and 'lazy' which only sends health-check queries after a configurable amount of regular queries have failed (see ``lazyHealthCheckSampleSize``, ``lazyHealthCheckMinSampleCount``, ``lazyHealthCheckThreshold``, ``lazyHealthCheckFailedInterval`` and ``lazyHealthCheckMode`` for more information). Default is 'auto'. See :ref:`Healthcheck` for a more detailed explanation. - lazyHealthCheckFailedInterval=NUM, -- The interval, in seconds, between health-check queries in 'lazy' mode. These queries are only sent when a threshold of failing regular queries has been reached, and only ``maxCheckFailures`` of them are sent. Default is 30 seconds. - lazyHealthCheckMinSampleCount=NUM, -- The minimum amount of regular queries that should have been recorded before the ``lazyHealthCheckThreshold`` threshold can be applied. Default is 1 which means only one query is needed. - lazyHealthCheckMode=STRING, -- The 'lazy' health-check mode: 'TimeoutOnly' means that only timeout and I/O errors of regular queries will be considered for the ``lazyHealthCheckThreshold``, while 'TimeoutOrServFail' will also consider 'Server Failure' answers. Default is 'TimeoutOrServFail'. - lazyHealthCheckSampleSize=NUM, -- The maximum size of the sample of queries to record and consider for the ``lazyHealthCheckThreshold``. Default is 100, which means the result (failure or success) of the last 100 queries will be considered. - lazyHealthCheckThreshold=NUM -- The threshold, as a percentage, of queries that should fail for the 'lazy' health-check to be triggered when ``healthCheckMode`` is set to ``lazy``. The default is 20 which means 20% of the last ``lazyHealthCheckSampleSize`` queries should fail for a health-check to be triggered. + address="IP:PORT", -- IP and PORT of the backend server (mandatory) + id=STRING, -- Use a pre-defined UUID instead of a random one + qps=NUM, -- Limit the number of queries per second to NUM, when using the `firstAvailable` policy + order=NUM, -- The order of this server, used by the `leastOutstanding` and `firstAvailable` policies + weight=NUM, -- The weight of this server, used by the `wrandom`, `whashed` and `chashed` policies, default: 1 + -- Supported values are a minimum of 1, and a maximum of 2147483647. + pool=STRING|{STRING}, -- The pools this server belongs to (unset or empty string means default pool) as a string or table of strings + retries=NUM, -- The number of TCP connection attempts to the backend, for a given query + tcpConnectTimeout=NUM, -- The timeout (in seconds) of a TCP connection attempt + tcpSendTimeout=NUM, -- The timeout (in seconds) of a TCP write attempt + tcpRecvTimeout=NUM, -- The timeout (in seconds) of a TCP read attempt + tcpFastOpen=BOOL, -- Whether to enable TCP Fast Open + ipBindAddrNoPort=BOOL, -- Whether to enable IP_BIND_ADDRESS_NO_PORT if available, default: true + name=STRING, -- The name associated to this backend, for display purpose + checkClass=NUM, -- Use NUM as QCLASS in the health-check query, default: DNSClass.IN + checkName=STRING, -- Use STRING as QNAME in the health-check query, default: "a.root-servers.net." + checkType=STRING, -- Use STRING as QTYPE in the health-check query, default: "A" + checkFunction=FUNCTION, -- Use this function to dynamically set the QNAME, QTYPE and QCLASS to use in the health-check query (see :ref:`Healthcheck`) + checkTimeout=NUM, -- The timeout (in milliseconds) of a health-check query, default: 1000 (1s) + setCD=BOOL, -- Set the CD (Checking Disabled) flag in the health-check query, default: false + maxCheckFailures=NUM, -- Allow NUM check failures before declaring the backend down, default: 1 + checkInterval=NUM -- The time in seconds between health checks + mustResolve=BOOL, -- Set to true when the health check MUST return a RCODE different from NXDomain, ServFail and Refused. Default is false, meaning that every RCODE except ServFail is considered valid + useClientSubnet=BOOL, -- Add the client's IP address in the EDNS Client Subnet option when forwarding the query to this backend + source=STRING, -- The source address or interface to use for queries to this backend, by default this is left to the kernel's address selection + -- The following formats are supported: + -- "address", e.g. "192.0.2.2" + -- "interface name", e.g. "eth0" + -- "address@interface", e.g. "192.0.2.2@eth0" + addXPF=NUM, -- Add the client's IP address and port to the query, along with the original destination address and port, + -- using the experimental XPF record from `draft-bellis-dnsop-xpf `_ and the specified option code. Default is disabled (0). This is a deprecated feature that will be removed in the near future. + sockets=NUM, -- Number of UDP sockets (and thus source ports) used toward the backend server, defaults to a single one. Note that for backends which are multithreaded, this setting will have an effect on the number of cores that will be used to process traffic from dnsdist. For example you may want to set 'sockets' to a number somewhat higher than the number of worker threads configured in the backend, particularly if the Linux kernel is being used to distribute traffic to multiple threads listening on the same socket (via `reuseport`). + disableZeroScope=BOOL, -- Disable the EDNS Client Subnet 'zero scope' feature, which does a cache lookup for an answer valid for all subnets (ECS scope of 0) before adding ECS information to the query and doing the regular lookup. This requires the ``parseECS`` option of the corresponding cache to be set to true + rise=NUM, -- Require NUM consecutive successful checks before declaring the backend up, default: 1 + useProxyProtocol=BOOL, -- Add a proxy protocol header to the query, passing along the client's IP address and port along with the original destination address and port. Default is disabled. + reconnectOnUp=BOOL, -- Close and reopen the sockets when a server transits from Down to Up. This helps when an interface is missing when dnsdist is started. Default is disabled. + maxInFlight=NUM, -- Maximum number of in-flight queries. The default is 0, which disables out-of-order processing. It should only be enabled if the backend does support out-of-order processing. As of 1.6.0, out-of-order processing needs to be enabled on the frontend as well, via :func:`addLocal` and/or :func:`addTLSLocal`. Note that out-of-order is always enabled on DoH frontends. + tcpOnly=BOOL, -- Always forward queries to that backend over TCP, never over UDP. Always enabled for TLS backends. Default is false. + checkTCP=BOOL, -- Whether to do healthcheck queries over TCP, instead of UDP. Always enabled for DNS over TLS backend. Default is false. + tls=STRING, -- Enable DNS over TLS communications for this backend, or DNS over HTTPS if ``dohPath`` is set, using the TLS provider ("openssl" or "gnutls") passed in parameter. Default is an empty string, which means this backend is used for plain UDP and TCP. + caStore=STRING, -- Specifies the path to the CA certificate file, in PEM format, to use to check the certificate presented by the backend. Default is an empty string, which means to use the system CA store. Note that this directive is only used if ``validateCertificates`` is set. + ciphers=STRING, -- The TLS ciphers to use. The exact format depends on the provider used. When the OpenSSL provider is used, ciphers for TLS 1.3 must be specified via ``ciphersTLS13``. + ciphersTLS13=STRING, -- The ciphers to use for TLS 1.3, when the OpenSSL provider is used. When the GnuTLS provider is used, ``ciphers`` applies regardless of the TLS protocol and this setting is not used. + subjectName=STRING, -- The subject name passed in the SNI value of the TLS handshake, and against which to validate the certificate presented by the backend. Default is empty. If set this value supersedes any ``subjectAddr`` one. + subjectAddr=STRING, -- The subject IP address passed in the SNI value of the TLS handshake, and against which to validate the certificate presented by the backend. Default is empty. + validateCertificates=BOOL, -- Whether the certificate presented by the backend should be validated against the CA store (see ``caStore``). Default is true. + dohPath=STRING, -- Enable DNS over HTTPS communication for this backend, using POST queries to the HTTP host supplied as ``subjectName`` and the HTTP path supplied in this parameter. + addXForwardedHeaders=BOOL, -- Whether to add X-Forwarded-For, X-Forwarded-Port and X-Forwarded-Proto headers to a DNS over HTTPS backend. + releaseBuffers=BOOL, -- Whether OpenSSL should release its I/O buffers when a connection goes idle, saving roughly 35 kB of memory per connection. Default to true. + enableRenegotiation=BOOL, -- Whether secure TLS renegotiation should be enabled. Disabled by default since it increases the attack surface and is seldom used for DNS. + autoUpgrade=BOOL, -- Whether to use the 'Discovery of Designated Resolvers' mechanism to automatically upgrade a Do53 backend to DoT or DoH, depending on the priorities present in the SVCB record returned by the backend. Default to false. + autoUpgradeInterval=NUM, -- If ``autoUpgrade`` is set, how often to check if an upgrade is available, in seconds. Default is 3600 seconds. + autoUpgradeKeep=BOOL, -- If ``autoUpgrade`` is set, whether to keep the existing Do53 backend around after an upgrade. Default is false which means the Do53 backend will be replaced by the upgraded one. + autoUpgradePool=STRING, -- If ``autoUpgrade`` is set, in which pool to place the newly upgraded backend. Default is empty which means the backend is placed in the default pool. + autoUpgradeDoHKey=NUM, -- If ``autoUpgrade`` is set, the value to use for the SVC key corresponding to the DoH path. Default is 7. + maxConcurrentTCPConnections=NUM, -- Maximum number of TCP connections to that backend. When that limit is reached, queries routed to that backend that cannot be forwarded over an existing connection will be dropped. Default is 0 which means no limit. + healthCheckMode=STRING -- The health-check mode to use: 'auto' which sends health-check queries every ``checkInterval`` seconds, 'up' which considers that the backend is always available, 'down' that it is always not available, and 'lazy' which only sends health-check queries after a configurable amount of regular queries have failed (see ``lazyHealthCheckSampleSize``, ``lazyHealthCheckMinSampleCount``, ``lazyHealthCheckThreshold``, ``lazyHealthCheckFailedInterval`` and ``lazyHealthCheckMode`` for more information). Default is 'auto'. See :ref:`Healthcheck` for a more detailed explanation. + lazyHealthCheckFailedInterval=NUM, -- The interval, in seconds, between health-check queries in 'lazy' mode. Note that when ``lazyHealthCheckUseExponentialBackOff`` is set to true, the interval doubles between every queries. These queries are only sent when a threshold of failing regular queries has been reached, and until the backend is available again. Default is 30 seconds. + lazyHealthCheckMinSampleCount=NUM, -- The minimum amount of regular queries that should have been recorded before the ``lazyHealthCheckThreshold`` threshold can be applied. Default is 1 which means only one query is needed. + lazyHealthCheckMode=STRING, -- The 'lazy' health-check mode: 'TimeoutOnly' means that only timeout and I/O errors of regular queries will be considered for the ``lazyHealthCheckThreshold``, while 'TimeoutOrServFail' will also consider 'Server Failure' answers. Default is 'TimeoutOrServFail'. + lazyHealthCheckSampleSize=NUM, -- The maximum size of the sample of queries to record and consider for the ``lazyHealthCheckThreshold``. Default is 100, which means the result (failure or success) of the last 100 queries will be considered. + lazyHealthCheckThreshold=NUM, -- The threshold, as a percentage, of queries that should fail for the 'lazy' health-check to be triggered when ``healthCheckMode`` is set to ``lazy``. The default is 20 which means 20% of the last ``lazyHealthCheckSampleSize`` queries should fail for a health-check to be triggered. + lazyHealthCheckUseExponentialBackOff=BOOL, -- Whether the 'lazy' health-check should use an exponential back-off instead of a fixed value, between health-check probes. The default is false which means that after a backend has been moved to the 'down' state health-check probes are sent every ``lazyHealthCheckFailedInterval`` seconds. When set to true, the delay between each probe starts at ``lazyHealthCheckFailedInterval`` seconds and double between every probe, capped at ``lazyHealthCheckMaxBackOff`` seconds. + lazyHealthCheckMaxBackOff=NUM -- This value, in seconds, caps the time between two health-check queries when ``lazyHealthCheckUseExponentialBackOff`` is set to true. The default is 3600 which means that at most one hour will pass between two health-check queries. }) :param str server_string: A simple IP:PORT string. diff --git a/pdns/dnsdistdist/test-dnsdistbackend_cc.cc b/pdns/dnsdistdist/test-dnsdistbackend_cc.cc index e063ec2b6d..3e56c918a3 100644 --- a/pdns/dnsdistdist/test-dnsdistbackend_cc.cc +++ b/pdns/dnsdistdist/test-dnsdistbackend_cc.cc @@ -120,6 +120,7 @@ BOOST_AUTO_TEST_CASE(test_Lazy) config.maxCheckFailures = 3; config.d_lazyHealthChecksMinSampleCount = 11; config.d_lazyHealthChecksThreshold = 20; + config.d_lazyHealthChecksUseExponentialBackOff = false; config.availability = DownstreamState::Availability::Lazy; /* prevents a re-connection */ config.remote = ComboAddress("0.0.0.0"); @@ -198,7 +199,69 @@ BOOST_AUTO_TEST_CASE(test_Lazy) ds.reportResponse(RCode::NoError); } - /* we need minRiseSuccesses successful health-checks to go down */ + /* we need minRiseSuccesses successful health-checks to go up */ + for (size_t idx = 0; idx < static_cast(config.minRiseSuccesses - 1); idx++) { + ds.submitHealthCheckResult(false, true); + } + BOOST_CHECK_EQUAL(ds.isUp(), false); + BOOST_CHECK_EQUAL(ds.getStatus(), "down"); + + ds.submitHealthCheckResult(false, true); + BOOST_CHECK_EQUAL(ds.isUp(), true); + BOOST_CHECK_EQUAL(ds.getStatus(), "up"); + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), false); +} + +BOOST_AUTO_TEST_CASE(test_LazyExponentialBackOff) +{ + DownstreamState::Config config; + config.minRiseSuccesses = 5; + config.maxCheckFailures = 3; + config.d_lazyHealthChecksMinSampleCount = 11; + config.d_lazyHealthChecksThreshold = 20; + config.d_lazyHealthChecksUseExponentialBackOff = true; + config.d_lazyHealthChecksMaxBackOff = 60; + config.d_lazyHealthChecksFailedInterval = 30; + config.availability = DownstreamState::Availability::Lazy; + /* prevents a re-connection */ + config.remote = ComboAddress("0.0.0.0"); + + DownstreamState ds(std::move(config), nullptr, false); + BOOST_CHECK(ds.d_config.availability == DownstreamState::Availability::Lazy); + BOOST_CHECK_EQUAL(ds.isUp(), true); + BOOST_CHECK_EQUAL(ds.getStatus(), "up"); + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), false); + + /* submit a few failed results */ + for (size_t idx = 0; idx < config.d_lazyHealthChecksMinSampleCount; idx++) { + ds.reportTimeoutOrError(); + } + BOOST_CHECK_EQUAL(ds.isUp(), true); + BOOST_CHECK_EQUAL(ds.getStatus(), "up"); + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), true); + + /* we should be in Potential Failure mode now, and thus always returning true */ + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), true); + + /* we need maxCheckFailures failed health-checks to go down */ + for (size_t idx = 0; idx < static_cast(config.maxCheckFailures - 1); idx++) { + ds.submitHealthCheckResult(false, false); + } + BOOST_CHECK_EQUAL(ds.isUp(), true); + BOOST_CHECK_EQUAL(ds.getStatus(), "up"); + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), true); + time_t failedCheckTime = time(nullptr); + ds.submitHealthCheckResult(false, false); + + /* now we are in Failed state */ + BOOST_CHECK_EQUAL(ds.isUp(), false); + BOOST_CHECK_EQUAL(ds.getStatus(), "down"); + BOOST_CHECK_EQUAL(ds.healthCheckRequired(), false); + /* and the wait time between two checks will double every time a failure occurs */ + BOOST_CHECK_EQUAL(ds.getNextLazyHealthCheck(), (failedCheckTime + (config.d_lazyHealthChecksFailedInterval * std::pow(2U, ds.currentCheckFailures)))); + BOOST_CHECK_EQUAL(ds.currentCheckFailures, 0U); + + /* we need minRiseSuccesses successful health-checks to go up */ for (size_t idx = 0; idx < static_cast(config.minRiseSuccesses - 1); idx++) { ds.submitHealthCheckResult(false, true); } diff --git a/regression-tests.dnsdist/test_HealthChecks.py b/regression-tests.dnsdist/test_HealthChecks.py index 32749890bc..a42cb5ff0b 100644 --- a/regression-tests.dnsdist/test_HealthChecks.py +++ b/regression-tests.dnsdist/test_HealthChecks.py @@ -263,9 +263,10 @@ class TestLazyHealthChecks(HealthCheckTest): """ Lazy Healthchecks: Do53 """ - self.assertEqual(_do53HealthCheckQueries, 0) + # there is one initial query on startup + self.assertEqual(_do53HealthCheckQueries, 1) time.sleep(1) - self.assertEqual(_do53HealthCheckQueries, 0) + self.assertEqual(_do53HealthCheckQueries, 1) name = 'do53.lazy.test.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN') @@ -281,7 +282,7 @@ class TestLazyHealthChecks(HealthCheckTest): (_, receivedResponse) = sender(query, response=None, useQueue=False) self.assertEqual(receivedResponse, response) - self.assertEqual(_do53HealthCheckQueries, 0) + self.assertEqual(_do53HealthCheckQueries, 1) # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough for _ in range(2): @@ -289,16 +290,17 @@ class TestLazyHealthChecks(HealthCheckTest): self.assertEqual(receivedResponse, failedResponse) time.sleep(1.5) - self.assertEqual(_do53HealthCheckQueries, 1) + self.assertEqual(_do53HealthCheckQueries, 2) self.assertEqual(self.getBackendStatus(), 'up') def testDoTLazy(self): """ Lazy Healthchecks: DoT """ - self.assertEqual(_dotHealthCheckQueries, 0) + # there is one initial query on startup + self.assertEqual(_dotHealthCheckQueries, 1) time.sleep(1) - self.assertEqual(_dotHealthCheckQueries, 0) + self.assertEqual(_dotHealthCheckQueries, 1) name = 'dot.lazy.test.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN') @@ -314,7 +316,7 @@ class TestLazyHealthChecks(HealthCheckTest): (_, receivedResponse) = sender(query, response=None, useQueue=False) self.assertEqual(receivedResponse, response) - self.assertEqual(_dotHealthCheckQueries, 0) + self.assertEqual(_dotHealthCheckQueries, 1) # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough for _ in range(2): @@ -322,16 +324,17 @@ class TestLazyHealthChecks(HealthCheckTest): self.assertEqual(receivedResponse, failedResponse) time.sleep(1.5) - self.assertEqual(_dotHealthCheckQueries, 1) + self.assertEqual(_dotHealthCheckQueries, 2) self.assertEqual(self.getBackendStatus(), 'up') def testDoHLazy(self): """ Lazy Healthchecks: DoH """ - self.assertEqual(_dohHealthCheckQueries, 0) + # there is one initial query on startup + self.assertEqual(_dohHealthCheckQueries, 1) time.sleep(1) - self.assertEqual(_dohHealthCheckQueries, 0) + self.assertEqual(_dohHealthCheckQueries, 1) name = 'doh.lazy.test.powerdns.com.' query = dns.message.make_query(name, 'A', 'IN') @@ -347,7 +350,7 @@ class TestLazyHealthChecks(HealthCheckTest): (_, receivedResponse) = sender(query, response=None, useQueue=False) self.assertEqual(receivedResponse, response) - self.assertEqual(_dohHealthCheckQueries, 0) + self.assertEqual(_dohHealthCheckQueries, 1) # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough for _ in range(2): @@ -355,5 +358,5 @@ class TestLazyHealthChecks(HealthCheckTest): self.assertEqual(receivedResponse, failedResponse) time.sleep(1.5) - self.assertEqual(_dohHealthCheckQueries, 1) + self.assertEqual(_dohHealthCheckQueries, 2) self.assertEqual(self.getBackendStatus(), 'up')