lastResults.clear();
vinfolog("Backend %s reached the lazy health-check threshold (%f%% out of %f%%, looking at sample of %d items with %d failures), moving to Potential Failure state", getNameWithAddr(), current, maxFailureRate, totalCount, failures);
stats->d_status = LazyHealthCheckStats::LazyStatus::PotentialFailure;
+ consecutiveSuccessfulChecks = 0;
/* we update the next check time here because the check might time out,
and we do not want to send a second check during that time unless
the timer is actually very short */
time_t backOff = d_config.d_lazyHealthCheckMaxBackOff;
const ExponentialBackOffTimer backOffTimer(d_config.d_lazyHealthCheckMaxBackOff);
- auto backOffCoeffTmp = backOffTimer.get(failedTests);
+ auto backOffCoeffTmp = backOffTimer.get(failedTests - 1);
/* backOffCoeffTmp cannot be higher than d_config.d_lazyHealthCheckMaxBackOff */
const auto backOffCoeff = static_cast<time_t>(backOffCoeffTmp);
if ((std::numeric_limits<time_t>::max() / d_config.d_lazyHealthCheckFailedInterval) >= backOffCoeff) {
if (newResult) {
/* check succeeded */
currentCheckFailures = 0;
+ consecutiveSuccessfulChecks++;
if (!upStatus) {
/* we were previously marked as "down" and had a successful health-check,
let's see if this is enough to move to the "up" state or if we need
more successful health-checks for that */
- consecutiveSuccessfulChecks++;
if (consecutiveSuccessfulChecks < d_config.minRiseSuccesses) {
/* we need more than one successful check to rise
and we didn't reach the threshold yet, let's stay down */
auto stats = d_lazyHealthCheckStats.lock();
vinfolog("Backend %s failed its health-check, moving from Potential failure to Failed", getNameWithAddr());
stats->d_status = LazyHealthCheckStats::LazyStatus::Failed;
- currentCheckFailures = 0;
+ currentCheckFailures = 1;
updateNextLazyHealthCheck(*stats, false);
}
}
BOOST_CHECK_EQUAL(ds.getStatus(), "down");
BOOST_CHECK_EQUAL(ds.healthCheckRequired(currentTime), false);
/* and the wait time between two checks will double every time a failure occurs */
- BOOST_CHECK_EQUAL(ds.getNextLazyHealthCheck(), (currentTime + (config.d_lazyHealthCheckFailedInterval * std::pow(2U, ds.currentCheckFailures))));
- BOOST_CHECK_EQUAL(ds.currentCheckFailures, 0U);
+ BOOST_CHECK_EQUAL(ds.getNextLazyHealthCheck(), (currentTime + (config.d_lazyHealthCheckFailedInterval * std::pow(2U, ds.currentCheckFailures - 1))));
+ BOOST_CHECK_EQUAL(ds.currentCheckFailures, 1U);
/* so after 5 failures */
const size_t nbFailures = 5;
BOOST_CHECK(ds.healthCheckRequired(currentTime));
ds.submitHealthCheckResult(false, false);
}
- BOOST_CHECK_EQUAL(ds.currentCheckFailures, nbFailures);
- BOOST_CHECK_EQUAL(ds.getNextLazyHealthCheck(), (currentTime + (config.d_lazyHealthCheckFailedInterval * std::pow(2U, ds.currentCheckFailures))));
+ BOOST_CHECK_EQUAL(ds.currentCheckFailures, nbFailures + 1);
+ BOOST_CHECK_EQUAL(ds.getNextLazyHealthCheck(), (currentTime + (config.d_lazyHealthCheckFailedInterval * std::pow(2U, ds.currentCheckFailures - 1))));
/* we need minRiseSuccesses successful health-checks to go up */
BOOST_REQUIRE(config.minRiseSuccesses >= 1);