regression-tests.dnsdist/test_HealthChecks.py

   1 #!/usr/bin/env python
   2 import base64
   3 import requests
   4 import ssl
   5 import threading
   6 import time
   7 import dns
   8 from dnsdisttests import DNSDistTest, pickAvailablePort
   9
  10 class HealthCheckTest(DNSDistTest):
  11     _consoleKey = DNSDistTest.generateConsoleKey()
  12     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
  13     _webTimeout = 2.0
  14     _webServerPort = pickAvailablePort()
  15     _webServerAPIKey = 'apisecret'
  16     _webServerAPIKeyHashed = '$scrypt$ln=10,p=1,r=8$9v8JxDfzQVyTpBkTbkUqYg==$bDQzAOHeK1G9UvTPypNhrX48w974ZXbFPtRKS34+aso='
  17     _config_params = ['_consoleKeyB64', '_consolePort', '_webServerPort', '_webServerAPIKeyHashed', '_testServerPort']
  18     _config_template = """
  19     setKey("%s")
  20     controlSocket("127.0.0.1:%d")
  21     webserver("127.0.0.1:%s")
  22     setWebserverConfig({apiKey="%s"})
  23     newServer{address="127.0.0.1:%d"}
  24     """
  25
  26     def getBackendStatus(self):
  27         return self.sendConsoleCommand("if getServer(0):isUp() then return 'up' else return 'down' end").strip("\n")
  28
  29     def getBackendMetric(self, backendID, metricName):
  30         headers = {'x-api-key': self._webServerAPIKey}
  31         url = 'http://127.0.0.1:' + str(self._webServerPort) + '/api/v1/servers/localhost'
  32         r = requests.get(url, headers=headers, timeout=self._webTimeout)
  33         self.assertTrue(r)
  34         self.assertEqual(r.status_code, 200)
  35         self.assertTrue(r.json())
  36         content = r.json()
  37         self.assertIn('servers', content)
  38         servers = content['servers']
  39         server = servers[backendID]
  40         return int(server[metricName])
  41
  42 class TestDefaultHealthCheck(HealthCheckTest):
  43     # this test suite uses a different responder port
  44     # because we need fresh counters
  45     _testServerPort = pickAvailablePort()
  46
  47     def testDefault(self):
  48         """
  49         HealthChecks: Default
  50         """
  51         before = TestDefaultHealthCheck._healthCheckCounter
  52         time.sleep(1.5)
  53         self.assertGreater(TestDefaultHealthCheck._healthCheckCounter, before)
  54         self.assertEqual(self.getBackendMetric(0, 'healthCheckFailures'), 0)
  55         self.assertEqual(self.getBackendStatus(), 'up')
  56
  57         self.sendConsoleCommand("getServer(0):setUp()")
  58         self.assertEqual(self.getBackendStatus(), 'up')
  59
  60         before = TestDefaultHealthCheck._healthCheckCounter
  61         time.sleep(1.5)
  62         self.assertEqual(TestDefaultHealthCheck._healthCheckCounter, before)
  63
  64         self.sendConsoleCommand("getServer(0):setDown()")
  65         self.assertEqual(self.getBackendStatus(), 'down')
  66
  67         before = TestDefaultHealthCheck._healthCheckCounter
  68         time.sleep(1.5)
  69         self.assertEqual(TestDefaultHealthCheck._healthCheckCounter, before)
  70
  71         self.sendConsoleCommand("getServer(0):setAuto()")
  72         # we get back the previous state, which was up
  73         self.assertEqual(self.getBackendStatus(), 'up')
  74
  75         before = TestDefaultHealthCheck._healthCheckCounter
  76         time.sleep(1.5)
  77         self.assertGreater(TestDefaultHealthCheck._healthCheckCounter, before)
  78         self.assertEqual(self.getBackendStatus(), 'up')
  79
  80         self.sendConsoleCommand("getServer(0):setDown()")
  81         self.assertEqual(self.getBackendStatus(), 'down')
  82         self.sendConsoleCommand("getServer(0):setAuto(false)")
  83
  84         before = TestDefaultHealthCheck._healthCheckCounter
  85         time.sleep(1.5)
  86         self.assertGreater(TestDefaultHealthCheck._healthCheckCounter, before)
  87         self.assertEqual(self.getBackendStatus(), 'up')
  88         self.assertEqual(self.getBackendMetric(0, 'healthCheckFailures'), 0)
  89
  90 class TestHealthCheckForcedUP(HealthCheckTest):
  91     # this test suite uses a different responder port
  92     # because we need fresh counters
  93     _testServerPort = pickAvailablePort()
  94
  95     _config_template = """
  96     setKey("%s")
  97     controlSocket("127.0.0.1:%d")
  98     webserver("127.0.0.1:%s")
  99     setWebserverConfig({apiKey="%s"})
 100     srv = newServer{address="127.0.0.1:%d"}
 101     srv:setUp()
 102     """
 103
 104     def testForcedUp(self):
 105         """
 106         HealthChecks: Forced UP
 107         """
 108         before = TestHealthCheckForcedUP._healthCheckCounter
 109         time.sleep(1.5)
 110         self.assertEqual(TestHealthCheckForcedUP._healthCheckCounter, before)
 111         self.assertEqual(self.getBackendStatus(), 'up')
 112         self.assertEqual(self.getBackendMetric(0, 'healthCheckFailures'), 0)
 113
 114 class TestHealthCheckForcedDown(HealthCheckTest):
 115     # this test suite uses a different responder port
 116     # because we need fresh counters
 117     _testServerPort = pickAvailablePort()
 118
 119     _config_template = """
 120     setKey("%s")
 121     controlSocket("127.0.0.1:%d")
 122     webserver("127.0.0.1:%s")
 123     setWebserverConfig({apiKey="%s"})
 124     srv = newServer{address="127.0.0.1:%d"}
 125     srv:setDown()
 126     """
 127
 128     def testForcedDown(self):
 129         """
 130         HealthChecks: Forced Down
 131         """
 132         before = TestHealthCheckForcedDown._healthCheckCounter
 133         time.sleep(1.5)
 134         self.assertEqual(TestHealthCheckForcedDown._healthCheckCounter, before)
 135         self.assertEqual(self.getBackendMetric(0, 'healthCheckFailures'), 0)
 136
 137 class TestHealthCheckCustomName(HealthCheckTest):
 138     # this test suite uses a different responder port
 139     # because it uses a different health check name
 140     _testServerPort = pickAvailablePort()
 141
 142     _healthCheckName = 'powerdns.com.'
 143     _config_params = ['_consoleKeyB64', '_consolePort', '_webServerPort', '_webServerAPIKeyHashed', '_testServerPort', '_healthCheckName']
 144     _config_template = """
 145     setKey("%s")
 146     controlSocket("127.0.0.1:%d")
 147     webserver("127.0.0.1:%s")
 148     setWebserverConfig({apiKey="%s"})
 149     srv = newServer{address="127.0.0.1:%d", checkName='%s'}
 150     """
 151
 152     def testAuto(self):
 153         """
 154         HealthChecks: Custom name
 155         """
 156         before = TestHealthCheckCustomName._healthCheckCounter
 157         time.sleep(1.5)
 158         self.assertGreater(TestHealthCheckCustomName._healthCheckCounter, before)
 159         self.assertEqual(self.getBackendStatus(), 'up')
 160         self.assertEqual(self.getBackendMetric(0, 'healthCheckFailures'), 0)
 161
 162 class TestHealthCheckCustomNameNoAnswer(HealthCheckTest):
 163     # this test suite uses a different responder port
 164     # because it uses a different health check configuration
 165     _testServerPort = pickAvailablePort()
 166
 167     _answerUnexpected = False
 168     _config_template = """
 169     setKey("%s")
 170     controlSocket("127.0.0.1:%d")
 171     webserver("127.0.0.1:%s")
 172     setWebserverConfig({apiKey="%s"})
 173     srv = newServer{address="127.0.0.1:%d", checkName='powerdns.com.'}
 174     """
 175
 176     def testAuto(self):
 177         """
 178         HealthChecks: Custom name not expected by the responder
 179         """
 180         before = TestHealthCheckCustomNameNoAnswer._healthCheckCounter
 181         time.sleep(1.5)
 182         self.assertEqual(TestHealthCheckCustomNameNoAnswer._healthCheckCounter, before)
 183         self.assertEqual(self.getBackendStatus(), 'down')
 184         self.assertGreater(self.getBackendMetric(0, 'healthCheckFailures'), 0)
 185         self.assertGreater(self.getBackendMetric(0, 'healthCheckFailuresTimeout'), 0)
 186
 187 class TestHealthCheckCustomFunction(HealthCheckTest):
 188     # this test suite uses a different responder port
 189     # because it uses a different health check configuration
 190     _testServerPort = pickAvailablePort()
 191     _answerUnexpected = False
 192
 193     _healthCheckName = 'powerdns.com.'
 194     _config_template = """
 195     setKey("%s")
 196     controlSocket("127.0.0.1:%d")
 197     webserver("127.0.0.1:%s")
 198     setWebserverConfig({apiKey="%s"})
 199
 200     function myHealthCheckFunction(qname, qtype, qclass, dh)
 201       dh:setCD(true)
 202
 203       return newDNSName('powerdns.com.'), DNSQType.AAAA, qclass
 204     end
 205
 206     srv = newServer{address="127.0.0.1:%d", checkName='powerdns.org.', checkFunction=myHealthCheckFunction}
 207     """
 208
 209     def testAuto(self):
 210         """
 211         HealthChecks: Custom function
 212         """
 213         before = TestHealthCheckCustomFunction._healthCheckCounter
 214         time.sleep(1.5)
 215         self.assertGreater(TestHealthCheckCustomFunction._healthCheckCounter, before)
 216         self.assertEqual(self.getBackendStatus(), 'up')
 217
 218 _do53HealthCheckQueries = 0
 219 _dotHealthCheckQueries = 0
 220 _dohHealthCheckQueries = 0
 221
 222 class TestLazyHealthChecks(HealthCheckTest):
 223     _extraStartupSleep = 1
 224     _do53Port = pickAvailablePort()
 225     _dotPort = pickAvailablePort()
 226     _dohPort = pickAvailablePort()
 227
 228     _consoleKey = DNSDistTest.generateConsoleKey()
 229     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 230     _config_params = ['_consoleKeyB64', '_consolePort', '_do53Port', '_dotPort', '_dohPort']
 231     _config_template = """
 232     setKey("%s")
 233     controlSocket("127.0.0.1:%d")
 234
 235     newServer{address="127.0.0.1:%s", healthCheckMode='lazy', checkInterval=1, lazyHealthCheckFailedInterval=1, lazyHealthCheckThreshold=10, lazyHealthCheckSampleSize=100,  lazyHealthCheckMinSampleCount=10, lazyHealthCheckMode='TimeoutOrServFail', pool=''}
 236
 237     newServer{address="127.0.0.1:%s", tls='openssl', caStore='ca.pem', healthCheckMode='lazy', checkInterval=1, lazyHealthCheckFailedInterval=1, lazyHealthCheckThreshold=10, lazyHealthCheckSampleSize=100,  lazyHealthCheckMinSampleCount=10, lazyHealthCheckMode='TimeoutOrServFail', pool='dot'}
 238     addAction('dot.lazy.test.powerdns.com.', PoolAction('dot'))
 239
 240     newServer{address="127.0.0.1:%s", tls='openssl', dohPath='/dns-query', caStore='ca.pem', healthCheckMode='lazy', checkInterval=1, lazyHealthCheckFailedInterval=1, lazyHealthCheckThreshold=10, lazyHealthCheckSampleSize=100,  lazyHealthCheckMinSampleCount=10, lazyHealthCheckMode='TimeoutOrServFail', pool='doh'}
 241     addAction('doh.lazy.test.powerdns.com.', PoolAction('doh'))
 242     """
 243     _verboseMode = True
 244
 245     @staticmethod
 246     def HandleDNSQuery(request):
 247         response = dns.message.make_response(request)
 248         if str(request.question[0].name).startswith('server-failure'):
 249             response.set_rcode(dns.rcode.SERVFAIL)
 250         return response.to_wire()
 251
 252     @classmethod
 253     def Do53Callback(cls, request):
 254         global _do53HealthCheckQueries
 255         if str(request.question[0].name).startswith('a.root-servers.net'):
 256             _do53HealthCheckQueries = _do53HealthCheckQueries + 1
 257             response = dns.message.make_response(request)
 258             return response.to_wire()
 259         return cls.HandleDNSQuery(request)
 260
 261     @classmethod
 262     def DoTCallback(cls, request):
 263         global _dotHealthCheckQueries
 264         if str(request.question[0].name).startswith('a.root-servers.net'):
 265             _dotHealthCheckQueries = _dotHealthCheckQueries + 1
 266             response = dns.message.make_response(request)
 267             return response.to_wire()
 268         return cls.HandleDNSQuery(request)
 269
 270     @classmethod
 271     def DoHCallback(cls, request, requestHeaders, fromQueue, toQueue):
 272         global _dohHealthCheckQueries
 273         if str(request.question[0].name).startswith('a.root-servers.net'):
 274             _dohHealthCheckQueries = _dohHealthCheckQueries + 1
 275             response = dns.message.make_response(request)
 276             return 200, response.to_wire()
 277         return 200, cls.HandleDNSQuery(request)
 278
 279     @classmethod
 280     def startResponders(cls):
 281         tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
 282         tlsContext.load_cert_chain('server.chain', 'server.key')
 283
 284         Do53Responder = threading.Thread(name='Do53 Lazy Responder', target=cls.UDPResponder, args=[cls._do53Port, cls._toResponderQueue, cls._fromResponderQueue, False, cls.Do53Callback])
 285         Do53Responder.daemon = True
 286         Do53Responder.start()
 287
 288         Do53TCPResponder = threading.Thread(name='Do53 TCP Lazy Responder', target=cls.TCPResponder, args=[cls._do53Port, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.Do53Callback])
 289         Do53TCPResponder.daemon = True
 290         Do53TCPResponder.start()
 291
 292         DoTResponder = threading.Thread(name='DoT Lazy Responder', target=cls.TCPResponder, args=[cls._dotPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.DoTCallback, tlsContext])
 293         DoTResponder.daemon = True
 294         DoTResponder.start()
 295
 296         DoHResponder = threading.Thread(name='DoH Lazy Responder', target=cls.DOHResponder, args=[cls._dohPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.DoHCallback, tlsContext])
 297         DoHResponder.daemon = True
 298         DoHResponder.start()
 299
 300     def testDo53Lazy(self):
 301         """
 302         Lazy Healthchecks: Do53
 303         """
 304         # there is one initial query on startup
 305         self.assertEqual(_do53HealthCheckQueries, 1)
 306         time.sleep(1)
 307         self.assertEqual(_do53HealthCheckQueries, 1)
 308
 309         name = 'do53.lazy.test.powerdns.com.'
 310         query = dns.message.make_query(name, 'A', 'IN')
 311         response = dns.message.make_response(query)
 312         failedQuery = dns.message.make_query('server-failure.do53.lazy.test.powerdns.com.', 'A', 'IN')
 313         failedResponse = dns.message.make_response(failedQuery)
 314         failedResponse.set_rcode(dns.rcode.SERVFAIL)
 315
 316         # send a few valid queries
 317         for _ in range(5):
 318             for method in ("sendUDPQuery", "sendTCPQuery"):
 319                 sender = getattr(self, method)
 320                 (_, receivedResponse) = sender(query, response=None, useQueue=False)
 321                 self.assertEqual(receivedResponse, response)
 322
 323         self.assertEqual(_do53HealthCheckQueries, 1)
 324
 325         # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough
 326         for _ in range(2):
 327             (_, receivedResponse) = self.sendUDPQuery(failedQuery, response=None, useQueue=False)
 328             self.assertEqual(receivedResponse, failedResponse)
 329
 330         time.sleep(1.5)
 331         self.assertEqual(_do53HealthCheckQueries, 2)
 332         self.assertEqual(self.getBackendStatus(), 'up')
 333
 334     def testDoTLazy(self):
 335         """
 336         Lazy Healthchecks: DoT
 337         """
 338         # there is one initial query on startup
 339         self.assertEqual(_dotHealthCheckQueries, 1)
 340         time.sleep(1)
 341         self.assertEqual(_dotHealthCheckQueries, 1)
 342
 343         name = 'dot.lazy.test.powerdns.com.'
 344         query = dns.message.make_query(name, 'A', 'IN')
 345         response = dns.message.make_response(query)
 346         failedQuery = dns.message.make_query('server-failure.dot.lazy.test.powerdns.com.', 'A', 'IN')
 347         failedResponse = dns.message.make_response(failedQuery)
 348         failedResponse.set_rcode(dns.rcode.SERVFAIL)
 349
 350         # send a few valid queries
 351         for _ in range(5):
 352             for method in ("sendUDPQuery", "sendTCPQuery"):
 353                 sender = getattr(self, method)
 354                 (_, receivedResponse) = sender(query, response=None, useQueue=False)
 355                 self.assertEqual(receivedResponse, response)
 356
 357         self.assertEqual(_dotHealthCheckQueries, 1)
 358
 359         # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough
 360         for _ in range(2):
 361             (_, receivedResponse) = self.sendUDPQuery(failedQuery, response=None, useQueue=False)
 362             self.assertEqual(receivedResponse, failedResponse)
 363
 364         time.sleep(1.5)
 365         self.assertEqual(_dotHealthCheckQueries, 2)
 366         self.assertEqual(self.getBackendStatus(), 'up')
 367
 368     def testDoHLazy(self):
 369         """
 370         Lazy Healthchecks: DoH
 371         """
 372         # there is one initial query on startup
 373         self.assertEqual(_dohHealthCheckQueries, 1)
 374         time.sleep(1)
 375         self.assertEqual(_dohHealthCheckQueries, 1)
 376
 377         name = 'doh.lazy.test.powerdns.com.'
 378         query = dns.message.make_query(name, 'A', 'IN')
 379         response = dns.message.make_response(query)
 380         failedQuery = dns.message.make_query('server-failure.doh.lazy.test.powerdns.com.', 'A', 'IN')
 381         failedResponse = dns.message.make_response(failedQuery)
 382         failedResponse.set_rcode(dns.rcode.SERVFAIL)
 383
 384         # send a few valid queries
 385         for _ in range(5):
 386             for method in ("sendUDPQuery", "sendTCPQuery"):
 387                 sender = getattr(self, method)
 388                 (_, receivedResponse) = sender(query, response=None, useQueue=False)
 389                 self.assertEqual(receivedResponse, response)
 390
 391         self.assertEqual(_dohHealthCheckQueries, 1)
 392
 393         # we need at least 10 samples, and 10 percent of them failing, so two failing queries should be enough
 394         for _ in range(2):
 395             (_, receivedResponse) = self.sendUDPQuery(failedQuery, response=None, useQueue=False)
 396             self.assertEqual(receivedResponse, failedResponse)
 397
 398         time.sleep(1.5)
 399         self.assertEqual(_dohHealthCheckQueries, 2)
 400         self.assertEqual(self.getBackendStatus(), 'up')