regression-tests.dnsdist/test_BackendDiscovery.py

   1 #!/usr/bin/env python
   2 import base64
   3 import dns
   4 import threading
   5 import time
   6 import ssl
   7
   8 from dnsdisttests import DNSDistTest
   9
  10 class TestBackendDiscovery(DNSDistTest):
  11     # these ports are hardcoded for now, sorry about that!
  12     _noSVCBackendPort = 10600
  13     _svcNoUpgradeBackendPort = 10601
  14     _svcUpgradeDoTBackendPort = 10602
  15     _svcUpgradeDoHBackendPort = 10603
  16     _svcUpgradeDoTBackendDifferentAddrPort1 = 10604
  17     _svcUpgradeDoTBackendDifferentAddrPort2 = 10605
  18     _svcUpgradeDoTUnreachableBackendPort = 10606
  19     _svcBrokenDNSResponseBackendPort = 10607
  20     _svcUpgradeDoHBackendWithoutPathPort = 10608
  21     _connectionRefusedBackendPort = 10609
  22     _eofBackendPort = 10610
  23     _servfailBackendPort = 10611
  24     _wrongNameBackendPort = 10612
  25     _wrongIDBackendPort = 10613
  26     _tooManyQuestionsBackendPort = 10614
  27     _badQNameBackendPort = 10615
  28     _svcUpgradeDoTNoPortBackendPort = 10616
  29     _svcUpgradeDoHNoPortBackendPort = 10617
  30     _upgradedBackendsPool = 'upgraded'
  31
  32     _consoleKey = DNSDistTest.generateConsoleKey()
  33     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
  34     _config_params = ['_consoleKeyB64', '_consolePort', '_noSVCBackendPort', '_svcNoUpgradeBackendPort', '_svcUpgradeDoTBackendPort', '_upgradedBackendsPool', '_svcUpgradeDoHBackendPort', '_svcUpgradeDoTBackendDifferentAddrPort1', '_svcUpgradeDoTBackendDifferentAddrPort2', '_svcUpgradeDoTUnreachableBackendPort', '_svcBrokenDNSResponseBackendPort', '_svcUpgradeDoHBackendWithoutPathPort', '_connectionRefusedBackendPort', '_eofBackendPort', '_servfailBackendPort', '_wrongNameBackendPort', '_wrongIDBackendPort', '_tooManyQuestionsBackendPort', '_badQNameBackendPort', '_svcUpgradeDoTNoPortBackendPort', '_svcUpgradeDoHNoPortBackendPort']
  35     _config_template = """
  36     setKey("%s")
  37     controlSocket("127.0.0.1:%d")
  38
  39     setMaxTCPClientThreads(1)
  40
  41     -- no SVCB
  42     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  43
  44     -- SVCB record but no upgrade path available
  45     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  46
  47     -- SVCB upgrade to DoT, same address, keep the backend, different pool
  48     newServer{address="127.0.0.1:%s", caStore='ca.pem', pool={'', 'another-pool'}, autoUpgrade=true, autoUpgradePool='%s', autoUpgradeKeep=true, source='127.0.0.1@lo'}:setUp()
  49
  50     -- SVCB upgrade to DoH, same address, do not keep the backend, same pool
  51     newServer{address="127.0.0.1:%s", caStore='ca.pem', pool={'another-pool'}, autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  52
  53     -- SVCB upgrade to DoT, different address, certificate is valid for the initial address
  54     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  55
  56     -- SVCB upgrade to DoT, different address, certificate is NOT valid for the initial address
  57     newServer{address="127.0.0.2:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  58
  59     -- SVCB upgrade to DoT but upgraded port is not reachable
  60     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  61
  62     -- The SVCB response is not valid
  63     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  64
  65     -- SVCB upgrade to DoH except the path is not specified
  66     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  67
  68     -- Connection refused
  69     newServer({address="127.0.0.1:%s", caStore='ca.pem', pool={"", "other-pool"}, autoUpgrade=true, source='127.0.0.1@lo'}):setUp()
  70
  71     -- EOF
  72     newServer({address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true}):setUp()
  73
  74     -- ServFail
  75     newServer({address="127.0.0.1:%s", autoUpgrade=true}):setUp()
  76
  77     -- Wrong name
  78     newServer({address="127.0.0.1:%s", autoUpgrade=true}):setUp()
  79
  80     -- Wrong ID
  81     newServer({address="127.0.0.1:%s", autoUpgrade=true}):setUp()
  82
  83     -- Too many questions
  84     newServer({address="127.0.0.1:%s", autoUpgrade=true}):setUp()
  85
  86     -- Bad QName
  87     newServer({address="127.0.0.1:%s", autoUpgrade=true}):setUp()
  88
  89     -- SVCB upgrade to DoT, same address, no port specified via SVCB
  90     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  91
  92     -- SVCB upgrade to DoH, same address, no port specified via SVCB
  93     newServer{address="127.0.0.1:%s", caStore='ca.pem', autoUpgrade=true, autoUpgradeKeep=false}:setUp()
  94     """
  95     _verboseMode = True
  96
  97     def NoSVCCallback(request):
  98         return dns.message.make_response(request).to_wire()
  99
 100     def NoUpgradePathCallback(request):
 101         response = dns.message.make_response(request)
 102         rrset = dns.rrset.from_text(request.question[0].name,
 103                                     60,
 104                                     dns.rdataclass.IN,
 105                                     dns.rdatatype.SVCB,
 106                                     '1 no-upgrade. alpn="h3"')
 107         response.answer.append(rrset)
 108         return response.to_wire()
 109
 110     def UpgradeDoTCallback(request):
 111         response = dns.message.make_response(request)
 112         rrset = dns.rrset.from_text(request.question[0].name,
 113                                     60,
 114                                     dns.rdataclass.IN,
 115                                     dns.rdatatype.SVCB,
 116                                     '1 tls.tests.dnsdist.org. alpn="dot" port=10652 ipv4hint=127.0.0.1')
 117         response.answer.append(rrset)
 118         # add a useless A record for good measure
 119         rrset = dns.rrset.from_text(request.question[0].name,
 120                                     60,
 121                                     dns.rdataclass.IN,
 122                                     dns.rdatatype.A,
 123                                     '192.0.2.1')
 124         response.answer.append(rrset)
 125         # plus more useless records in authority
 126         rrset = dns.rrset.from_text(request.question[0].name,
 127                                     60,
 128                                     dns.rdataclass.IN,
 129                                     dns.rdatatype.A,
 130                                     '192.0.2.1')
 131         response.authority.append(rrset)
 132         # and finally valid, albeit useless, hints
 133         rrset = dns.rrset.from_text('tls.tests.dnsdist.org.',
 134                                     60,
 135                                     dns.rdataclass.IN,
 136                                     dns.rdatatype.A,
 137                                     '127.0.0.1')
 138         response.additional.append(rrset)
 139         rrset = dns.rrset.from_text('tls.tests.dnsdist.org.',
 140                                     60,
 141                                     dns.rdataclass.IN,
 142                                     dns.rdatatype.AAAA,
 143                                     '::1')
 144         response.additional.append(rrset)
 145         return response.to_wire()
 146
 147     def UpgradeDoHCallback(request):
 148         response = dns.message.make_response(request)
 149         rrset = dns.rrset.from_text(request.question[0].name,
 150                                     60,
 151                                     dns.rdataclass.IN,
 152                                     dns.rdatatype.SVCB,
 153                                     '1 tls.tests.dnsdist.org. alpn="h2" port=10653 ipv4hint=127.0.0.1 key7="/dns-query{?dns}"')
 154         response.answer.append(rrset)
 155         return response.to_wire()
 156
 157     def UpgradeDoTDifferentAddr1Callback(request):
 158         response = dns.message.make_response(request)
 159         rrset = dns.rrset.from_text(request.question[0].name,
 160                                     60,
 161                                     dns.rdataclass.IN,
 162                                     dns.rdatatype.SVCB,
 163                                     '1 tls.tests.dnsdist.org. alpn="dot" port=10654 ipv4hint=127.0.0.2')
 164         response.answer.append(rrset)
 165         return response.to_wire()
 166
 167     def UpgradeDoTDifferentAddr2Callback(request):
 168         response = dns.message.make_response(request)
 169         rrset = dns.rrset.from_text(request.question[0].name,
 170                                     60,
 171                                     dns.rdataclass.IN,
 172                                     dns.rdatatype.SVCB,
 173                                     '1 tls.tests.dnsdist.org. alpn="dot" port=10655 ipv4hint=127.0.0.1')
 174         response.answer.append(rrset)
 175         return response.to_wire()
 176
 177     def UpgradeDoTUnreachableCallback(request):
 178         response = dns.message.make_response(request)
 179         rrset = dns.rrset.from_text(request.question[0].name,
 180                                     60,
 181                                     dns.rdataclass.IN,
 182                                     dns.rdatatype.SVCB,
 183                                     '1 tls.tests.dnsdist.org. alpn="dot" port=10656 ipv4hint=127.0.0.1')
 184         response.answer.append(rrset)
 185         return response.to_wire()
 186
 187     def BrokenResponseCallback(request):
 188         response = dns.message.make_response(request)
 189         response.use_edns(edns=False)
 190         response.question = []
 191         return response.to_wire()
 192
 193     def UpgradeDoHMissingPathCallback(request):
 194         response = dns.message.make_response(request)
 195         rrset = dns.rrset.from_text(request.question[0].name,
 196                                     60,
 197                                     dns.rdataclass.IN,
 198                                     dns.rdatatype.SVCB,
 199                                     '1 tls.tests.dnsdist.org. alpn="h2" port=10653 ipv4hint=127.0.0.1')
 200         response.answer.append(rrset)
 201         return response.to_wire()
 202
 203     def EOFCallback(request):
 204         return None
 205
 206     def ServFailCallback(request):
 207         response = dns.message.make_response(request)
 208         response.set_rcode(dns.rcode.SERVFAIL)
 209         return response.to_wire()
 210
 211     def WrongNameCallback(request):
 212         query = dns.message.make_query('not-the-right-one.', dns.rdatatype.SVCB)
 213         response = dns.message.make_response(query)
 214         response.id = request.id
 215         return response.to_wire()
 216
 217     def WrongIDCallback(request):
 218         response = dns.message.make_response(request)
 219         response.id = request.id ^ 42
 220         return response.to_wire()
 221
 222     def WrongIDCallback(request):
 223         response = dns.message.make_response(request)
 224         response.id = request.id ^ 42
 225         return response.to_wire()
 226
 227     def TooManyQuestionsCallback(request):
 228         response = dns.message.make_response(request)
 229         response.question.append(response.question[0])
 230         return response.to_wire()
 231
 232     def BadQNameCallback(request):
 233         response = dns.message.make_response(request)
 234         wire = bytearray(response.to_wire())
 235         # mess up the first label length
 236         wire[12] = 0xFF
 237         return wire
 238
 239     def UpgradeDoTNoPortCallback(request):
 240         response = dns.message.make_response(request)
 241         rrset = dns.rrset.from_text(request.question[0].name,
 242                                     60,
 243                                     dns.rdataclass.IN,
 244                                     dns.rdatatype.SVCB,
 245                                     '1 tls.tests.dnsdist.org. alpn="dot" ipv4hint=127.0.0.1')
 246         response.answer.append(rrset)
 247         return response.to_wire()
 248
 249     def UpgradeDoHNoPortCallback(request):
 250         response = dns.message.make_response(request)
 251         rrset = dns.rrset.from_text(request.question[0].name,
 252                                     60,
 253                                     dns.rdataclass.IN,
 254                                     dns.rdatatype.SVCB,
 255                                     '1 tls.tests.dnsdist.org. alpn="h2" ipv4hint=127.0.0.1 key7="/dns-query{?dns}"')
 256         response.answer.append(rrset)
 257         return response.to_wire()
 258
 259     @classmethod
 260     def startResponders(cls):
 261         tlsContext = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
 262         tlsContext.load_cert_chain('server.chain', 'server.key')
 263
 264         TCPNoSVCResponder = threading.Thread(name='TCP no SVC Responder', target=cls.TCPResponder, args=[cls._noSVCBackendPort, cls._toResponderQueue, cls._fromResponderQueue, True, False, cls.NoSVCCallback])
 265         TCPNoSVCResponder.daemon = True
 266         TCPNoSVCResponder.start()
 267
 268         TCPNoUpgradeResponder = threading.Thread(name='TCP no upgrade Responder', target=cls.TCPResponder, args=[cls._svcNoUpgradeBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.NoUpgradePathCallback])
 269         TCPNoUpgradeResponder.daemon = True
 270         TCPNoUpgradeResponder.start()
 271
 272         # this one is special, does partial writes!
 273         TCPUpgradeToDoTResponder = threading.Thread(name='TCP upgrade to DoT Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoTBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoTCallback, None, False, '127.0.0.1', True])
 274         TCPUpgradeToDoTResponder.daemon = True
 275         TCPUpgradeToDoTResponder.start()
 276         # and the corresponding DoT responder
 277         UpgradedDoTResponder = threading.Thread(name='DoT upgraded Responder', target=cls.TCPResponder, args=[10652, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext])
 278         UpgradedDoTResponder.daemon = True
 279         UpgradedDoTResponder.start()
 280
 281         TCPUpgradeToDoHResponder = threading.Thread(name='TCP upgrade to DoH Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoHBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoHCallback])
 282         TCPUpgradeToDoHResponder.daemon = True
 283         TCPUpgradeToDoHResponder.start()
 284         # and the corresponding DoH responder
 285         UpgradedDOHResponder = threading.Thread(name='DOH Responder', target=cls.DOHResponder, args=[10653, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext])
 286         UpgradedDOHResponder.daemon = True
 287         UpgradedDOHResponder.start()
 288
 289         TCPUpgradeToDoTDifferentAddrResponder = threading.Thread(name='TCP upgrade to DoT different addr 1 Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoTBackendDifferentAddrPort1, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoTDifferentAddr1Callback])
 290         TCPUpgradeToDoTDifferentAddrResponder.daemon = True
 291         TCPUpgradeToDoTDifferentAddrResponder.start()
 292         # and the corresponding DoT responder
 293         UpgradedDoTResponder = threading.Thread(name='DoT upgraded different addr 1 Responder', target=cls.TCPResponder, args=[10654, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext, False, '127.0.0.2'])
 294         UpgradedDoTResponder.daemon = True
 295         UpgradedDoTResponder.start()
 296
 297         TCPUpgradeToDoTDifferentAddrResponder = threading.Thread(name='TCP upgrade to DoT different addr 2 Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoTBackendDifferentAddrPort2, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoTDifferentAddr2Callback, None, False, '127.0.0.2'])
 298         TCPUpgradeToDoTDifferentAddrResponder.daemon = True
 299         TCPUpgradeToDoTDifferentAddrResponder.start()
 300         # and the corresponding DoT responder
 301         UpgradedDoTResponder = threading.Thread(name='DoT upgraded different addr 2 Responder', target=cls.TCPResponder, args=[10655, cls._toResponderQueue, cls._fromResponderQueue, False, False, None, tlsContext, False])
 302         UpgradedDoTResponder.daemon = True
 303         UpgradedDoTResponder.start()
 304
 305         TCPUpgradeToUnreachableDoTResponder = threading.Thread(name='TCP upgrade to unreachable DoT Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoTUnreachableBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoTUnreachableCallback])
 306         TCPUpgradeToUnreachableDoTResponder.daemon = True
 307         TCPUpgradeToUnreachableDoTResponder.start()
 308         # and NO corresponding DoT responder
 309         # this is not a mistake!
 310
 311         BrokenResponseResponder = threading.Thread(name='Broken response Responder', target=cls.TCPResponder, args=[cls._svcBrokenDNSResponseBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.BrokenResponseCallback])
 312         BrokenResponseResponder.daemon = True
 313         BrokenResponseResponder.start()
 314
 315         DOHMissingPathResponder = threading.Thread(name='DoH missing path Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoHBackendWithoutPathPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoHMissingPathCallback])
 316         DOHMissingPathResponder.daemon = True
 317         DOHMissingPathResponder.start()
 318
 319         EOFResponder = threading.Thread(name='EOF Responder', target=cls.TCPResponder, args=[cls._eofBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.EOFCallback])
 320         EOFResponder.daemon = True
 321         EOFResponder.start()
 322
 323         ServFailResponder = threading.Thread(name='ServFail Responder', target=cls.TCPResponder, args=[cls._servfailBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.ServFailCallback])
 324         ServFailResponder.daemon = True
 325         ServFailResponder.start()
 326
 327         WrongNameResponder = threading.Thread(name='Wrong Name Responder', target=cls.TCPResponder, args=[cls._wrongNameBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.WrongNameCallback])
 328         WrongNameResponder.daemon = True
 329         WrongNameResponder.start()
 330
 331         WrongIDResponder = threading.Thread(name='Wrong ID Responder', target=cls.TCPResponder, args=[cls._wrongIDBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.WrongIDCallback])
 332         WrongIDResponder.daemon = True
 333         WrongIDResponder.start()
 334
 335         TooManyQuestionsResponder = threading.Thread(name='Too many questions Responder', target=cls.TCPResponder, args=[cls._tooManyQuestionsBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.TooManyQuestionsCallback])
 336         TooManyQuestionsResponder.daemon = True
 337         TooManyQuestionsResponder.start()
 338
 339         badQNameResponder = threading.Thread(name='Bad QName Responder', target=cls.TCPResponder, args=[cls._badQNameBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.BadQNameCallback])
 340         badQNameResponder.daemon = True
 341         badQNameResponder.start()
 342
 343         TCPUpgradeToDoTNoPortResponder = threading.Thread(name='TCP upgrade to DoT (no port) Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoTNoPortBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoTNoPortCallback])
 344         TCPUpgradeToDoTNoPortResponder.daemon = True
 345         TCPUpgradeToDoTNoPortResponder.start()
 346
 347         TCPUpgradeToDoHNoPortResponder = threading.Thread(name='TCP upgrade to DoH (no port) Responder', target=cls.TCPResponder, args=[cls._svcUpgradeDoHNoPortBackendPort, cls._toResponderQueue, cls._fromResponderQueue, False, False, cls.UpgradeDoHNoPortCallback])
 348         TCPUpgradeToDoHNoPortResponder.daemon = True
 349         TCPUpgradeToDoHNoPortResponder.start()
 350
 351
 352     def checkBackendsUpgraded(self):
 353         output = self.sendConsoleCommand('showServers()')
 354         print(output)
 355
 356         backends = {}
 357         for line in output.splitlines(False):
 358             if line.startswith('#') or line.startswith('All'):
 359                 continue
 360             tokens = line.split()
 361             self.assertTrue(len(tokens) == 13 or len(tokens) == 14)
 362             if tokens[1] == '127.0.0.1:10652':
 363                 # in this particular case, the upgraded backend
 364                 # does not replace the existing one and thus
 365                 # the health-check is forced to auto (or lazy auto)
 366                 self.assertEqual(tokens[2], 'up')
 367             else:
 368                 self.assertEqual(tokens[2], 'UP')
 369             pool = ''
 370             if len(tokens) == 14:
 371                 pool = tokens[13]
 372             backends[tokens[1]] = pool
 373
 374         expected = {
 375             '127.0.0.1:10600': '',
 376             '127.0.0.1:10601': '',
 377             '127.0.0.1:10602': 'another-pool',
 378             # 10603 has been upgraded to 10653 and removed
 379             # 10604 has been upgraded to 10654 and removed
 380             '127.0.0.2:10605': '',
 381             '127.0.0.1:10606': '',
 382             '127.0.0.1:10607': '',
 383             '127.0.0.1:10608': '',
 384             '127.0.0.1:10609': 'other-pool',
 385             '127.0.0.1:10610': '',
 386             '127.0.0.1:10611': '',
 387             '127.0.0.1:10612': '',
 388             '127.0.0.1:10613': '',
 389             '127.0.0.1:10614': '',
 390             '127.0.0.1:10615': '',
 391             # these two are not upgraded because there is no backend listening on the default ports (443 and 853)
 392             '127.0.0.1:10616': '',
 393             '127.0.0.1:10617': '',
 394             '127.0.0.1:10652': 'upgraded',
 395             '127.0.0.1:10653': 'another-pool',
 396             '127.0.0.2:10654': ''
 397         }
 398         print(backends)
 399         return backends == expected
 400
 401     def testBackendUpgrade(self):
 402         """
 403         Backend Discovery: Upgrade
 404         """
 405         # enough time for discovery to happen
 406         # 5s is not enough with TSAN
 407         time.sleep(10)
 408         if not self.checkBackendsUpgraded():
 409             # let's wait a bit longer
 410             time.sleep(5)
 411             self.assertTrue(self.checkBackendsUpgraded())
 412
 413 class TestBackendDiscoveryByHostname(DNSDistTest):
 414     _consoleKey = DNSDistTest.generateConsoleKey()
 415     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 416     _config_params = ['_consoleKeyB64', '_consolePort']
 417     _config_template = """
 418     setKey("%s")
 419     controlSocket("127.0.0.1:%d")
 420
 421     function resolveCB(hostname, ips)
 422       print('Got response for '..hostname)
 423       for _, ip in ipairs(ips) do
 424         print(ip)
 425         newServer(ip:toString())
 426       end
 427     end
 428
 429     getAddressInfo('dns.quad9.net.', resolveCB)
 430     """
 431     def checkBackends(self):
 432         output = self.sendConsoleCommand('showServers()')
 433         print(output)
 434         backends = {}
 435         for line in output.splitlines(False):
 436             if line.startswith('#') or line.startswith('All'):
 437                 continue
 438             tokens = line.split()
 439             self.assertTrue(len(tokens) == 13 or len(tokens) == 14)
 440             backends[tokens[1]] = tokens[2]
 441
 442         if len(backends) != 4:
 443             return False
 444
 445         for expected in ['9.9.9.9:53', '149.112.112.112:53', '[2620:fe::9]:53', '[2620:fe::fe]:53']:
 446             self.assertIn(expected, backends)
 447         for backend in backends:
 448             self.assertTrue(backends[backend])
 449         return True
 450
 451     def testBackendFromHostname(self):
 452         """
 453         Backend Discovery: From hostname
 454         """
 455         # enough time for resolution to happen
 456         time.sleep(4)
 457         if not self.checkBackends():
 458             time.sleep(4)
 459             self.assertTrue(self.checkBackends())