regression-tests.dnsdist/test_Caching.py

   1 #!/usr/bin/env python
   2 import base64
   3 import time
   4 import dns
   5 import clientsubnetoption
   6 from dnsdisttests import DNSDistTest
   7
   8 class TestCaching(DNSDistTest):
   9
  10     _config_template = """
  11     -- maxTTL=86400, minTTL=1
  12     pc = newPacketCache(100, 86400, 1)
  13     getPool(""):setCache(pc)
  14     addAction(makeRule("nocache.cache.tests.powerdns.com."), SkipCacheAction())
  15     function skipViaLua(dq)
  16         dq.skipCache = true
  17         return DNSAction.None, ""
  18     end
  19     addAction("nocachevialua.cache.tests.powerdns.com.", LuaAction(skipViaLua))
  20     newServer{address="127.0.0.1:%d"}
  21     """
  22
  23     def testCached(self):
  24         """
  25         Cache: Served from cache
  26
  27         dnsdist is configured to cache entries, we are sending several
  28         identical requests and checking that the backend only receive
  29         the first one.
  30         """
  31         numberOfQueries = 10
  32         name = 'cached.cache.tests.powerdns.com.'
  33         query = dns.message.make_query(name, 'AAAA', 'IN')
  34         response = dns.message.make_response(query)
  35         rrset = dns.rrset.from_text(name,
  36                                     3600,
  37                                     dns.rdataclass.IN,
  38                                     dns.rdatatype.AAAA,
  39                                     '::1')
  40         response.answer.append(rrset)
  41
  42         # first query to fill the cache
  43         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
  44         self.assertTrue(receivedQuery)
  45         self.assertTrue(receivedResponse)
  46         receivedQuery.id = query.id
  47         self.assertEquals(query, receivedQuery)
  48         self.assertEquals(receivedResponse, response)
  49
  50         for _ in range(numberOfQueries):
  51             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
  52             self.assertEquals(receivedResponse, response)
  53
  54         total = 0
  55         for key in self._responsesCounter:
  56             total += self._responsesCounter[key]
  57             TestCaching._responsesCounter[key] = 0
  58
  59         self.assertEquals(total, 1)
  60
  61         # TCP should not be cached
  62         # first query to fill the cache
  63         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
  64         self.assertTrue(receivedQuery)
  65         self.assertTrue(receivedResponse)
  66         receivedQuery.id = query.id
  67         self.assertEquals(query, receivedQuery)
  68         self.assertEquals(receivedResponse, response)
  69
  70         for _ in range(numberOfQueries):
  71             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
  72             self.assertEquals(receivedResponse, response)
  73
  74         total = 0
  75         for key in self._responsesCounter:
  76             total += self._responsesCounter[key]
  77             TestCaching._responsesCounter[key] = 0
  78
  79         self.assertEquals(total, 1)
  80
  81     def testDOCached(self):
  82         """
  83         Cache: Served from cache, query has DO bit set
  84
  85         dnsdist is configured to cache entries, we are sending several
  86         identical requests and checking that the backend only receive
  87         the first one.
  88         """
  89         numberOfQueries = 10
  90         name = 'cached-do.cache.tests.powerdns.com.'
  91         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, want_dnssec=True)
  92         response = dns.message.make_response(query)
  93         rrset = dns.rrset.from_text(name,
  94                                     3600,
  95                                     dns.rdataclass.IN,
  96                                     dns.rdatatype.AAAA,
  97                                     '::1')
  98         response.answer.append(rrset)
  99
 100         # first query to fill the cache
 101         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 102         self.assertTrue(receivedQuery)
 103         self.assertTrue(receivedResponse)
 104         receivedQuery.id = query.id
 105         self.assertEquals(query, receivedQuery)
 106         self.assertEquals(receivedResponse, response)
 107
 108         for _ in range(numberOfQueries):
 109             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 110             self.assertEquals(receivedResponse, response)
 111
 112         total = 0
 113         for key in self._responsesCounter:
 114             total += self._responsesCounter[key]
 115             TestCaching._responsesCounter[key] = 0
 116
 117         self.assertEquals(total, 1)
 118
 119         # TCP should not be cached
 120         # first query to fill the cache
 121         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 122         self.assertTrue(receivedQuery)
 123         self.assertTrue(receivedResponse)
 124         receivedQuery.id = query.id
 125         self.assertEquals(query, receivedQuery)
 126         self.assertEquals(receivedResponse, response)
 127
 128         for _ in range(numberOfQueries):
 129             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
 130             self.assertEquals(receivedResponse, response)
 131
 132         total = 0
 133         for key in self._responsesCounter:
 134             total += self._responsesCounter[key]
 135             TestCaching._responsesCounter[key] = 0
 136
 137         self.assertEquals(total, 1)
 138
 139     def testSkipCache(self):
 140         """
 141         Cache: SkipCacheAction
 142
 143         dnsdist is configured to not cache entries for nocache.cache.tests.powerdns.com.
 144          we are sending several requests and checking that the backend get them all.
 145         """
 146         name = 'nocache.cache.tests.powerdns.com.'
 147         numberOfQueries = 10
 148         query = dns.message.make_query(name, 'AAAA', 'IN')
 149         response = dns.message.make_response(query)
 150         rrset = dns.rrset.from_text(name,
 151                                     3600,
 152                                     dns.rdataclass.IN,
 153                                     dns.rdatatype.AAAA,
 154                                     '::1')
 155         response.answer.append(rrset)
 156
 157         for _ in range(numberOfQueries):
 158             for method in ("sendUDPQuery", "sendTCPQuery"):
 159                 sender = getattr(self, method)
 160                 (receivedQuery, receivedResponse) = sender(query, response)
 161                 self.assertTrue(receivedQuery)
 162                 self.assertTrue(receivedResponse)
 163                 receivedQuery.id = query.id
 164                 self.assertEquals(query, receivedQuery)
 165                 self.assertEquals(receivedResponse, response)
 166
 167         for key in self._responsesCounter:
 168             value = self._responsesCounter[key]
 169             self.assertEquals(value, numberOfQueries)
 170
 171     def testSkipCacheViaLua(self):
 172         """
 173         Cache: SkipCache via Lua
 174
 175         dnsdist is configured to not cache entries for nocachevialua.cache.tests.powerdns.com.
 176          we are sending several requests and checking that the backend get them all.
 177         """
 178         name = 'nocachevialua.cache.tests.powerdns.com.'
 179         numberOfQueries = 10
 180         query = dns.message.make_query(name, 'AAAA', 'IN')
 181         response = dns.message.make_response(query)
 182         rrset = dns.rrset.from_text(name,
 183                                     3600,
 184                                     dns.rdataclass.IN,
 185                                     dns.rdatatype.AAAA,
 186                                     '::1')
 187         response.answer.append(rrset)
 188
 189         for _ in range(numberOfQueries):
 190             for method in ("sendUDPQuery", "sendTCPQuery"):
 191                 sender = getattr(self, method)
 192                 (receivedQuery, receivedResponse) = sender(query, response)
 193                 self.assertTrue(receivedQuery)
 194                 self.assertTrue(receivedResponse)
 195                 receivedQuery.id = query.id
 196                 self.assertEquals(query, receivedQuery)
 197                 self.assertEquals(receivedResponse, response)
 198
 199         for key in self._responsesCounter:
 200             value = self._responsesCounter[key]
 201             self.assertEquals(value, numberOfQueries)
 202
 203     def testCacheExpiration(self):
 204         """
 205         Cache: Cache expiration
 206
 207         dnsdist is configured to cache entries, we are sending one request
 208         (cache miss) with a very short TTL, checking that the next requests
 209         are cached. Then we wait for the TTL to expire, check that the
 210         next request is a miss but the following one a hit.
 211         """
 212         ttl = 2
 213         misses = 0
 214         name = 'cacheexpiration.cache.tests.powerdns.com.'
 215         query = dns.message.make_query(name, 'AAAA', 'IN')
 216         response = dns.message.make_response(query)
 217         rrset = dns.rrset.from_text(name,
 218                                     ttl,
 219                                     dns.rdataclass.IN,
 220                                     dns.rdatatype.AAAA,
 221                                     '::1')
 222         response.answer.append(rrset)
 223
 224         # first query to fill the cache
 225         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 226         self.assertTrue(receivedQuery)
 227         self.assertTrue(receivedResponse)
 228         receivedQuery.id = query.id
 229         self.assertEquals(query, receivedQuery)
 230         self.assertEquals(receivedResponse, response)
 231         misses += 1
 232
 233         # next queries should hit the cache
 234         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 235         self.assertEquals(receivedResponse, response)
 236
 237         # now we wait a bit for the cache entry to expire
 238         time.sleep(ttl + 1)
 239
 240         # next query should be a miss, fill the cache again
 241         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 242         self.assertTrue(receivedQuery)
 243         self.assertTrue(receivedResponse)
 244         receivedQuery.id = query.id
 245         self.assertEquals(query, receivedQuery)
 246         self.assertEquals(receivedResponse, response)
 247         misses += 1
 248
 249         # following queries should hit the cache again
 250         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 251         self.assertEquals(receivedResponse, response)
 252
 253         total = 0
 254         for key in self._responsesCounter:
 255             total += self._responsesCounter[key]
 256
 257         self.assertEquals(total, misses)
 258
 259     def testCacheExpirationDifferentSets(self):
 260         """
 261         Cache: Cache expiration with different sets
 262
 263         dnsdist is configured to cache entries, we are sending one request
 264         (cache miss) whose response has a long and a very short TTL,
 265         checking that the next requests are cached. Then we wait for the
 266         short TTL to expire, check that the
 267         next request is a miss but the following one a hit.
 268         """
 269         ttl = 2
 270         misses = 0
 271         name = 'cacheexpirationdifferentsets.cache.tests.powerdns.com.'
 272         query = dns.message.make_query(name, 'AAAA', 'IN')
 273         response = dns.message.make_response(query)
 274         rrset = dns.rrset.from_text(name,
 275                                     ttl,
 276                                     dns.rdataclass.IN,
 277                                     dns.rdatatype.CNAME,
 278                                     'cname.cacheexpirationdifferentsets.cache.tests.powerdns.com.')
 279         response.answer.append(rrset)
 280         rrset = dns.rrset.from_text('cname.cacheexpirationdifferentsets.cache.tests.powerdns.com.',
 281                                     ttl + 3600,
 282                                     dns.rdataclass.IN,
 283                                     dns.rdatatype.A,
 284                                     '192.2.0.1')
 285         response.additional.append(rrset)
 286
 287         # first query to fill the cache
 288         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 289         self.assertTrue(receivedQuery)
 290         self.assertTrue(receivedResponse)
 291         receivedQuery.id = query.id
 292         self.assertEquals(query, receivedQuery)
 293         self.assertEquals(receivedResponse, response)
 294         misses += 1
 295
 296         # next queries should hit the cache
 297         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 298         self.assertEquals(receivedResponse, response)
 299
 300         # now we wait a bit for the cache entry to expire
 301         time.sleep(ttl + 1)
 302
 303         # next query should be a miss, fill the cache again
 304         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 305         self.assertTrue(receivedQuery)
 306         self.assertTrue(receivedResponse)
 307         receivedQuery.id = query.id
 308         self.assertEquals(query, receivedQuery)
 309         self.assertEquals(receivedResponse, response)
 310         misses += 1
 311
 312         # following queries should hit the cache again
 313         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 314         self.assertEquals(receivedResponse, response)
 315
 316         total = 0
 317         for key in self._responsesCounter:
 318             total += self._responsesCounter[key]
 319
 320         self.assertEquals(total, misses)
 321
 322     def testCacheDecreaseTTL(self):
 323         """
 324         Cache: Cache decreases TTL
 325
 326         dnsdist is configured to cache entries, we are sending one request
 327         (cache miss) and verify that the cache hits have a decreasing TTL.
 328         """
 329         ttl = 600
 330         misses = 0
 331         name = 'cachedecreasettl.cache.tests.powerdns.com.'
 332         query = dns.message.make_query(name, 'AAAA', 'IN')
 333         response = dns.message.make_response(query)
 334         rrset = dns.rrset.from_text(name,
 335                                     ttl,
 336                                     dns.rdataclass.IN,
 337                                     dns.rdatatype.AAAA,
 338                                     '::1')
 339         response.answer.append(rrset)
 340
 341         # first query to fill the cache
 342         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 343         self.assertTrue(receivedQuery)
 344         self.assertTrue(receivedResponse)
 345         receivedQuery.id = query.id
 346         self.assertEquals(query, receivedQuery)
 347         self.assertEquals(receivedResponse, response)
 348         misses += 1
 349
 350         # next queries should hit the cache
 351         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 352         self.assertEquals(receivedResponse, response)
 353         for an in receivedResponse.answer:
 354             self.assertTrue(an.ttl <= ttl)
 355
 356         # now we wait a bit for the TTL to decrease
 357         time.sleep(1)
 358
 359         # next queries should hit the cache
 360         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 361         self.assertEquals(receivedResponse, response)
 362         for an in receivedResponse.answer:
 363             self.assertTrue(an.ttl < ttl)
 364
 365         total = 0
 366         for key in self._responsesCounter:
 367             total += self._responsesCounter[key]
 368
 369         self.assertEquals(total, misses)
 370
 371     def testCacheDifferentCase(self):
 372         """
 373         Cache: Cache matches different case
 374
 375         dnsdist is configured to cache entries, we are sending one request
 376         (cache miss) and verify that the same one with a different case
 377         matches.
 378         """
 379         ttl = 600
 380         name = 'cachedifferentcase.cache.tests.powerdns.com.'
 381         differentCaseName = 'CacheDifferentCASE.cache.tests.powerdns.com.'
 382         query = dns.message.make_query(name, 'AAAA', 'IN')
 383         differentCaseQuery = dns.message.make_query(differentCaseName, 'AAAA', 'IN')
 384         response = dns.message.make_response(query)
 385         differentCaseResponse = dns.message.make_response(differentCaseQuery)
 386         rrset = dns.rrset.from_text(name,
 387                                     ttl,
 388                                     dns.rdataclass.IN,
 389                                     dns.rdatatype.AAAA,
 390                                     '::1')
 391         response.answer.append(rrset)
 392         differentCaseResponse.answer.append(rrset)
 393
 394         # first query to fill the cache
 395         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 396         self.assertTrue(receivedQuery)
 397         self.assertTrue(receivedResponse)
 398         receivedQuery.id = query.id
 399         self.assertEquals(query, receivedQuery)
 400         self.assertEquals(receivedResponse, response)
 401
 402         # different case query should still hit the cache
 403         (_, receivedResponse) = self.sendUDPQuery(differentCaseQuery, response=None, useQueue=False)
 404         self.assertEquals(receivedResponse, differentCaseResponse)
 405
 406
 407 class TestTempFailureCacheTTLAction(DNSDistTest):
 408
 409     _config_template = """
 410     -- maxTTL=86400, minTTL=1
 411     pc = newPacketCache(100, 86400, 1)
 412     getPool(""):setCache(pc)
 413     addAction("servfail.cache.tests.powerdns.com.", TempFailureCacheTTLAction(1))
 414     newServer{address="127.0.0.1:%d"}
 415     """
 416
 417     def testTempFailureCacheTTLAction(self):
 418         """
 419         Cache: When a TempFailure TTL is set, it should be honored
 420
 421         dnsdist is configured to cache packets, plus a specific qname is
 422         set up with a lower TempFailure Cache TTL. we are sending one request
 423         (cache miss) and verify that the cache is hit for the following query,
 424         but the TTL then expires before the larger "good" packetcache TTL.
 425         """
 426         name = 'servfail.cache.tests.powerdns.com.'
 427         query = dns.message.make_query(name, 'AAAA', 'IN')
 428         response = dns.message.make_response(query)
 429         response.set_rcode(dns.rcode.SERVFAIL)
 430
 431         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 432         self.assertTrue(receivedQuery)
 433         self.assertTrue(receivedResponse)
 434         receivedQuery.id = query.id
 435         self.assertEquals(query, receivedQuery)
 436         self.assertEquals(receivedResponse, response)
 437
 438         # next query should hit the cache
 439         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 440         self.assertFalse(receivedQuery)
 441         self.assertTrue(receivedResponse)
 442         self.assertEquals(receivedResponse, response)
 443
 444         # now we wait a bit for the Failure-Cache TTL to expire
 445         time.sleep(2)
 446
 447         # next query should NOT hit the cache
 448         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 449         self.assertTrue(receivedQuery)
 450         self.assertTrue(receivedResponse)
 451         self.assertEquals(receivedResponse, response)
 452
 453
 454 class TestCachingWithExistingEDNS(DNSDistTest):
 455
 456     _config_template = """
 457     -- maxTTL=86400, minTTL=1
 458     pc = newPacketCache(5, 86400, 1)
 459     getPool(""):setCache(pc)
 460     newServer{address="127.0.0.1:%d"}
 461     """
 462     def testCacheWithEDNS(self):
 463         """
 464         Cache: Cache should not match different EDNS value
 465
 466         dnsdist is configured to cache entries, we are sending one request
 467         (cache miss) and verify that the same one with a different EDNS UDP
 468         Payload size is not served from the cache.
 469         """
 470         misses = 0
 471         name = 'cachedifferentedns.cache.tests.powerdns.com.'
 472         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=512)
 473         response = dns.message.make_response(query)
 474         rrset = dns.rrset.from_text(name,
 475                                     3600,
 476                                     dns.rdataclass.IN,
 477                                     dns.rdatatype.A,
 478                                     '127.0.0.1')
 479         response.answer.append(rrset)
 480
 481         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 482         self.assertTrue(receivedQuery)
 483         self.assertTrue(receivedResponse)
 484         receivedQuery.id = query.id
 485         self.assertEquals(query, receivedQuery)
 486         self.assertEquals(response, receivedResponse)
 487         misses += 1
 488
 489         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096)
 490         response = dns.message.make_response(query)
 491         rrset = dns.rrset.from_text(name,
 492                                     3600,
 493                                     dns.rdataclass.IN,
 494                                     dns.rdatatype.A,
 495                                     '127.0.0.1')
 496         response.answer.append(rrset)
 497
 498         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 499         self.assertTrue(receivedQuery)
 500         self.assertTrue(receivedResponse)
 501         receivedQuery.id = query.id
 502         self.assertEquals(query, receivedQuery)
 503         self.assertEquals(response, receivedResponse)
 504         misses += 1
 505
 506         total = 0
 507         for key in self._responsesCounter:
 508             total += self._responsesCounter[key]
 509
 510         self.assertEquals(total, misses)
 511
 512 class TestCachingCacheFull(DNSDistTest):
 513
 514     _config_template = """
 515     -- maxTTL=86400, minTTL=1
 516     pc = newPacketCache(1, 86400, 1)
 517     getPool(""):setCache(pc)
 518     newServer{address="127.0.0.1:%d"}
 519     """
 520     def testCacheFull(self):
 521         """
 522         Cache: No new entries are cached when the cache is full
 523
 524         """
 525         misses = 0
 526         name = 'cachenotfullyet.cache.tests.powerdns.com.'
 527         query = dns.message.make_query(name, 'A', 'IN')
 528         response = dns.message.make_response(query)
 529         rrset = dns.rrset.from_text(name,
 530                                     3600,
 531                                     dns.rdataclass.IN,
 532                                     dns.rdatatype.A,
 533                                     '127.0.0.1')
 534         response.answer.append(rrset)
 535
 536         # Miss
 537         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 538         self.assertTrue(receivedQuery)
 539         self.assertTrue(receivedResponse)
 540         receivedQuery.id = query.id
 541         self.assertEquals(query, receivedQuery)
 542         self.assertEquals(response, receivedResponse)
 543         misses += 1
 544
 545         # next queries should hit the cache
 546         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 547         self.assertEquals(receivedResponse, response)
 548
 549         # ok, now the cache is full, send another query
 550         name = 'cachefull.cache.tests.powerdns.com.'
 551         query = dns.message.make_query(name, 'AAAA', 'IN')
 552         response = dns.message.make_response(query)
 553         rrset = dns.rrset.from_text(name,
 554                                     3600,
 555                                     dns.rdataclass.IN,
 556                                     dns.rdatatype.AAAA,
 557                                     '::1')
 558         response.answer.append(rrset)
 559
 560         # Miss
 561         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 562         self.assertTrue(receivedQuery)
 563         self.assertTrue(receivedResponse)
 564         receivedQuery.id = query.id
 565         self.assertEquals(query, receivedQuery)
 566         self.assertEquals(response, receivedResponse)
 567         misses += 1
 568
 569         # next queries should NOT hit the cache
 570         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 571         self.assertTrue(receivedQuery)
 572         self.assertTrue(receivedResponse)
 573         receivedQuery.id = query.id
 574         self.assertEquals(query, receivedQuery)
 575         self.assertEquals(response, receivedResponse)
 576         misses += 1
 577
 578         total = 0
 579         for key in self._responsesCounter:
 580             total += self._responsesCounter[key]
 581
 582         self.assertEquals(total, misses)
 583
 584 class TestCachingNoStale(DNSDistTest):
 585
 586     _consoleKey = DNSDistTest.generateConsoleKey()
 587     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 588     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
 589     _config_template = """
 590     -- maxTTL=86400, minTTL=1
 591     pc = newPacketCache(100, 86400, 1)
 592     getPool(""):setCache(pc)
 593     setKey("%s")
 594     controlSocket("127.0.0.1:%d")
 595     newServer{address="127.0.0.1:%d"}
 596     """
 597     def testCacheNoStale(self):
 598         """
 599         Cache: Cache entry, set backend down, we should not get a stale entry
 600
 601         """
 602         ttl = 2
 603         name = 'nostale.cache.tests.powerdns.com.'
 604         query = dns.message.make_query(name, 'A', 'IN')
 605         response = dns.message.make_response(query)
 606         rrset = dns.rrset.from_text(name,
 607                                     1,
 608                                     dns.rdataclass.IN,
 609                                     dns.rdatatype.A,
 610                                     '127.0.0.1')
 611         response.answer.append(rrset)
 612
 613         # Miss
 614         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 615         self.assertTrue(receivedQuery)
 616         self.assertTrue(receivedResponse)
 617         receivedQuery.id = query.id
 618         self.assertEquals(query, receivedQuery)
 619         self.assertEquals(response, receivedResponse)
 620
 621         # next queries should hit the cache
 622         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 623         self.assertEquals(receivedResponse, response)
 624
 625         # ok, we mark the backend as down
 626         self.sendConsoleCommand("getServer(0):setDown()")
 627         # and we wait for the entry to expire
 628         time.sleep(ttl + 1)
 629
 630         # we should NOT get a cached, stale, entry
 631         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 632         self.assertEquals(receivedResponse, None)
 633
 634
 635 class TestCachingStale(DNSDistTest):
 636
 637     _consoleKey = DNSDistTest.generateConsoleKey()
 638     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 639     _staleCacheTTL = 60
 640     _config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
 641     _config_template = """
 642     -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=XX
 643     pc = newPacketCache(100, 86400, 1, 0, %d)
 644     getPool(""):setCache(pc)
 645     setStaleCacheEntriesTTL(600)
 646     setKey("%s")
 647     controlSocket("127.0.0.1:%d")
 648     newServer{address="127.0.0.1:%d"}
 649     """
 650     def testCacheStale(self):
 651         """
 652         Cache: Cache entry, set backend down, get stale entry
 653
 654         """
 655         misses = 0
 656         ttl = 2
 657         name = 'stale.cache.tests.powerdns.com.'
 658         query = dns.message.make_query(name, 'A', 'IN')
 659         response = dns.message.make_response(query)
 660         rrset = dns.rrset.from_text(name,
 661                                     ttl,
 662                                     dns.rdataclass.IN,
 663                                     dns.rdatatype.A,
 664                                     '127.0.0.1')
 665         response.answer.append(rrset)
 666
 667         # Miss
 668         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 669         self.assertTrue(receivedQuery)
 670         self.assertTrue(receivedResponse)
 671         receivedQuery.id = query.id
 672         self.assertEquals(query, receivedQuery)
 673         self.assertEquals(response, receivedResponse)
 674         misses += 1
 675
 676         # next queries should hit the cache
 677         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 678         self.assertEquals(receivedResponse, response)
 679
 680         # ok, we mark the backend as down
 681         self.sendConsoleCommand("getServer(0):setDown()")
 682         # and we wait for the entry to expire
 683         time.sleep(ttl + 1)
 684
 685         # we should get a cached, stale, entry
 686         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 687         self.assertEquals(receivedResponse, response)
 688         for an in receivedResponse.answer:
 689             self.assertEquals(an.ttl, self._staleCacheTTL)
 690
 691         total = 0
 692         for key in self._responsesCounter:
 693             total += self._responsesCounter[key]
 694
 695         self.assertEquals(total, misses)
 696
 697 class TestCachingStaleExpunged(DNSDistTest):
 698
 699     _consoleKey = DNSDistTest.generateConsoleKey()
 700     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 701     _staleCacheTTL = 60
 702     _config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
 703     _config_template = """
 704     -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=XX
 705     pc = newPacketCache(100, 86400, 1, 0, %d)
 706     getPool(""):setCache(pc)
 707     setStaleCacheEntriesTTL(600)
 708     -- try to remove all expired entries
 709     setCacheCleaningPercentage(100)
 710     -- clean the cache every second
 711     setCacheCleaningDelay(1)
 712     setKey("%s")
 713     controlSocket("127.0.0.1:%d")
 714     newServer{address="127.0.0.1:%d"}
 715     """
 716     def testCacheStale(self):
 717         """
 718         Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, get no entry
 719         """
 720         misses = 0
 721         drops = 0
 722         ttl = 2
 723         name = 'stale-but-expunged.cache.tests.powerdns.com.'
 724         query = dns.message.make_query(name, 'A', 'IN')
 725         response = dns.message.make_response(query)
 726         rrset = dns.rrset.from_text(name,
 727                                     ttl,
 728                                     dns.rdataclass.IN,
 729                                     dns.rdatatype.A,
 730                                     '127.0.0.1')
 731         response.answer.append(rrset)
 732
 733         # Miss
 734         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 735         self.assertTrue(receivedQuery)
 736         self.assertTrue(receivedResponse)
 737         receivedQuery.id = query.id
 738         self.assertEquals(query, receivedQuery)
 739         self.assertEquals(response, receivedResponse)
 740         misses += 1
 741         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
 742
 743         # next queries should hit the cache
 744         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 745         self.assertEquals(receivedResponse, response)
 746         # the cache should have one entry
 747         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
 748         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
 749
 750         # ok, we mark the backend as down
 751         self.sendConsoleCommand("getServer(0):setDown()")
 752         # and we wait for the entry to expire
 753         time.sleep(ttl + 1)
 754         # wait a bit more to be sure that the cache cleaning algo has been run
 755         time.sleep(1)
 756         # the cache should be empty now
 757         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 0)
 758
 759         # we should get a DROP (backend is down, nothing in the cache anymore)
 760         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 761         self.assertEquals(receivedResponse, None)
 762         drops += 1
 763
 764         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
 765         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
 766
 767         total = 0
 768         for key in self._responsesCounter:
 769             total += self._responsesCounter[key]
 770
 771         self.assertEquals(total, misses)
 772
 773 class TestCachingStaleExpungePrevented(DNSDistTest):
 774
 775     _consoleKey = DNSDistTest.generateConsoleKey()
 776     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 777     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
 778     _config_template = """
 779     -- maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, ecsParsing=false, keepStaleData=true
 780     pc = newPacketCache(100, 86400, 1, 0, 60, false, 1, true, 3600, false, { keepStaleData=true})
 781     getPool(""):setCache(pc)
 782     setStaleCacheEntriesTTL(600)
 783     -- try to remove all expired entries
 784     setCacheCleaningPercentage(100)
 785     -- clean the cache every second
 786     setCacheCleaningDelay(1)
 787     setKey("%s")
 788     controlSocket("127.0.0.1:%d")
 789     newServer{address="127.0.0.1:%d"}
 790     """
 791     def testCacheStale(self):
 792         """
 793         Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, still get a cache HIT because the stale entry was not removed
 794         """
 795         misses = 0
 796         ttl = 2
 797         name = 'stale-not-expunged.cache.tests.powerdns.com.'
 798         query = dns.message.make_query(name, 'A', 'IN')
 799         response = dns.message.make_response(query)
 800         rrset = dns.rrset.from_text(name,
 801                                     ttl,
 802                                     dns.rdataclass.IN,
 803                                     dns.rdatatype.A,
 804                                     '127.0.0.1')
 805         response.answer.append(rrset)
 806
 807         # Miss
 808         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 809         self.assertTrue(receivedQuery)
 810         self.assertTrue(receivedResponse)
 811         receivedQuery.id = query.id
 812         self.assertEquals(query, receivedQuery)
 813         self.assertEquals(response, receivedResponse)
 814         misses += 1
 815         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
 816
 817         # next queries should hit the cache
 818         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 819         self.assertEquals(receivedResponse, response)
 820         # the cache should have one entry
 821         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
 822         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
 823
 824         # ok, we mark the backend as down
 825         self.sendConsoleCommand("getServer(0):setDown()")
 826         # and we wait for the entry to expire
 827         time.sleep(ttl + 1)
 828         # wait a bit more to be sure that the cache cleaning algo has been run
 829         time.sleep(1)
 830         # the cache should NOT be empty because the removal of the expired entry should have been prevented
 831         # since all backends for this pool are down
 832         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
 833
 834         # we should get a HIT
 835         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 836         self.assertEquals(receivedResponse, response)
 837
 838         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
 839         self.assertEquals(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 2)
 840
 841         total = 0
 842         for key in self._responsesCounter:
 843             total += self._responsesCounter[key]
 844
 845         self.assertEquals(total, misses)
 846
 847 class TestCacheManagement(DNSDistTest):
 848
 849     _consoleKey = DNSDistTest.generateConsoleKey()
 850     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
 851     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
 852     _config_template = """
 853     -- maxTTL=86400, minTTL=1
 854     pc = newPacketCache(100, 86400, 1)
 855     getPool(""):setCache(pc)
 856     setKey("%s")
 857     controlSocket("127.0.0.1:%d")
 858     newServer{address="127.0.0.1:%d"}
 859     """
 860     def testCacheExpunge(self):
 861         """
 862         Cache: Expunge
 863
 864         """
 865         misses = 0
 866         ttl = 600
 867         name = 'expunge.cache.tests.powerdns.com.'
 868         query = dns.message.make_query(name, 'A', 'IN')
 869         response = dns.message.make_response(query)
 870         rrset = dns.rrset.from_text(name,
 871                                     ttl,
 872                                     dns.rdataclass.IN,
 873                                     dns.rdatatype.A,
 874                                     '127.0.0.1')
 875         response.answer.append(rrset)
 876
 877         # Miss
 878         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 879         self.assertTrue(receivedQuery)
 880         self.assertTrue(receivedResponse)
 881         receivedQuery.id = query.id
 882         self.assertEquals(query, receivedQuery)
 883         self.assertEquals(response, receivedResponse)
 884         misses += 1
 885
 886         # next queries should hit the cache
 887         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 888         self.assertEquals(receivedResponse, response)
 889
 890         # remove cached entries
 891         self.sendConsoleCommand("getPool(\"\"):getCache():expunge(0)")
 892
 893         # Miss
 894         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 895         self.assertTrue(receivedQuery)
 896         self.assertTrue(receivedResponse)
 897         receivedQuery.id = query.id
 898         self.assertEquals(query, receivedQuery)
 899         self.assertEquals(response, receivedResponse)
 900         misses += 1
 901
 902         # next queries should hit the cache again
 903         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 904         self.assertEquals(receivedResponse, response)
 905
 906         total = 0
 907         for key in self._responsesCounter:
 908             total += self._responsesCounter[key]
 909
 910         self.assertEquals(total, misses)
 911
 912     def testCacheExpungeByName(self):
 913         """
 914         Cache: Expunge by name
 915
 916         """
 917         misses = 0
 918         ttl = 600
 919         name = 'expungebyname.cache.tests.powerdns.com.'
 920         query = dns.message.make_query(name, 'A', 'IN')
 921         response = dns.message.make_response(query)
 922         rrset = dns.rrset.from_text(name,
 923                                     ttl,
 924                                     dns.rdataclass.IN,
 925                                     dns.rdatatype.A,
 926                                     '127.0.0.1')
 927         response.answer.append(rrset)
 928
 929         name2 = 'expungebynameother.cache.tests.powerdns.com.'
 930         query2 = dns.message.make_query(name2, 'A', 'IN')
 931         response2 = dns.message.make_response(query2)
 932         rrset2 = dns.rrset.from_text(name2,
 933                                      ttl,
 934                                      dns.rdataclass.IN,
 935                                      dns.rdatatype.A,
 936                                      '127.0.0.1')
 937         response2.answer.append(rrset2)
 938
 939         # Miss
 940         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 941         self.assertTrue(receivedQuery)
 942         self.assertTrue(receivedResponse)
 943         receivedQuery.id = query.id
 944         self.assertEquals(query, receivedQuery)
 945         self.assertEquals(response, receivedResponse)
 946         misses += 1
 947
 948         # next queries should hit the cache
 949         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 950         self.assertEquals(receivedResponse, response)
 951
 952         # cache another entry
 953         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
 954         self.assertTrue(receivedQuery)
 955         self.assertTrue(receivedResponse)
 956         receivedQuery.id = query2.id
 957         self.assertEquals(query2, receivedQuery)
 958         self.assertEquals(response2, receivedResponse)
 959         misses += 1
 960
 961         # queries for name and name 2 should hit the cache
 962         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 963         self.assertEquals(receivedResponse, response)
 964
 965         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
 966         self.assertEquals(receivedResponse, response2)
 967
 968         # remove cached entries from name
 969         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"" + name + "\"))")
 970
 971         # Miss for name
 972         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 973         self.assertTrue(receivedQuery)
 974         self.assertTrue(receivedResponse)
 975         receivedQuery.id = query.id
 976         self.assertEquals(query, receivedQuery)
 977         self.assertEquals(response, receivedResponse)
 978         misses += 1
 979
 980         # next queries for name should hit the cache again
 981         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 982         self.assertEquals(receivedResponse, response)
 983
 984         # queries for name2 should still hit the cache
 985         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
 986         self.assertEquals(receivedResponse, response2)
 987
 988         total = 0
 989         for key in self._responsesCounter:
 990             total += self._responsesCounter[key]
 991
 992         self.assertEquals(total, misses)
 993
 994     def testCacheExpungeByNameAndType(self):
 995         """
 996         Cache: Expunge by name and type
 997
 998         """
 999         misses = 0
1000         ttl = 600
1001         name = 'expungebynameandtype.cache.tests.powerdns.com.'
1002         query = dns.message.make_query(name, 'A', 'IN')
1003         response = dns.message.make_response(query)
1004         rrset = dns.rrset.from_text(name,
1005                                     ttl,
1006                                     dns.rdataclass.IN,
1007                                     dns.rdatatype.A,
1008                                     '127.0.0.1')
1009         response.answer.append(rrset)
1010
1011         query2 = dns.message.make_query(name, 'AAAA', 'IN')
1012         response2 = dns.message.make_response(query2)
1013         rrset2 = dns.rrset.from_text(name,
1014                                      ttl,
1015                                      dns.rdataclass.IN,
1016                                      dns.rdatatype.AAAA,
1017                                      '::1')
1018         response2.answer.append(rrset2)
1019
1020         # Miss
1021         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1022         self.assertTrue(receivedQuery)
1023         self.assertTrue(receivedResponse)
1024         receivedQuery.id = query.id
1025         self.assertEquals(query, receivedQuery)
1026         self.assertEquals(response, receivedResponse)
1027         misses += 1
1028
1029         # next queries should hit the cache
1030         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1031         self.assertEquals(receivedResponse, response)
1032
1033         # cache another entry
1034         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1035         self.assertTrue(receivedQuery)
1036         self.assertTrue(receivedResponse)
1037         receivedQuery.id = query2.id
1038         self.assertEquals(query2, receivedQuery)
1039         self.assertEquals(response2, receivedResponse)
1040         misses += 1
1041
1042         # queries for name A and AAAA should hit the cache
1043         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1044         self.assertEquals(receivedResponse, response)
1045
1046         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1047         self.assertEquals(receivedResponse, response2)
1048
1049         # remove cached entries from name A
1050         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"" + name + "\"), dnsdist.A)")
1051
1052         # Miss for name A
1053         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1054         self.assertTrue(receivedQuery)
1055         self.assertTrue(receivedResponse)
1056         receivedQuery.id = query.id
1057         self.assertEquals(query, receivedQuery)
1058         self.assertEquals(response, receivedResponse)
1059         misses += 1
1060
1061         # next queries for name A should hit the cache again
1062         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1063         self.assertEquals(receivedResponse, response)
1064
1065         # queries for name AAAA should still hit the cache
1066         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1067         self.assertEquals(receivedResponse, response2)
1068
1069         total = 0
1070         for key in self._responsesCounter:
1071             total += self._responsesCounter[key]
1072         self.assertEquals(total, misses)
1073
1074     def testCacheExpungeByNameAndSuffix(self):
1075         """
1076         Cache: Expunge by name
1077
1078         """
1079         misses = 0
1080         ttl = 600
1081         name = 'expungebyname.suffix.cache.tests.powerdns.com.'
1082         query = dns.message.make_query(name, 'A', 'IN')
1083         response = dns.message.make_response(query)
1084         rrset = dns.rrset.from_text(name,
1085                                     ttl,
1086                                     dns.rdataclass.IN,
1087                                     dns.rdatatype.A,
1088                                     '127.0.0.1')
1089         response.answer.append(rrset)
1090
1091         name2 = 'expungebyname.suffixother.cache.tests.powerdns.com.'
1092         query2 = dns.message.make_query(name2, 'A', 'IN')
1093         response2 = dns.message.make_response(query2)
1094         rrset2 = dns.rrset.from_text(name2,
1095                                      ttl,
1096                                      dns.rdataclass.IN,
1097                                      dns.rdatatype.A,
1098                                      '127.0.0.1')
1099         response2.answer.append(rrset2)
1100
1101         # Miss
1102         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1103         self.assertTrue(receivedQuery)
1104         self.assertTrue(receivedResponse)
1105         receivedQuery.id = query.id
1106         self.assertEquals(query, receivedQuery)
1107         self.assertEquals(response, receivedResponse)
1108         misses += 1
1109
1110         # next queries should hit the cache
1111         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1112         self.assertEquals(receivedResponse, response)
1113
1114         # cache another entry
1115         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1116         self.assertTrue(receivedQuery)
1117         self.assertTrue(receivedResponse)
1118         receivedQuery.id = query2.id
1119         self.assertEquals(query2, receivedQuery)
1120         self.assertEquals(response2, receivedResponse)
1121         misses += 1
1122
1123         # queries for name and name 2 should hit the cache
1124         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1125         self.assertEquals(receivedResponse, response)
1126
1127         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1128         self.assertEquals(receivedResponse, response2)
1129
1130         # remove cached entries from name
1131         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"suffix.cache.tests.powerdns.com.\"), dnsdist.ANY, true)")
1132
1133         # Miss for name
1134         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1135         self.assertTrue(receivedQuery)
1136         self.assertTrue(receivedResponse)
1137         receivedQuery.id = query.id
1138         self.assertEquals(query, receivedQuery)
1139         self.assertEquals(response, receivedResponse)
1140         misses += 1
1141
1142         # next queries for name should hit the cache again
1143         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1144         self.assertEquals(receivedResponse, response)
1145
1146         # queries for name2 should still hit the cache
1147         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1148         self.assertEquals(receivedResponse, response2)
1149
1150         total = 0
1151         for key in self._responsesCounter:
1152             total += self._responsesCounter[key]
1153
1154         self.assertEquals(total, misses)
1155
1156     def testCacheExpungeByNameAndTypeAndSuffix(self):
1157         """
1158         Cache: Expunge by name and type
1159
1160         """
1161         misses = 0
1162         ttl = 600
1163         name = 'expungebynameandtype.suffixtype.cache.tests.powerdns.com.'
1164         query = dns.message.make_query(name, 'A', 'IN')
1165         response = dns.message.make_response(query)
1166         rrset = dns.rrset.from_text(name,
1167                                     ttl,
1168                                     dns.rdataclass.IN,
1169                                     dns.rdatatype.A,
1170                                     '127.0.0.1')
1171         response.answer.append(rrset)
1172
1173         query2 = dns.message.make_query(name, 'AAAA', 'IN')
1174         response2 = dns.message.make_response(query2)
1175         rrset2 = dns.rrset.from_text(name,
1176                                      ttl,
1177                                      dns.rdataclass.IN,
1178                                      dns.rdatatype.AAAA,
1179                                      '::1')
1180         response2.answer.append(rrset2)
1181
1182         # Miss
1183         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1184         self.assertTrue(receivedQuery)
1185         self.assertTrue(receivedResponse)
1186         receivedQuery.id = query.id
1187         self.assertEquals(query, receivedQuery)
1188         self.assertEquals(response, receivedResponse)
1189         misses += 1
1190
1191         # next queries should hit the cache
1192         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1193         self.assertEquals(receivedResponse, response)
1194
1195         # cache another entry
1196         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1197         self.assertTrue(receivedQuery)
1198         self.assertTrue(receivedResponse)
1199         receivedQuery.id = query2.id
1200         self.assertEquals(query2, receivedQuery)
1201         self.assertEquals(response2, receivedResponse)
1202         misses += 1
1203
1204         # queries for name A and AAAA should hit the cache
1205         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1206         self.assertEquals(receivedResponse, response)
1207
1208         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1209         self.assertEquals(receivedResponse, response2)
1210
1211         # remove cached entries from name A
1212         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"suffixtype.cache.tests.powerdns.com.\"), dnsdist.A, true)")
1213
1214         # Miss for name A
1215         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1216         self.assertTrue(receivedQuery)
1217         self.assertTrue(receivedResponse)
1218         receivedQuery.id = query.id
1219         self.assertEquals(query, receivedQuery)
1220         self.assertEquals(response, receivedResponse)
1221         misses += 1
1222
1223         # next queries for name A should hit the cache again
1224         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1225         self.assertEquals(receivedResponse, response)
1226
1227         # queries for name AAAA should still hit the cache
1228         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1229         self.assertEquals(receivedResponse, response2)
1230
1231         total = 0
1232         for key in self._responsesCounter:
1233             total += self._responsesCounter[key]
1234         self.assertEquals(total, misses)
1235
1236 class TestCachingTTL(DNSDistTest):
1237
1238     _maxCacheTTL = 86400
1239     _minCacheTTL = 600
1240     _config_params = ['_maxCacheTTL', '_minCacheTTL', '_testServerPort']
1241     _config_template = """
1242     -- maxTTL=XX, minTTL=XX
1243     pc = newPacketCache(1000, %d, %d)
1244     getPool(""):setCache(pc)
1245     newServer{address="127.0.0.1:%d"}
1246     """
1247     def testCacheShortTTL(self):
1248         """
1249         Cache: Entries with a TTL shorter than minTTL
1250
1251         """
1252         misses = 0
1253         ttl = 60
1254         name = 'ttltooshort.cache.tests.powerdns.com.'
1255         query = dns.message.make_query(name, 'A', 'IN')
1256         response = dns.message.make_response(query)
1257         rrset = dns.rrset.from_text(name,
1258                                     ttl,
1259                                     dns.rdataclass.IN,
1260                                     dns.rdatatype.A,
1261                                     '127.0.0.1')
1262         response.answer.append(rrset)
1263
1264         # Miss
1265         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1266         self.assertTrue(receivedQuery)
1267         self.assertTrue(receivedResponse)
1268         receivedQuery.id = query.id
1269         self.assertEquals(query, receivedQuery)
1270         self.assertEquals(response, receivedResponse)
1271         for an in receivedResponse.answer:
1272             self.assertEquals(an.ttl, ttl)
1273         misses += 1
1274
1275         # We should not have been cached
1276         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1277         self.assertTrue(receivedQuery)
1278         self.assertTrue(receivedResponse)
1279         receivedQuery.id = query.id
1280         self.assertEquals(query, receivedQuery)
1281         self.assertEquals(response, receivedResponse)
1282         for an in receivedResponse.answer:
1283             self.assertEquals(an.ttl, ttl)
1284         misses += 1
1285
1286         total = 0
1287         for key in self._responsesCounter:
1288             total += self._responsesCounter[key]
1289
1290         self.assertEquals(total, misses)
1291
1292     def testCacheNXWithNoRR(self):
1293         """
1294         Cache: NX with no RR
1295
1296         """
1297         misses = 0
1298         name = 'nxwithnorr.cache.tests.powerdns.com.'
1299         query = dns.message.make_query(name, 'A', 'IN')
1300         response = dns.message.make_response(query)
1301         response.set_rcode(dns.rcode.NXDOMAIN)
1302
1303         # Miss
1304         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1305         self.assertTrue(receivedQuery)
1306         self.assertTrue(receivedResponse)
1307         receivedQuery.id = query.id
1308         self.assertEquals(query, receivedQuery)
1309         self.assertEquals(response, receivedResponse)
1310         misses += 1
1311
1312         # We should not have been cached
1313         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1314         self.assertTrue(receivedQuery)
1315         self.assertTrue(receivedResponse)
1316         receivedQuery.id = query.id
1317         self.assertEquals(query, receivedQuery)
1318         self.assertEquals(response, receivedResponse)
1319         misses += 1
1320
1321         total = 0
1322         for key in self._responsesCounter:
1323             total += self._responsesCounter[key]
1324
1325         self.assertEquals(total, misses)
1326
1327 class TestCachingLongTTL(DNSDistTest):
1328
1329     _maxCacheTTL = 2
1330     _config_params = ['_maxCacheTTL', '_testServerPort']
1331     _config_template = """
1332     -- maxTTL=XX
1333     pc = newPacketCache(1000, %d)
1334     getPool(""):setCache(pc)
1335     newServer{address="127.0.0.1:%d"}
1336     """
1337     def testCacheLongTTL(self):
1338         """
1339         Cache: Entries with a longer TTL than the maximum
1340
1341         """
1342         misses = 0
1343         ttl = 172800
1344         name = 'longttl.cache.tests.powerdns.com.'
1345         query = dns.message.make_query(name, 'A', 'IN')
1346         response = dns.message.make_response(query)
1347         rrset = dns.rrset.from_text(name,
1348                                     ttl,
1349                                     dns.rdataclass.IN,
1350                                     dns.rdatatype.A,
1351                                     '127.0.0.1')
1352         response.answer.append(rrset)
1353
1354         # Miss
1355         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1356         self.assertTrue(receivedQuery)
1357         self.assertTrue(receivedResponse)
1358         receivedQuery.id = query.id
1359         self.assertEquals(query, receivedQuery)
1360         self.assertEquals(response, receivedResponse)
1361         for an in receivedResponse.answer:
1362             self.assertEquals(an.ttl, ttl)
1363         misses += 1
1364
1365         # next queries should hit the cache
1366         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1367         self.assertEquals(receivedResponse, response)
1368         for an in receivedResponse.answer:
1369             self.assertTrue(an.ttl <= ttl)
1370
1371         time.sleep(self._maxCacheTTL + 1)
1372
1373         # we should not have cached for longer than max cache
1374         # so it should be a miss
1375         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1376         self.assertTrue(receivedQuery)
1377         self.assertTrue(receivedResponse)
1378         receivedQuery.id = query.id
1379         self.assertEquals(query, receivedQuery)
1380         self.assertEquals(response, receivedResponse)
1381         for an in receivedResponse.answer:
1382             self.assertEquals(an.ttl, ttl)
1383         misses += 1
1384
1385         total = 0
1386         for key in self._responsesCounter:
1387             total += self._responsesCounter[key]
1388
1389         self.assertEquals(total, misses)
1390
1391 class TestCachingFailureTTL(DNSDistTest):
1392
1393     _failureCacheTTL = 2
1394     _config_params = ['_failureCacheTTL', '_testServerPort']
1395     _config_template = """
1396     -- maxTTL=86400, minTTL=0, temporaryFailureTTL=XX, staleTTL=60
1397     pc = newPacketCache(1000, 86400, 0, %d, 60)
1398     getPool(""):setCache(pc)
1399     newServer{address="127.0.0.1:%d"}
1400     """
1401     def testCacheServFailTTL(self):
1402         """
1403         Cache: ServFail TTL
1404
1405         """
1406         misses = 0
1407         name = 'servfail.failure.cache.tests.powerdns.com.'
1408         query = dns.message.make_query(name, 'A', 'IN')
1409         response = dns.message.make_response(query)
1410         response.set_rcode(dns.rcode.SERVFAIL)
1411
1412         # Miss
1413         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1414         self.assertTrue(receivedQuery)
1415         self.assertTrue(receivedResponse)
1416         receivedQuery.id = query.id
1417         self.assertEquals(query, receivedQuery)
1418         self.assertEquals(response, receivedResponse)
1419         misses += 1
1420
1421         # next queries should hit the cache
1422         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1423         self.assertEquals(receivedResponse, response)
1424
1425         time.sleep(self._failureCacheTTL + 1)
1426
1427         # we should not have cached for longer than failure cache
1428         # so it should be a miss
1429         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1430         self.assertTrue(receivedQuery)
1431         self.assertTrue(receivedResponse)
1432         receivedQuery.id = query.id
1433         self.assertEquals(query, receivedQuery)
1434         self.assertEquals(response, receivedResponse)
1435         misses += 1
1436
1437         total = 0
1438         for key in self._responsesCounter:
1439             total += self._responsesCounter[key]
1440
1441         self.assertEquals(total, misses)
1442
1443     def testCacheRefusedTTL(self):
1444         """
1445         Cache: Refused TTL
1446
1447         """
1448         misses = 0
1449         name = 'refused.failure.cache.tests.powerdns.com.'
1450         query = dns.message.make_query(name, 'A', 'IN')
1451         response = dns.message.make_response(query)
1452         response.set_rcode(dns.rcode.REFUSED)
1453
1454         # Miss
1455         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1456         self.assertTrue(receivedQuery)
1457         self.assertTrue(receivedResponse)
1458         receivedQuery.id = query.id
1459         self.assertEquals(query, receivedQuery)
1460         self.assertEquals(response, receivedResponse)
1461         misses += 1
1462
1463         # next queries should hit the cache
1464         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1465         self.assertEquals(receivedResponse, response)
1466
1467         time.sleep(self._failureCacheTTL + 1)
1468
1469         # we should not have cached for longer than failure cache
1470         # so it should be a miss
1471         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1472         self.assertTrue(receivedQuery)
1473         self.assertTrue(receivedResponse)
1474         receivedQuery.id = query.id
1475         self.assertEquals(query, receivedQuery)
1476         self.assertEquals(response, receivedResponse)
1477         misses += 1
1478
1479         total = 0
1480         for key in self._responsesCounter:
1481             total += self._responsesCounter[key]
1482
1483         self.assertEquals(total, misses)
1484
1485     def testCacheHeaderOnlyRefusedTTL(self):
1486         """
1487         Cache: Header-Only Refused TTL
1488
1489         """
1490         misses = 0
1491         name = 'header-only-refused.failure.cache.tests.powerdns.com.'
1492         query = dns.message.make_query(name, 'A', 'IN')
1493         response = dns.message.make_response(query)
1494         response.set_rcode(dns.rcode.REFUSED)
1495         response.question = []
1496
1497         # Miss
1498         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1499         self.assertTrue(receivedQuery)
1500         self.assertTrue(receivedResponse)
1501         receivedQuery.id = query.id
1502         self.assertEquals(query, receivedQuery)
1503         self.assertEquals(response, receivedResponse)
1504         misses += 1
1505
1506         # next queries should hit the cache
1507         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1508         self.assertEquals(receivedResponse, response)
1509
1510         time.sleep(self._failureCacheTTL + 1)
1511
1512         # we should not have cached for longer than failure cache
1513         # so it should be a miss
1514         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1515         self.assertTrue(receivedQuery)
1516         self.assertTrue(receivedResponse)
1517         receivedQuery.id = query.id
1518         self.assertEquals(query, receivedQuery)
1519         self.assertEquals(response, receivedResponse)
1520         misses += 1
1521
1522         total = 0
1523         for key in self._responsesCounter:
1524             total += self._responsesCounter[key]
1525
1526         self.assertEquals(total, misses)
1527
1528 class TestCachingNegativeTTL(DNSDistTest):
1529
1530     _negCacheTTL = 1
1531     _config_params = ['_negCacheTTL', '_testServerPort']
1532     _config_template = """
1533     -- maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=XX
1534     pc = newPacketCache(1000, 86400, 0, 60, 60, false, 1, true, %d)
1535     getPool(""):setCache(pc)
1536     newServer{address="127.0.0.1:%d"}
1537     """
1538
1539     def testCacheNegativeTTLNXDomain(self):
1540         """
1541         Cache: Negative TTL on NXDOMAIN
1542
1543         """
1544         misses = 0
1545         name = 'nxdomain.negativettl.cache.tests.powerdns.com.'
1546         query = dns.message.make_query(name, 'A', 'IN')
1547         response = dns.message.make_response(query)
1548         response.set_rcode(dns.rcode.NXDOMAIN)
1549         soa = dns.rrset.from_text(name,
1550                                   60,
1551                                   dns.rdataclass.IN,
1552                                   dns.rdatatype.SOA,
1553                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
1554         response.authority.append(soa)
1555
1556         # Miss
1557         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1558         self.assertTrue(receivedQuery)
1559         self.assertTrue(receivedResponse)
1560         receivedQuery.id = query.id
1561         self.assertEquals(query, receivedQuery)
1562         self.assertEquals(response, receivedResponse)
1563         misses += 1
1564
1565         # next queries should hit the cache
1566         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1567         self.assertEquals(receivedResponse, response)
1568
1569         time.sleep(self._negCacheTTL + 1)
1570
1571         # we should not have cached for longer than the negative TTL
1572         # so it should be a miss
1573         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1574         self.assertTrue(receivedQuery)
1575         self.assertTrue(receivedResponse)
1576         receivedQuery.id = query.id
1577         self.assertEquals(query, receivedQuery)
1578         self.assertEquals(response, receivedResponse)
1579         misses += 1
1580
1581         total = 0
1582         for key in self._responsesCounter:
1583             total += self._responsesCounter[key]
1584
1585         self.assertEquals(total, misses)
1586
1587     def testCacheNegativeTTLNoData(self):
1588         """
1589         Cache: Negative TTL on NoData
1590
1591         """
1592         misses = 0
1593         name = 'nodata.negativettl.cache.tests.powerdns.com.'
1594         query = dns.message.make_query(name, 'A', 'IN')
1595         response = dns.message.make_response(query)
1596         response.set_rcode(dns.rcode.NOERROR)
1597         soa = dns.rrset.from_text(name,
1598                                   60,
1599                                   dns.rdataclass.IN,
1600                                   dns.rdatatype.SOA,
1601                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
1602         response.authority.append(soa)
1603
1604         # Miss
1605         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1606         self.assertTrue(receivedQuery)
1607         self.assertTrue(receivedResponse)
1608         receivedQuery.id = query.id
1609         self.assertEquals(query, receivedQuery)
1610         self.assertEquals(response, receivedResponse)
1611         misses += 1
1612
1613         # next queries should hit the cache
1614         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1615         self.assertEquals(receivedResponse, response)
1616
1617         time.sleep(self._negCacheTTL + 1)
1618
1619         # we should not have cached for longer than the negativel TTL
1620         # so it should be a miss
1621         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1622         self.assertTrue(receivedQuery)
1623         self.assertTrue(receivedResponse)
1624         receivedQuery.id = query.id
1625         self.assertEquals(query, receivedQuery)
1626         self.assertEquals(response, receivedResponse)
1627         misses += 1
1628
1629         total = 0
1630         for key in self._responsesCounter:
1631             total += self._responsesCounter[key]
1632
1633         self.assertEquals(total, misses)
1634
1635 class TestCachingDontAge(DNSDistTest):
1636
1637     _config_template = """
1638     -- maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=true
1639     pc = newPacketCache(100, 86400, 0, 60, 60, true)
1640     getPool(""):setCache(pc)
1641     newServer{address="127.0.0.1:%d"}
1642     """
1643     def testCacheDoesntDecreaseTTL(self):
1644         """
1645         Cache: Cache doesn't decrease TTL with 'don't age' set
1646
1647         dnsdist is configured to cache entries but without aging the TTL,
1648         we are sending one request (cache miss) and verify that the cache
1649         hits don't have a decreasing TTL.
1650         """
1651         ttl = 600
1652         misses = 0
1653         name = 'cachedoesntdecreasettl.cache-dont-age.tests.powerdns.com.'
1654         query = dns.message.make_query(name, 'AAAA', 'IN')
1655         response = dns.message.make_response(query)
1656         rrset = dns.rrset.from_text(name,
1657                                     ttl,
1658                                     dns.rdataclass.IN,
1659                                     dns.rdatatype.AAAA,
1660                                     '::1')
1661         response.answer.append(rrset)
1662
1663         # first query to fill the cache
1664         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1665         self.assertTrue(receivedQuery)
1666         self.assertTrue(receivedResponse)
1667         receivedQuery.id = query.id
1668         self.assertEquals(query, receivedQuery)
1669         self.assertEquals(receivedResponse, response)
1670         misses += 1
1671
1672         # next queries should hit the cache
1673         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1674         self.assertEquals(receivedResponse, response)
1675         for an in receivedResponse.answer:
1676             self.assertTrue(an.ttl == ttl)
1677
1678         # now we wait a bit for the TTL to decrease
1679         time.sleep(1)
1680
1681         # next queries should hit the cache
1682         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1683         self.assertEquals(receivedResponse, response)
1684         for an in receivedResponse.answer:
1685             self.assertTrue(an.ttl == ttl)
1686
1687         total = 0
1688         for key in self._responsesCounter:
1689             total += self._responsesCounter[key]
1690
1691         self.assertEquals(total, misses)
1692
1693 class TestCachingECSWithoutPoolECS(DNSDistTest):
1694
1695     _consoleKey = DNSDistTest.generateConsoleKey()
1696     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1697     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
1698     _config_template = """
1699     -- maxTTL=86400, minTTL=1
1700     pc = newPacketCache(100, 86400, 1)
1701     getPool(""):setCache(pc)
1702     setKey("%s")
1703     controlSocket("127.0.0.1:%d")
1704     newServer{address="127.0.0.1:%d", useClientSubnet=true}
1705     """
1706
1707     def testCached(self):
1708         """
1709         Cache: Cached entry with ECS is a miss when no backend are available
1710         """
1711         ttl = 600
1712         name = 'cached.cache-ecs-without-pool-ecs.tests.powerdns.com.'
1713         query = dns.message.make_query(name, 'AAAA', 'IN')
1714         response = dns.message.make_response(query)
1715         rrset = dns.rrset.from_text(name,
1716                                     ttl,
1717                                     dns.rdataclass.IN,
1718                                     dns.rdatatype.AAAA,
1719                                     '::1')
1720         response.answer.append(rrset)
1721
1722         # first query to fill the cache
1723         for method in ("sendUDPQuery", "sendTCPQuery"):
1724             sender = getattr(self, method)
1725             (receivedQuery, receivedResponse) = sender(query, response)
1726             self.assertTrue(receivedQuery)
1727             self.assertTrue(receivedResponse)
1728             receivedQuery.id = query.id
1729             self.assertEquals(query, receivedQuery)
1730             self.assertEquals(receivedResponse, response)
1731
1732         # next queries should hit the cache
1733         for method in ("sendUDPQuery", "sendTCPQuery"):
1734             sender = getattr(self, method)
1735             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1736             self.assertEquals(receivedResponse, response)
1737
1738         # we mark the backend as down
1739         self.sendConsoleCommand("getServer(0):setDown()")
1740
1741         # we should NOT get a cached entry since it has ECS and we haven't asked the pool
1742         # to add ECS when no backend is up
1743         for method in ("sendUDPQuery", "sendTCPQuery"):
1744             sender = getattr(self, method)
1745             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1746             self.assertEquals(receivedResponse, None)
1747
1748 class TestCachingECSWithPoolECS(DNSDistTest):
1749
1750     _consoleKey = DNSDistTest.generateConsoleKey()
1751     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1752     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
1753     _config_template = """
1754     -- maxTTL=86400, minTTL=1
1755     pc = newPacketCache(100, 86400, 1)
1756     getPool(""):setCache(pc)
1757     getPool(""):setECS(true)
1758     setKey("%s")
1759     controlSocket("127.0.0.1:%d")
1760     newServer{address="127.0.0.1:%d", useClientSubnet=true}
1761     """
1762
1763     def testCached(self):
1764         """
1765         Cache: Cached entry with ECS is a hit when no backend are available
1766         """
1767         ttl = 600
1768         name = 'cached.cache-ecs-with-pool-ecs.tests.powerdns.com.'
1769         query = dns.message.make_query(name, 'AAAA', 'IN')
1770         response = dns.message.make_response(query)
1771         rrset = dns.rrset.from_text(name,
1772                                     ttl,
1773                                     dns.rdataclass.IN,
1774                                     dns.rdatatype.AAAA,
1775                                     '::1')
1776         response.answer.append(rrset)
1777
1778         # first query to fill the cache
1779         for method in ("sendUDPQuery", "sendTCPQuery"):
1780             sender = getattr(self, method)
1781             (receivedQuery, receivedResponse) = sender(query, response)
1782             self.assertTrue(receivedQuery)
1783             self.assertTrue(receivedResponse)
1784             receivedQuery.id = query.id
1785             self.assertEquals(query, receivedQuery)
1786             self.assertEquals(receivedResponse, response)
1787
1788         # next queries should hit the cache
1789         for method in ("sendUDPQuery", "sendTCPQuery"):
1790             sender = getattr(self, method)
1791             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1792             self.assertEquals(receivedResponse, response)
1793
1794         # we mark the backend as down
1795         self.sendConsoleCommand("getServer(0):setDown()")
1796
1797         # we should STILL get a cached entry since it has ECS and we have asked the pool
1798         # to add ECS when no backend is up
1799         for method in ("sendUDPQuery", "sendTCPQuery"):
1800             sender = getattr(self, method)
1801             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1802             self.assertEquals(receivedResponse, response)
1803
1804 class TestCachingCollisionNoECSParsing(DNSDistTest):
1805
1806     _config_template = """
1807     -- maxTTL=86400, minTTL=1
1808     pc = newPacketCache(100, 86400, 1)
1809     getPool(""):setCache(pc)
1810     newServer{address="127.0.0.1:%d"}
1811     """
1812
1813     def testCacheCollisionNoECSParsing(self):
1814         """
1815         Cache: Collision with no ECS parsing
1816         """
1817         name = 'collision-no-ecs-parsing.cache.tests.powerdns.com.'
1818         ecso = clientsubnetoption.ClientSubnetOption('10.0.188.3', 32)
1819         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1820         query.flags = dns.flags.RD
1821         response = dns.message.make_response(query)
1822         rrset = dns.rrset.from_text(name,
1823                                     3600,
1824                                     dns.rdataclass.IN,
1825                                     dns.rdatatype.AAAA,
1826                                     '::1')
1827         response.answer.append(rrset)
1828
1829         # first query should to fill the cache
1830         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1831         self.assertTrue(receivedQuery)
1832         self.assertTrue(receivedResponse)
1833         receivedQuery.id = query.id
1834         self.assertEquals(query, receivedQuery)
1835         self.assertEquals(receivedResponse, response)
1836
1837         # second query will hash to the same key, triggering a collision which
1838         # will not be detected because the qname, qtype, qclass and flags will
1839         # match and EDNS Client Subnet parsing has not been enabled
1840         ecso2 = clientsubnetoption.ClientSubnetOption('10.0.192.138', 32)
1841         query2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
1842         query2.flags = dns.flags.RD
1843         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1844         receivedResponse.id = response.id
1845         self.assertEquals(receivedResponse, response)
1846
1847 class TestCachingCollisionWithECSParsing(DNSDistTest):
1848
1849     _config_template = """
1850     -- maxTTL=86400, minTTL=1, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, parseECS=true
1851     pc = newPacketCache(100, 86400, 1, 60, 60, false, 1, true, 3600, true)
1852     getPool(""):setCache(pc)
1853     newServer{address="127.0.0.1:%d"}
1854     """
1855
1856     def testCacheCollisionWithECSParsing(self):
1857         """
1858         Cache: Collision with ECS parsing
1859         """
1860         name = 'collision-with-ecs-parsing.cache.tests.powerdns.com.'
1861         ecso = clientsubnetoption.ClientSubnetOption('10.0.115.61', 32)
1862         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1863         query.flags = dns.flags.RD
1864         response = dns.message.make_response(query)
1865         rrset = dns.rrset.from_text(name,
1866                                     3600,
1867                                     dns.rdataclass.IN,
1868                                     dns.rdatatype.AAAA,
1869                                     '::1')
1870         response.answer.append(rrset)
1871
1872         # first query should to fill the cache
1873         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1874         self.assertTrue(receivedQuery)
1875         self.assertTrue(receivedResponse)
1876         receivedQuery.id = query.id
1877         self.assertEquals(query, receivedQuery)
1878         self.assertEquals(receivedResponse, response)
1879
1880         # second query will hash to the same key, triggering a collision which
1881         # _will_ be detected this time because the qname, qtype, qclass and flags will
1882         # match but EDNS Client Subnet parsing is now enabled and will detect the issue
1883         ecso2 = clientsubnetoption.ClientSubnetOption('10.0.143.21', 32)
1884         query2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
1885         query2.flags = dns.flags.RD
1886         response2 = dns.message.make_response(query2)
1887         rrset = dns.rrset.from_text(name,
1888                                     3600,
1889                                     dns.rdataclass.IN,
1890                                     dns.rdatatype.AAAA,
1891                                     '2001:DB8::1')
1892         response2.answer.append(rrset)
1893         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1894         self.assertEquals(receivedResponse, response2)
1895
1896 class TestCachingScopeZero(DNSDistTest):
1897
1898     _config_template = """
1899     -- Be careful to enable ECS parsing in the packet cache, otherwise scope zero is disabled
1900     pc = newPacketCache(100, 86400, 1, 60, 60, false, 1, true, 3600, true)
1901     getPool(""):setCache(pc)
1902     newServer{address="127.0.0.1:%d", useClientSubnet=true}
1903     -- to simulate a second client coming from a different IP address,
1904     -- we will force the ECS value added to the query if RD is set (note that we need
1905     -- to unset it using rules before the first cache lookup)
1906     addAction(RDRule(), SetECSAction("192.0.2.1/32"))
1907     addAction(RDRule(), NoRecurseAction())
1908     """
1909
1910     def testScopeZero(self):
1911         """
1912         Cache: Test the scope-zero feature, backend returns a scope of zero
1913         """
1914         ttl = 600
1915         name = 'scope-zero.cache.tests.powerdns.com.'
1916         query = dns.message.make_query(name, 'AAAA', 'IN')
1917         query.flags &= ~dns.flags.RD
1918         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
1919         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1920         expectedQuery.flags &= ~dns.flags.RD
1921         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 0)
1922         expectedResponse = dns.message.make_response(query)
1923         scopedResponse = dns.message.make_response(query)
1924         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
1925         rrset = dns.rrset.from_text(name,
1926                                     ttl,
1927                                     dns.rdataclass.IN,
1928                                     dns.rdatatype.AAAA,
1929                                     '::1')
1930         scopedResponse.answer.append(rrset)
1931         expectedResponse.answer.append(rrset)
1932
1933         for method in ("sendUDPQuery", "sendTCPQuery"):
1934             sender = getattr(self, method)
1935             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
1936             receivedQuery.id = expectedQuery.id
1937             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
1938             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
1939
1940         # next query should hit the cache, nothing special about that
1941         for method in ("sendUDPQuery", "sendTCPQuery"):
1942             sender = getattr(self, method)
1943             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1944             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
1945
1946         query = dns.message.make_query(name, 'AAAA', 'IN')
1947         query.flags &= dns.flags.RD
1948         # next query FROM A DIFFERENT CLIENT since RD is now set should STILL hit the cache
1949         for method in ("sendUDPQuery", "sendTCPQuery"):
1950             sender = getattr(self, method)
1951             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1952             receivedResponse.id = expectedResponse.id
1953             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
1954
1955         name = 'scope-zero-with-ecs.cache.tests.powerdns.com.'
1956         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24)
1957         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1958         query.flags &= ~dns.flags.RD
1959         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1960         expectedQuery.flags &= ~dns.flags.RD
1961         expectedResponse = dns.message.make_response(query)
1962         expectedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
1963         expectedResponse.answer.append(rrset)
1964         scopedResponse = dns.message.make_response(query)
1965         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
1966         scopedResponse.answer.append(rrset)
1967         # this query has ECS, it should NOT be able to use the scope-zero cached entry since the hash will be
1968         # different
1969         for method in ("sendUDPQuery", "sendTCPQuery"):
1970             sender = getattr(self, method)
1971             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
1972             receivedQuery.id = expectedQuery.id
1973             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
1974             self.checkMessageEDNSWithECS(receivedResponse, expectedResponse)
1975
1976         # it should still have been cached, though, so the next query should be a hit
1977         for method in ("sendUDPQuery", "sendTCPQuery"):
1978             sender = getattr(self, method)
1979             (_, receivedResponse) = sender(query, response=None, useQueue=False)
1980             self.checkMessageEDNSWithECS(receivedResponse, expectedResponse)
1981
1982     def testScopeNotZero(self):
1983         """
1984         Cache: Test the scope-zero feature, backend returns a scope of non-zero
1985         """
1986         ttl = 600
1987         name = 'scope-not-zero.cache.tests.powerdns.com.'
1988         query = dns.message.make_query(name, 'AAAA', 'IN')
1989         query.flags &= ~dns.flags.RD
1990         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
1991         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
1992         expectedQuery.flags &= ~dns.flags.RD
1993         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
1994         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
1995         expectedQuery2.flags &= ~dns.flags.RD
1996         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 24)
1997         ecsoResponse2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32, 24)
1998         rrset = dns.rrset.from_text(name,
1999                                     ttl,
2000                                     dns.rdataclass.IN,
2001                                     dns.rdatatype.AAAA,
2002                                     '::1')
2003         expectedResponse = dns.message.make_response(query)
2004         expectedResponse.answer.append(rrset)
2005         scopedResponse = dns.message.make_response(query)
2006         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2007         scopedResponse.answer.append(rrset)
2008         scopedResponse2 = dns.message.make_response(query)
2009         scopedResponse2.use_edns(edns=True, payload=4096, options=[ecsoResponse2])
2010         scopedResponse2.answer.append(rrset)
2011
2012         for method in ("sendUDPQuery", "sendTCPQuery"):
2013             sender = getattr(self, method)
2014             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2015             receivedQuery.id = expectedQuery.id
2016             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2017             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2018
2019         # next query should hit the cache, nothing special about that
2020         for method in ("sendUDPQuery", "sendTCPQuery"):
2021             sender = getattr(self, method)
2022             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2023             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2024
2025         query = dns.message.make_query(name, 'AAAA', 'IN')
2026         query.flags &= dns.flags.RD
2027         expectedResponse = dns.message.make_response(query)
2028         expectedResponse.answer.append(rrset)
2029         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2030         for method in ("sendUDPQuery", "sendTCPQuery"):
2031             sender = getattr(self, method)
2032             (receivedQuery, receivedResponse) = sender(query, scopedResponse2)
2033             receivedQuery.id = expectedQuery2.id
2034             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2035             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2036
2037     def testNoECS(self):
2038         """
2039         Cache: Test the scope-zero feature, backend returns no ECS at all
2040         """
2041         ttl = 600
2042         name = 'scope-zero-no-ecs.cache.tests.powerdns.com.'
2043         query = dns.message.make_query(name, 'AAAA', 'IN')
2044         query.flags &= ~dns.flags.RD
2045         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2046         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2047         expectedQuery.flags &= ~dns.flags.RD
2048         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
2049         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2050         expectedQuery2.flags &= ~dns.flags.RD
2051         rrset = dns.rrset.from_text(name,
2052                                     ttl,
2053                                     dns.rdataclass.IN,
2054                                     dns.rdatatype.AAAA,
2055                                     '::1')
2056         response = dns.message.make_response(query)
2057         response.answer.append(rrset)
2058
2059         for method in ("sendUDPQuery", "sendTCPQuery"):
2060             sender = getattr(self, method)
2061             (receivedQuery, receivedResponse) = sender(query, response)
2062             receivedQuery.id = expectedQuery.id
2063             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2064             self.checkMessageNoEDNS(receivedResponse, response)
2065
2066         # next query should hit the cache, nothing special about that
2067         for method in ("sendUDPQuery", "sendTCPQuery"):
2068             sender = getattr(self, method)
2069             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2070             self.checkMessageNoEDNS(receivedResponse, response)
2071
2072         query = dns.message.make_query(name, 'AAAA', 'IN')
2073         query.flags &= dns.flags.RD
2074         response = dns.message.make_response(query)
2075         response.answer.append(rrset)
2076         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2077         for method in ("sendUDPQuery", "sendTCPQuery"):
2078             sender = getattr(self, method)
2079             (receivedQuery, receivedResponse) = sender(query, response)
2080             receivedQuery.id = expectedQuery2.id
2081             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2082             self.checkMessageNoEDNS(receivedResponse, response)
2083
2084 class TestCachingScopeZeroButNoSubnetcheck(DNSDistTest):
2085
2086     _config_template = """
2087     -- We disable ECS parsing in the packet cache, meaning scope zero is disabled
2088     pc = newPacketCache(100, 86400, 1, 60, 60, false, 1, true, 3600, false)
2089     getPool(""):setCache(pc)
2090     newServer{address="127.0.0.1:%d", useClientSubnet=true}
2091     -- to simulate a second client coming from a different IP address,
2092     -- we will force the ECS value added to the query if RD is set (note that we need
2093     -- to unset it using rules before the first cache lookup)
2094     addAction(RDRule(), SetECSAction("192.0.2.1/32"))
2095     addAction(RDRule(), NoRecurseAction())
2096     """
2097
2098     def testScopeZero(self):
2099         """
2100         Cache: Test that the scope-zero feature is disabled when ECS parsing is not enabled in the cache
2101         """
2102         ttl = 600
2103         name = 'scope-zero-no-subnet.cache.tests.powerdns.com.'
2104         query = dns.message.make_query(name, 'AAAA', 'IN')
2105         query.flags &= ~dns.flags.RD
2106         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2107         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2108         expectedQuery.flags &= ~dns.flags.RD
2109         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
2110         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2111         expectedQuery2.flags &= ~dns.flags.RD
2112         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 0)
2113         expectedResponse = dns.message.make_response(query)
2114         scopedResponse = dns.message.make_response(query)
2115         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2116         rrset = dns.rrset.from_text(name,
2117                                     ttl,
2118                                     dns.rdataclass.IN,
2119                                     dns.rdatatype.AAAA,
2120                                     '::1')
2121         scopedResponse.answer.append(rrset)
2122         expectedResponse.answer.append(rrset)
2123
2124         for method in ("sendUDPQuery", "sendTCPQuery"):
2125             sender = getattr(self, method)
2126             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2127             receivedQuery.id = expectedQuery.id
2128             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2129             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2130
2131         # next query should hit the cache, nothing special about that
2132         for method in ("sendUDPQuery", "sendTCPQuery"):
2133             sender = getattr(self, method)
2134             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2135             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2136
2137         query = dns.message.make_query(name, 'AAAA', 'IN')
2138         query.flags &= dns.flags.RD
2139         response = dns.message.make_response(query)
2140         response.answer.append(rrset)
2141         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2142         for method in ("sendUDPQuery", "sendTCPQuery"):
2143             sender = getattr(self, method)
2144             (receivedQuery, receivedResponse) = sender(query, response)
2145             receivedQuery.id = expectedQuery2.id
2146             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2147             self.checkMessageNoEDNS(receivedResponse, response)