regression-tests.dnsdist/test_Caching.py

   1 #!/usr/bin/env python
   2 import base64
   3 import time
   4 import dns
   5 import clientsubnetoption
   6 import cookiesoption
   7 from dnsdisttests import DNSDistTest
   8
   9 class TestCaching(DNSDistTest):
  10
  11     _config_template = """
  12     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
  13     getPool(""):setCache(pc)
  14     addAction(makeRule("nocache.cache.tests.powerdns.com."), SetSkipCacheAction())
  15     addResponseAction(makeRule("nocache-response.cache.tests.powerdns.com."), SetSkipCacheResponseAction())
  16     function skipViaLua(dq)
  17         dq.skipCache = true
  18         return DNSAction.None, ""
  19     end
  20     addAction("nocachevialua.cache.tests.powerdns.com.", LuaAction(skipViaLua))
  21     newServer{address="127.0.0.1:%d"}
  22     """
  23
  24     def testCached(self):
  25         """
  26         Cache: Served from cache
  27
  28         dnsdist is configured to cache entries, we are sending several
  29         identical requests and checking that the backend only receive
  30         the first one.
  31         """
  32         numberOfQueries = 10
  33         name = 'cached.cache.tests.powerdns.com.'
  34         query = dns.message.make_query(name, 'AAAA', 'IN')
  35         response = dns.message.make_response(query)
  36         rrset = dns.rrset.from_text(name,
  37                                     3600,
  38                                     dns.rdataclass.IN,
  39                                     dns.rdatatype.AAAA,
  40                                     '::1')
  41         response.answer.append(rrset)
  42
  43         # first query to fill the cache
  44         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
  45         self.assertTrue(receivedQuery)
  46         self.assertTrue(receivedResponse)
  47         receivedQuery.id = query.id
  48         self.assertEqual(query, receivedQuery)
  49         self.assertEqual(receivedResponse, response)
  50
  51         for _ in range(numberOfQueries):
  52             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
  53             self.assertEqual(receivedResponse, response)
  54
  55         total = 0
  56         for key in self._responsesCounter:
  57             total += self._responsesCounter[key]
  58             TestCaching._responsesCounter[key] = 0
  59
  60         self.assertEqual(total, 1)
  61
  62         # TCP should not be cached
  63         # first query to fill the cache
  64         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
  65         self.assertTrue(receivedQuery)
  66         self.assertTrue(receivedResponse)
  67         receivedQuery.id = query.id
  68         self.assertEqual(query, receivedQuery)
  69         self.assertEqual(receivedResponse, response)
  70
  71         for _ in range(numberOfQueries):
  72             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
  73             self.assertEqual(receivedResponse, response)
  74
  75         total = 0
  76         for key in self._responsesCounter:
  77             total += self._responsesCounter[key]
  78             TestCaching._responsesCounter[key] = 0
  79
  80         self.assertEqual(total, 1)
  81
  82     def testDOCached(self):
  83         """
  84         Cache: Served from cache, query has DO bit set
  85
  86         dnsdist is configured to cache entries, we are sending several
  87         identical requests and checking that the backend only receive
  88         the first one.
  89         """
  90         numberOfQueries = 10
  91         name = 'cached-do.cache.tests.powerdns.com.'
  92         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, want_dnssec=True)
  93         response = dns.message.make_response(query)
  94         rrset = dns.rrset.from_text(name,
  95                                     3600,
  96                                     dns.rdataclass.IN,
  97                                     dns.rdatatype.AAAA,
  98                                     '::1')
  99         response.answer.append(rrset)
 100
 101         # first query to fill the cache
 102         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 103         self.assertTrue(receivedQuery)
 104         self.assertTrue(receivedResponse)
 105         receivedQuery.id = query.id
 106         self.assertEqual(query, receivedQuery)
 107         self.assertEqual(receivedResponse, response)
 108
 109         for _ in range(numberOfQueries):
 110             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 111             self.assertEqual(receivedResponse, response)
 112
 113         total = 0
 114         for key in self._responsesCounter:
 115             total += self._responsesCounter[key]
 116             TestCaching._responsesCounter[key] = 0
 117
 118         self.assertEqual(total, 1)
 119
 120         # TCP should not be cached
 121         # first query to fill the cache
 122         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 123         self.assertTrue(receivedQuery)
 124         self.assertTrue(receivedResponse)
 125         receivedQuery.id = query.id
 126         self.assertEqual(query, receivedQuery)
 127         self.assertEqual(receivedResponse, response)
 128
 129         for _ in range(numberOfQueries):
 130             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
 131             self.assertEqual(receivedResponse, response)
 132
 133         total = 0
 134         for key in self._responsesCounter:
 135             total += self._responsesCounter[key]
 136             TestCaching._responsesCounter[key] = 0
 137
 138         self.assertEqual(total, 1)
 139
 140     def testSkipCache(self):
 141         """
 142         Cache: SetSkipCacheAction
 143
 144         dnsdist is configured to not cache entries for nocache.cache.tests.powerdns.com.
 145          we are sending several requests and checking that the backend get them all.
 146         """
 147         name = 'nocache.cache.tests.powerdns.com.'
 148         numberOfQueries = 10
 149         query = dns.message.make_query(name, 'AAAA', 'IN')
 150         response = dns.message.make_response(query)
 151         rrset = dns.rrset.from_text(name,
 152                                     3600,
 153                                     dns.rdataclass.IN,
 154                                     dns.rdatatype.AAAA,
 155                                     '::1')
 156         response.answer.append(rrset)
 157
 158         for _ in range(numberOfQueries):
 159             for method in ("sendUDPQuery", "sendTCPQuery"):
 160                 sender = getattr(self, method)
 161                 (receivedQuery, receivedResponse) = sender(query, response)
 162                 self.assertTrue(receivedQuery)
 163                 self.assertTrue(receivedResponse)
 164                 receivedQuery.id = query.id
 165                 self.assertEqual(query, receivedQuery)
 166                 self.assertEqual(receivedResponse, response)
 167
 168         for key in self._responsesCounter:
 169             value = self._responsesCounter[key]
 170             self.assertEqual(value, numberOfQueries)
 171
 172     def testSkipCacheViaLua(self):
 173         """
 174         Cache: SkipCache via Lua
 175
 176         dnsdist is configured to not cache entries for nocachevialua.cache.tests.powerdns.com.
 177          we are sending several requests and checking that the backend get them all.
 178         """
 179         name = 'nocachevialua.cache.tests.powerdns.com.'
 180         numberOfQueries = 10
 181         query = dns.message.make_query(name, 'AAAA', 'IN')
 182         response = dns.message.make_response(query)
 183         rrset = dns.rrset.from_text(name,
 184                                     3600,
 185                                     dns.rdataclass.IN,
 186                                     dns.rdatatype.AAAA,
 187                                     '::1')
 188         response.answer.append(rrset)
 189
 190         for _ in range(numberOfQueries):
 191             for method in ("sendUDPQuery", "sendTCPQuery"):
 192                 sender = getattr(self, method)
 193                 (receivedQuery, receivedResponse) = sender(query, response)
 194                 self.assertTrue(receivedQuery)
 195                 self.assertTrue(receivedResponse)
 196                 receivedQuery.id = query.id
 197                 self.assertEqual(query, receivedQuery)
 198                 self.assertEqual(receivedResponse, response)
 199
 200         for key in self._responsesCounter:
 201             value = self._responsesCounter[key]
 202             self.assertEqual(value, numberOfQueries)
 203
 204     def testSkipCacheResponse(self):
 205         """
 206         Cache: SetSkipCacheResponseAction
 207
 208         dnsdist is configured to not cache entries for answer matching nocache-response.cache.tests.powerdns.com.
 209          we are sending several requests and checking that the backend get them all.
 210         """
 211         name = 'nocache-response.cache.tests.powerdns.com.'
 212         numberOfQueries = 10
 213         query = dns.message.make_query(name, 'AAAA', 'IN')
 214         response = dns.message.make_response(query)
 215         rrset = dns.rrset.from_text(name,
 216                                     3600,
 217                                     dns.rdataclass.IN,
 218                                     dns.rdatatype.AAAA,
 219                                     '::1')
 220         response.answer.append(rrset)
 221
 222         for _ in range(numberOfQueries):
 223             for method in ("sendUDPQuery", "sendTCPQuery"):
 224                 sender = getattr(self, method)
 225                 (receivedQuery, receivedResponse) = sender(query, response)
 226                 self.assertTrue(receivedQuery)
 227                 self.assertTrue(receivedResponse)
 228                 receivedQuery.id = query.id
 229                 self.assertEqual(query, receivedQuery)
 230                 self.assertEqual(receivedResponse, response)
 231
 232         for key in self._responsesCounter:
 233             value = self._responsesCounter[key]
 234             self.assertEqual(value, numberOfQueries)
 235
 236     def testAXFRResponse(self):
 237         """
 238         Cache: AXFR should not be cached
 239
 240         dnsdist should not cache responses to AXFR queries.
 241         """
 242         name = 'axfr.cache.tests.powerdns.com.'
 243         query = dns.message.make_query(name, 'AXFR', 'IN')
 244         response = dns.message.make_response(query)
 245         soa = dns.rrset.from_text(name,
 246                                   60,
 247                                   dns.rdataclass.IN,
 248                                   dns.rdatatype.SOA,
 249                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
 250         response.answer.append(soa)
 251         response.answer.append(dns.rrset.from_text(name,
 252                                                    60,
 253                                                    dns.rdataclass.IN,
 254                                                    dns.rdatatype.A,
 255                                                    '192.0.2.1'))
 256         response.answer.append(soa)
 257         numberOfQueries = 5
 258
 259         for _ in range(numberOfQueries):
 260             for method in ("sendUDPQuery", "sendTCPQuery"):
 261                 sender = getattr(self, method)
 262                 (receivedQuery, receivedResponse) = sender(query, response)
 263                 self.assertTrue(receivedQuery)
 264                 self.assertTrue(receivedResponse)
 265                 receivedQuery.id = query.id
 266                 self.assertEqual(query, receivedQuery)
 267                 self.assertEqual(receivedResponse, response)
 268
 269         for key in self._responsesCounter:
 270             value = self._responsesCounter[key]
 271             self.assertEqual(value, numberOfQueries)
 272
 273     def testIXFRResponse(self):
 274         """
 275         Cache: IXFR should not be cached
 276
 277         dnsdist should not cache responses to IXFR queries.
 278         """
 279         name = 'ixfr.cache.tests.powerdns.com.'
 280         query = dns.message.make_query(name, 'IXFR', 'IN')
 281         response = dns.message.make_response(query)
 282         soa = dns.rrset.from_text(name,
 283                                   60,
 284                                   dns.rdataclass.IN,
 285                                   dns.rdatatype.SOA,
 286                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
 287         response.answer.append(soa)
 288         response.answer.append(dns.rrset.from_text(name,
 289                                                    60,
 290                                                    dns.rdataclass.IN,
 291                                                    dns.rdatatype.A,
 292                                                    '192.0.2.1'))
 293         response.answer.append(soa)
 294         numberOfQueries = 5
 295
 296         for _ in range(numberOfQueries):
 297             for method in ("sendUDPQuery", "sendTCPQuery"):
 298                 sender = getattr(self, method)
 299                 (receivedQuery, receivedResponse) = sender(query, response)
 300                 self.assertTrue(receivedQuery)
 301                 self.assertTrue(receivedResponse)
 302                 receivedQuery.id = query.id
 303                 self.assertEqual(query, receivedQuery)
 304                 self.assertEqual(receivedResponse, response)
 305
 306         for key in self._responsesCounter:
 307             value = self._responsesCounter[key]
 308             self.assertEqual(value, numberOfQueries)
 309
 310     def testCacheExpiration(self):
 311         """
 312         Cache: Cache expiration
 313
 314         dnsdist is configured to cache entries, we are sending one request
 315         (cache miss) with a very short TTL, checking that the next requests
 316         are cached. Then we wait for the TTL to expire, check that the
 317         next request is a miss but the following one a hit.
 318         """
 319         ttl = 2
 320         misses = 0
 321         name = 'cacheexpiration.cache.tests.powerdns.com.'
 322         query = dns.message.make_query(name, 'AAAA', 'IN')
 323         response = dns.message.make_response(query)
 324         rrset = dns.rrset.from_text(name,
 325                                     ttl,
 326                                     dns.rdataclass.IN,
 327                                     dns.rdatatype.AAAA,
 328                                     '::1')
 329         response.answer.append(rrset)
 330
 331         # first query to fill the cache
 332         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 333         self.assertTrue(receivedQuery)
 334         self.assertTrue(receivedResponse)
 335         receivedQuery.id = query.id
 336         self.assertEqual(query, receivedQuery)
 337         self.assertEqual(receivedResponse, response)
 338         misses += 1
 339
 340         # next queries should hit the cache
 341         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 342         self.assertEqual(receivedResponse, response)
 343
 344         # now we wait a bit for the cache entry to expire
 345         time.sleep(ttl + 1)
 346
 347         # next query should be a miss, fill the cache again
 348         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 349         self.assertTrue(receivedQuery)
 350         self.assertTrue(receivedResponse)
 351         receivedQuery.id = query.id
 352         self.assertEqual(query, receivedQuery)
 353         self.assertEqual(receivedResponse, response)
 354         misses += 1
 355
 356         # following queries should hit the cache again
 357         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 358         self.assertEqual(receivedResponse, response)
 359
 360         total = 0
 361         for key in self._responsesCounter:
 362             total += self._responsesCounter[key]
 363
 364         self.assertEqual(total, misses)
 365
 366     def testCacheExpirationDifferentSets(self):
 367         """
 368         Cache: Cache expiration with different sets
 369
 370         dnsdist is configured to cache entries, we are sending one request
 371         (cache miss) whose response has a long and a very short TTL,
 372         checking that the next requests are cached. Then we wait for the
 373         short TTL to expire, check that the
 374         next request is a miss but the following one a hit.
 375         """
 376         ttl = 2
 377         misses = 0
 378         name = 'cacheexpirationdifferentsets.cache.tests.powerdns.com.'
 379         query = dns.message.make_query(name, 'AAAA', 'IN')
 380         response = dns.message.make_response(query)
 381         rrset = dns.rrset.from_text(name,
 382                                     ttl,
 383                                     dns.rdataclass.IN,
 384                                     dns.rdatatype.CNAME,
 385                                     'cname.cacheexpirationdifferentsets.cache.tests.powerdns.com.')
 386         response.answer.append(rrset)
 387         rrset = dns.rrset.from_text('cname.cacheexpirationdifferentsets.cache.tests.powerdns.com.',
 388                                     ttl + 3600,
 389                                     dns.rdataclass.IN,
 390                                     dns.rdatatype.A,
 391                                     '192.2.0.1')
 392         response.additional.append(rrset)
 393
 394         # first query to fill the cache
 395         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 396         self.assertTrue(receivedQuery)
 397         self.assertTrue(receivedResponse)
 398         receivedQuery.id = query.id
 399         self.assertEqual(query, receivedQuery)
 400         self.assertEqual(receivedResponse, response)
 401         misses += 1
 402
 403         # next queries should hit the cache
 404         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 405         self.assertEqual(receivedResponse, response)
 406
 407         # now we wait a bit for the cache entry to expire
 408         time.sleep(ttl + 1)
 409
 410         # next query should be a miss, fill the cache again
 411         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 412         self.assertTrue(receivedQuery)
 413         self.assertTrue(receivedResponse)
 414         receivedQuery.id = query.id
 415         self.assertEqual(query, receivedQuery)
 416         self.assertEqual(receivedResponse, response)
 417         misses += 1
 418
 419         # following queries should hit the cache again
 420         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 421         self.assertEqual(receivedResponse, response)
 422
 423         total = 0
 424         for key in self._responsesCounter:
 425             total += self._responsesCounter[key]
 426
 427         self.assertEqual(total, misses)
 428
 429     def testCacheDecreaseTTL(self):
 430         """
 431         Cache: Cache decreases TTL
 432
 433         dnsdist is configured to cache entries, we are sending one request
 434         (cache miss) and verify that the cache hits have a decreasing TTL.
 435         """
 436         ttl = 600
 437         misses = 0
 438         name = 'cachedecreasettl.cache.tests.powerdns.com.'
 439         query = dns.message.make_query(name, 'AAAA', 'IN')
 440         response = dns.message.make_response(query)
 441         rrset = dns.rrset.from_text(name,
 442                                     ttl,
 443                                     dns.rdataclass.IN,
 444                                     dns.rdatatype.AAAA,
 445                                     '::1')
 446         response.answer.append(rrset)
 447
 448         # first query to fill the cache
 449         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 450         self.assertTrue(receivedQuery)
 451         self.assertTrue(receivedResponse)
 452         receivedQuery.id = query.id
 453         self.assertEqual(query, receivedQuery)
 454         self.assertEqual(receivedResponse, response)
 455         misses += 1
 456
 457         # next queries should hit the cache
 458         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 459         self.assertEqual(receivedResponse, response)
 460         for an in receivedResponse.answer:
 461             self.assertTrue(an.ttl <= ttl)
 462
 463         # now we wait a bit for the TTL to decrease
 464         time.sleep(1)
 465
 466         # next queries should hit the cache
 467         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 468         self.assertEqual(receivedResponse, response)
 469         for an in receivedResponse.answer:
 470             self.assertTrue(an.ttl < ttl)
 471
 472         total = 0
 473         for key in self._responsesCounter:
 474             total += self._responsesCounter[key]
 475
 476         self.assertEqual(total, misses)
 477
 478     def testCacheDifferentCase(self):
 479         """
 480         Cache: Cache matches different case
 481
 482         dnsdist is configured to cache entries, we are sending one request
 483         (cache miss) and verify that the same one with a different case
 484         matches.
 485         """
 486         ttl = 600
 487         name = 'cachedifferentcase.cache.tests.powerdns.com.'
 488         differentCaseName = 'CacheDifferentCASE.cache.tests.powerdns.com.'
 489         query = dns.message.make_query(name, 'AAAA', 'IN')
 490         differentCaseQuery = dns.message.make_query(differentCaseName, 'AAAA', 'IN')
 491         response = dns.message.make_response(query)
 492         differentCaseResponse = dns.message.make_response(differentCaseQuery)
 493         rrset = dns.rrset.from_text(name,
 494                                     ttl,
 495                                     dns.rdataclass.IN,
 496                                     dns.rdatatype.AAAA,
 497                                     '::1')
 498         response.answer.append(rrset)
 499         differentCaseResponse.answer.append(rrset)
 500
 501         # first query to fill the cache
 502         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 503         self.assertTrue(receivedQuery)
 504         self.assertTrue(receivedResponse)
 505         receivedQuery.id = query.id
 506         self.assertEqual(query, receivedQuery)
 507         self.assertEqual(receivedResponse, response)
 508
 509         # different case query should still hit the cache
 510         (_, receivedResponse) = self.sendUDPQuery(differentCaseQuery, response=None, useQueue=False)
 511         self.assertEqual(receivedResponse, differentCaseResponse)
 512
 513     def testLargeAnswer(self):
 514         """
 515         Cache: Check that we can cache (and retrieve) large answers
 516
 517         We should be able to get answers as large as 4096 bytes
 518         """
 519         numberOfQueries = 10
 520         name = 'large-answer.cache.tests.powerdns.com.'
 521         query = dns.message.make_query(name, 'TXT', 'IN')
 522         response = dns.message.make_response(query)
 523         # we prepare a large answer
 524         content = ""
 525         for i in range(44):
 526             if len(content) > 0:
 527                 content = content + ', '
 528             content = content + (str(i)*50)
 529         # pad up to 4096
 530         content = content + 'A'*42
 531
 532         rrset = dns.rrset.from_text(name,
 533                                     3600,
 534                                     dns.rdataclass.IN,
 535                                     dns.rdatatype.TXT,
 536                                     content)
 537         response.answer.append(rrset)
 538         self.assertEqual(len(response.to_wire()), 4096)
 539
 540         # first query to fill the cache
 541         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 542         self.assertTrue(receivedQuery)
 543         self.assertTrue(receivedResponse)
 544         receivedQuery.id = query.id
 545         self.assertEqual(query, receivedQuery)
 546         self.assertEqual(receivedResponse, response)
 547
 548         for _ in range(numberOfQueries):
 549             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 550             self.assertEqual(receivedResponse, response)
 551
 552         total = 0
 553         for key in self._responsesCounter:
 554             total += self._responsesCounter[key]
 555             TestCaching._responsesCounter[key] = 0
 556
 557         self.assertEqual(total, 1)
 558
 559         # TCP should not be cached
 560         # first query to fill the cache
 561         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 562         self.assertTrue(receivedQuery)
 563         self.assertTrue(receivedResponse)
 564         receivedQuery.id = query.id
 565         self.assertEqual(query, receivedQuery)
 566         self.assertEqual(receivedResponse, response)
 567
 568         for _ in range(numberOfQueries):
 569             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
 570             self.assertEqual(receivedResponse, response)
 571
 572         total = 0
 573         for key in self._responsesCounter:
 574             total += self._responsesCounter[key]
 575             TestCaching._responsesCounter[key] = 0
 576
 577         self.assertEqual(total, 1)
 578
 579     def testCacheDifferentCookies(self):
 580         """
 581         Cache: The content of cookies should be ignored by the cache
 582         """
 583         ttl = 600
 584         name = 'cache-different-cookies.cache.tests.powerdns.com.'
 585         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 586         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco])
 587         response = dns.message.make_response(query)
 588         rrset = dns.rrset.from_text(name,
 589                                     ttl,
 590                                     dns.rdataclass.IN,
 591                                     dns.rdatatype.AAAA,
 592                                     '::1')
 593         response.answer.append(rrset)
 594
 595         # first query to fill the cache
 596         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 597         self.assertTrue(receivedQuery)
 598         self.assertTrue(receivedResponse)
 599         receivedQuery.id = query.id
 600         self.assertEqual(query, receivedQuery)
 601         self.assertEqual(receivedResponse, response)
 602
 603         eco = cookiesoption.CookiesOption(b'badc0fee', b'badc0fee')
 604         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco])
 605         # second query should be served from the cache
 606         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 607         receivedResponse.id = response.id
 608         self.assertEqual(receivedResponse, response)
 609
 610     def testCacheCookies(self):
 611         """
 612         Cache: A query with a cookie should not match one without any cookie
 613         """
 614         ttl = 600
 615         name = 'cache-cookie.cache.tests.powerdns.com.'
 616         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 617         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, options=[eco])
 618         response = dns.message.make_response(query)
 619         rrset = dns.rrset.from_text(name,
 620                                     ttl,
 621                                     dns.rdataclass.IN,
 622                                     dns.rdatatype.A,
 623                                     '192.0.2.1')
 624         response.answer.append(rrset)
 625
 626         # first query to fill the cache
 627         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 628         self.assertTrue(receivedQuery)
 629         self.assertTrue(receivedResponse)
 630         receivedQuery.id = query.id
 631         self.assertEqual(query, receivedQuery)
 632         self.assertEqual(receivedResponse, response)
 633
 634         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, options=[])
 635         response = dns.message.make_response(query)
 636         rrset = dns.rrset.from_text(name,
 637                                     ttl,
 638                                     dns.rdataclass.IN,
 639                                     dns.rdatatype.A,
 640                                     '127.0.0.1')
 641         response.answer.append(rrset)
 642         # second query should NOT be served from the cache
 643         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 644         self.assertTrue(receivedQuery)
 645         self.assertTrue(receivedResponse)
 646         receivedQuery.id = query.id
 647         self.assertEqual(query, receivedQuery)
 648         self.assertEqual(receivedResponse, response)
 649
 650     def testCacheSameCookieDifferentECS(self):
 651         """
 652         Cache: The content of cookies should be ignored by the cache but not the ECS one
 653         """
 654         ttl = 600
 655         name = 'cache-different-cookies-different-ecs.cache.tests.powerdns.com.'
 656         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 657         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
 658         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 659         response = dns.message.make_response(query)
 660         rrset = dns.rrset.from_text(name,
 661                                     ttl,
 662                                     dns.rdataclass.IN,
 663                                     dns.rdatatype.AAAA,
 664                                     '::1')
 665         response.answer.append(rrset)
 666
 667         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 668         self.assertTrue(receivedQuery)
 669         self.assertTrue(receivedResponse)
 670         receivedQuery.id = query.id
 671         self.assertEqual(query, receivedQuery)
 672         self.assertEqual(receivedResponse, response)
 673
 674         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 675         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.2', 32)
 676         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 677         response = dns.message.make_response(query)
 678         rrset = dns.rrset.from_text(name,
 679                                     ttl,
 680                                     dns.rdataclass.IN,
 681                                     dns.rdatatype.AAAA,
 682                                     '::1')
 683         response.answer.append(rrset)
 684
 685         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 686         self.assertTrue(receivedQuery)
 687         self.assertTrue(receivedResponse)
 688         receivedQuery.id = query.id
 689         self.assertEqual(query, receivedQuery)
 690         self.assertEqual(receivedResponse, response)
 691
 692 class TestCachingHashingOptions(DNSDistTest):
 693
 694     _config_template = """
 695     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, cookieHashing=true, skipOptions={8}})
 696     getPool(""):setCache(pc)
 697     newServer{address="127.0.0.1:%d"}
 698     """
 699
 700     def testCacheDifferentECSSameCookie(self):
 701         """
 702         Cache: ECS should be ignored by the cache even if cookie is present
 703         """
 704         ttl = 600
 705         name = 'cache-different-ecs.cache.tests.powerdns.com.'
 706         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 707         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.2', 32)
 708         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 709         response = dns.message.make_response(query)
 710         rrset = dns.rrset.from_text(name,
 711                                     ttl,
 712                                     dns.rdataclass.IN,
 713                                     dns.rdatatype.AAAA,
 714                                     '::1')
 715         response.answer.append(rrset)
 716
 717         # first query to fill the cache
 718         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 719         self.assertTrue(receivedQuery)
 720         self.assertTrue(receivedResponse)
 721         receivedQuery.id = query.id
 722         self.assertEqual(query, receivedQuery)
 723         self.assertEqual(receivedResponse, response)
 724
 725         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 726         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
 727         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 728         # second query should be served from the cache
 729         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 730         receivedResponse.id = response.id
 731         self.assertEqual(receivedResponse, response)
 732
 733 class TestCachingHashingCookies(DNSDistTest):
 734
 735     _config_template = """
 736     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, cookieHashing=true})
 737     getPool(""):setCache(pc)
 738     newServer{address="127.0.0.1:%d"}
 739     """
 740
 741     def testCached(self):
 742         """
 743         Cache: Served from cache
 744
 745         dnsdist is configured to cache entries, we are sending several
 746         identical requests and checking that the backend only receive
 747         the first one.
 748         """
 749         numberOfQueries = 10
 750         name = 'cached.cache.tests.powerdns.com.'
 751         query = dns.message.make_query(name, 'AAAA', 'IN')
 752         response = dns.message.make_response(query)
 753         rrset = dns.rrset.from_text(name,
 754                                     3600,
 755                                     dns.rdataclass.IN,
 756                                     dns.rdatatype.AAAA,
 757                                     '::1')
 758         response.answer.append(rrset)
 759
 760         # first query to fill the cache
 761         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 762         self.assertTrue(receivedQuery)
 763         self.assertTrue(receivedResponse)
 764         receivedQuery.id = query.id
 765         self.assertEqual(query, receivedQuery)
 766         self.assertEqual(receivedResponse, response)
 767
 768         for _ in range(numberOfQueries):
 769             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 770             self.assertEqual(receivedResponse, response)
 771
 772         total = 0
 773         for key in self._responsesCounter:
 774             total += self._responsesCounter[key]
 775             TestCaching._responsesCounter[key] = 0
 776
 777         self.assertEqual(total, 1)
 778
 779         # TCP should not be cached
 780         # first query to fill the cache
 781         (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
 782         self.assertTrue(receivedQuery)
 783         self.assertTrue(receivedResponse)
 784         receivedQuery.id = query.id
 785         self.assertEqual(query, receivedQuery)
 786         self.assertEqual(receivedResponse, response)
 787
 788         for _ in range(numberOfQueries):
 789             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
 790             self.assertEqual(receivedResponse, response)
 791
 792         total = 0
 793         for key in self._responsesCounter:
 794             total += self._responsesCounter[key]
 795             TestCaching._responsesCounter[key] = 0
 796
 797         self.assertEqual(total, 1)
 798
 799
 800     def testCacheDifferentCookies(self):
 801         """
 802         Cache: The content of cookies should NOT be ignored by the cache (cookieHashing is set)
 803         """
 804         ttl = 600
 805         name = 'cache-different-cookies.cache-cookie-hashing.tests.powerdns.com.'
 806         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 807         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco])
 808         response = dns.message.make_response(query)
 809         rrset = dns.rrset.from_text(name,
 810                                     ttl,
 811                                     dns.rdataclass.IN,
 812                                     dns.rdatatype.AAAA,
 813                                     '::1')
 814         response.answer.append(rrset)
 815
 816         # first query to fill the cache
 817         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 818         self.assertTrue(receivedQuery)
 819         self.assertTrue(receivedResponse)
 820         receivedQuery.id = query.id
 821         self.assertEqual(query, receivedQuery)
 822         self.assertEqual(receivedResponse, response)
 823
 824         eco = cookiesoption.CookiesOption(b'badc0fee', b'badc0fee')
 825         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco])
 826         differentResponse = dns.message.make_response(query)
 827         rrset = dns.rrset.from_text(name,
 828                                     ttl,
 829                                     dns.rdataclass.IN,
 830                                     dns.rdatatype.AAAA,
 831                                     '2001:DB8::1')
 832         differentResponse.answer.append(rrset)
 833         # second query should NOT be served from the cache
 834         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, differentResponse)
 835         self.assertTrue(receivedQuery)
 836         self.assertTrue(receivedResponse)
 837         receivedQuery.id = query.id
 838         self.assertEqual(query, receivedQuery)
 839         self.assertEqual(receivedResponse, differentResponse)
 840         self.assertNotEqual(receivedResponse, response)
 841
 842     def testCacheCookies(self):
 843         """
 844         Cache: A query with a cookie should not match one without any cookie (cookieHashing=true)
 845         """
 846         ttl = 600
 847         name = 'cache-cookie.cache-cookie-hashing.tests.powerdns.com.'
 848         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 849         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, options=[eco])
 850         response = dns.message.make_response(query)
 851         rrset = dns.rrset.from_text(name,
 852                                     ttl,
 853                                     dns.rdataclass.IN,
 854                                     dns.rdatatype.A,
 855                                     '192.0.2.1')
 856         response.answer.append(rrset)
 857
 858         # first query to fill the cache
 859         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 860         self.assertTrue(receivedQuery)
 861         self.assertTrue(receivedResponse)
 862         receivedQuery.id = query.id
 863         self.assertEqual(query, receivedQuery)
 864         self.assertEqual(receivedResponse, response)
 865
 866         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096, options=[])
 867         response = dns.message.make_response(query)
 868         rrset = dns.rrset.from_text(name,
 869                                     ttl,
 870                                     dns.rdataclass.IN,
 871                                     dns.rdatatype.A,
 872                                     '127.0.0.1')
 873         response.answer.append(rrset)
 874         # second query should NOT be served from the cache
 875         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 876         self.assertTrue(receivedQuery)
 877         self.assertTrue(receivedResponse)
 878         receivedQuery.id = query.id
 879         self.assertEqual(query, receivedQuery)
 880         self.assertEqual(receivedResponse, response)
 881
 882     def testCacheSameCookieDifferentECS(self):
 883         """
 884         Cache: The content of cookies should NOT be ignored by the cache (cookieHashing=true), even with ECS there
 885         """
 886         ttl = 600
 887         name = 'cache-different-cookies-different-ecs.cache-cookie-hashing.tests.powerdns.com.'
 888         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 889         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
 890         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 891         response = dns.message.make_response(query)
 892         rrset = dns.rrset.from_text(name,
 893                                     ttl,
 894                                     dns.rdataclass.IN,
 895                                     dns.rdatatype.AAAA,
 896                                     '::1')
 897         response.answer.append(rrset)
 898
 899         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 900         self.assertTrue(receivedQuery)
 901         self.assertTrue(receivedResponse)
 902         receivedQuery.id = query.id
 903         self.assertEqual(query, receivedQuery)
 904         self.assertEqual(receivedResponse, response)
 905
 906         eco = cookiesoption.CookiesOption(b'deadbeef', b'deadbeef')
 907         ecso = clientsubnetoption.ClientSubnetOption('192.0.2.2', 32)
 908         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, payload=4096, options=[eco,ecso])
 909         response = dns.message.make_response(query)
 910         rrset = dns.rrset.from_text(name,
 911                                     ttl,
 912                                     dns.rdataclass.IN,
 913                                     dns.rdatatype.AAAA,
 914                                     '::1')
 915         response.answer.append(rrset)
 916
 917         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 918         self.assertTrue(receivedQuery)
 919         self.assertTrue(receivedResponse)
 920         receivedQuery.id = query.id
 921         self.assertEqual(query, receivedQuery)
 922         self.assertEqual(receivedResponse, response)
 923
 924 class TestTempFailureCacheTTLAction(DNSDistTest):
 925
 926     _extraStartupSleep = 1
 927     _config_template = """
 928     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
 929     getPool(""):setCache(pc)
 930     addAction("servfail.cache.tests.powerdns.com.", SetTempFailureCacheTTLAction(1))
 931     newServer{address="127.0.0.1:%d"}
 932     """
 933
 934     def testTempFailureCacheTTLAction(self):
 935         """
 936         Cache: When a TempFailure TTL is set, it should be honored
 937
 938         dnsdist is configured to cache packets, plus a specific qname is
 939         set up with a lower TempFailure Cache TTL. we are sending one request
 940         (cache miss) and verify that the cache is hit for the following query,
 941         but the TTL then expires before the larger "good" packetcache TTL.
 942         """
 943         name = 'servfail.cache.tests.powerdns.com.'
 944         query = dns.message.make_query(name, 'AAAA', 'IN')
 945         response = dns.message.make_response(query)
 946         response.set_rcode(dns.rcode.SERVFAIL)
 947
 948         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 949         self.assertTrue(receivedQuery)
 950         self.assertTrue(receivedResponse)
 951         receivedQuery.id = query.id
 952         self.assertEqual(query, receivedQuery)
 953         self.assertEqual(receivedResponse, response)
 954
 955         # next query should hit the cache
 956         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
 957         self.assertFalse(receivedQuery)
 958         self.assertTrue(receivedResponse)
 959         self.assertEqual(receivedResponse, response)
 960
 961         # now we wait a bit for the Failure-Cache TTL to expire
 962         time.sleep(2)
 963
 964         # next query should NOT hit the cache
 965         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 966         self.assertTrue(receivedQuery)
 967         self.assertTrue(receivedResponse)
 968         self.assertEqual(receivedResponse, response)
 969
 970 class TestCachingWithExistingEDNS(DNSDistTest):
 971
 972     _config_template = """
 973     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
 974     getPool(""):setCache(pc)
 975     newServer{address="127.0.0.1:%d"}
 976     """
 977     def testCacheWithEDNS(self):
 978         """
 979         Cache: Cache should not match different EDNS value
 980
 981         dnsdist is configured to cache entries, we are sending one request
 982         (cache miss) and verify that the same one with a different EDNS UDP
 983         Payload size is not served from the cache.
 984         """
 985         misses = 0
 986         name = 'cachedifferentedns.cache.tests.powerdns.com.'
 987         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=512)
 988         response = dns.message.make_response(query)
 989         rrset = dns.rrset.from_text(name,
 990                                     3600,
 991                                     dns.rdataclass.IN,
 992                                     dns.rdatatype.A,
 993                                     '127.0.0.1')
 994         response.answer.append(rrset)
 995
 996         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
 997         self.assertTrue(receivedQuery)
 998         self.assertTrue(receivedResponse)
 999         receivedQuery.id = query.id
1000         self.assertEqual(query, receivedQuery)
1001         self.assertEqual(response, receivedResponse)
1002         misses += 1
1003
1004         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096)
1005         response = dns.message.make_response(query)
1006         rrset = dns.rrset.from_text(name,
1007                                     3600,
1008                                     dns.rdataclass.IN,
1009                                     dns.rdatatype.A,
1010                                     '127.0.0.1')
1011         response.answer.append(rrset)
1012
1013         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1014         self.assertTrue(receivedQuery)
1015         self.assertTrue(receivedResponse)
1016         receivedQuery.id = query.id
1017         self.assertEqual(query, receivedQuery)
1018         self.assertEqual(response, receivedResponse)
1019         misses += 1
1020
1021         total = 0
1022         for key in self._responsesCounter:
1023             total += self._responsesCounter[key]
1024
1025         self.assertEqual(total, misses)
1026
1027 class TestCachingCacheFull(DNSDistTest):
1028
1029     _config_template = """
1030     pc = newPacketCache(1, {maxTTL=86400, minTTL=1, numberOfShards=1})
1031     getPool(""):setCache(pc)
1032     newServer{address="127.0.0.1:%d"}
1033     """
1034     def testCacheFull(self):
1035         """
1036         Cache: No new entries are cached when the cache is full
1037
1038         """
1039         misses = 0
1040         name = 'cachenotfullyet.cache.tests.powerdns.com.'
1041         query = dns.message.make_query(name, 'A', 'IN')
1042         response = dns.message.make_response(query)
1043         rrset = dns.rrset.from_text(name,
1044                                     3600,
1045                                     dns.rdataclass.IN,
1046                                     dns.rdatatype.A,
1047                                     '127.0.0.1')
1048         response.answer.append(rrset)
1049
1050         # Miss
1051         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1052         self.assertTrue(receivedQuery)
1053         self.assertTrue(receivedResponse)
1054         receivedQuery.id = query.id
1055         self.assertEqual(query, receivedQuery)
1056         self.assertEqual(response, receivedResponse)
1057         misses += 1
1058
1059         # next queries should hit the cache
1060         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1061         self.assertEqual(receivedResponse, response)
1062
1063         # ok, now the cache is full, send another query
1064         name = 'cachefull.cache.tests.powerdns.com.'
1065         query = dns.message.make_query(name, 'AAAA', 'IN')
1066         response = dns.message.make_response(query)
1067         rrset = dns.rrset.from_text(name,
1068                                     3600,
1069                                     dns.rdataclass.IN,
1070                                     dns.rdatatype.AAAA,
1071                                     '::1')
1072         response.answer.append(rrset)
1073
1074         # Miss
1075         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1076         self.assertTrue(receivedQuery)
1077         self.assertTrue(receivedResponse)
1078         receivedQuery.id = query.id
1079         self.assertEqual(query, receivedQuery)
1080         self.assertEqual(response, receivedResponse)
1081         misses += 1
1082
1083         # next queries should NOT hit the cache
1084         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1085         self.assertTrue(receivedQuery)
1086         self.assertTrue(receivedResponse)
1087         receivedQuery.id = query.id
1088         self.assertEqual(query, receivedQuery)
1089         self.assertEqual(response, receivedResponse)
1090         misses += 1
1091
1092         total = 0
1093         for key in self._responsesCounter:
1094             total += self._responsesCounter[key]
1095
1096         self.assertEqual(total, misses)
1097
1098 class TestCachingNoStale(DNSDistTest):
1099
1100     _consoleKey = DNSDistTest.generateConsoleKey()
1101     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1102     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
1103     _config_template = """
1104     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
1105     getPool(""):setCache(pc)
1106     setKey("%s")
1107     controlSocket("127.0.0.1:%d")
1108     newServer{address="127.0.0.1:%d"}
1109     """
1110     def testCacheNoStale(self):
1111         """
1112         Cache: Cache entry, set backend down, we should not get a stale entry
1113
1114         """
1115         ttl = 2
1116         name = 'nostale.cache.tests.powerdns.com.'
1117         query = dns.message.make_query(name, 'A', 'IN')
1118         response = dns.message.make_response(query)
1119         rrset = dns.rrset.from_text(name,
1120                                     ttl,
1121                                     dns.rdataclass.IN,
1122                                     dns.rdatatype.A,
1123                                     '127.0.0.1')
1124         response.answer.append(rrset)
1125
1126         # Miss
1127         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1128         self.assertTrue(receivedQuery)
1129         self.assertTrue(receivedResponse)
1130         receivedQuery.id = query.id
1131         self.assertEqual(query, receivedQuery)
1132         self.assertEqual(response, receivedResponse)
1133
1134         # next queries should hit the cache
1135         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1136         self.assertEqual(receivedResponse, response)
1137
1138         # ok, we mark the backend as down
1139         self.sendConsoleCommand("getServer(0):setDown()")
1140         # and we wait for the entry to expire
1141         time.sleep(ttl + 1)
1142
1143         # we should NOT get a cached, stale, entry
1144         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1145         self.assertEqual(receivedResponse, None)
1146
1147
1148 class TestCachingStale(DNSDistTest):
1149
1150     _consoleKey = DNSDistTest.generateConsoleKey()
1151     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1152     _staleCacheTTL = 60
1153     _config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
1154     _config_template = """
1155     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=%d})
1156     getPool(""):setCache(pc)
1157     setStaleCacheEntriesTTL(600)
1158     setKey("%s")
1159     controlSocket("127.0.0.1:%d")
1160     newServer{address="127.0.0.1:%d"}
1161     """
1162     def testCacheStale(self):
1163         """
1164         Cache: Cache entry, set backend down, get stale entry
1165
1166         """
1167         misses = 0
1168         ttl = 2
1169         name = 'stale.cache.tests.powerdns.com.'
1170         query = dns.message.make_query(name, 'A', 'IN')
1171         response = dns.message.make_response(query)
1172         rrset = dns.rrset.from_text(name,
1173                                     ttl,
1174                                     dns.rdataclass.IN,
1175                                     dns.rdatatype.A,
1176                                     '127.0.0.1')
1177         response.answer.append(rrset)
1178
1179         # Miss
1180         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1181         self.assertTrue(receivedQuery)
1182         self.assertTrue(receivedResponse)
1183         receivedQuery.id = query.id
1184         self.assertEqual(query, receivedQuery)
1185         self.assertEqual(response, receivedResponse)
1186         misses += 1
1187
1188         # next queries should hit the cache
1189         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1190         self.assertEqual(receivedResponse, response)
1191
1192         # ok, we mark the backend as down
1193         self.sendConsoleCommand("getServer(0):setDown()")
1194         # and we wait for the entry to expire
1195         time.sleep(ttl + 1)
1196
1197         # we should get a cached, stale, entry
1198         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1199         self.assertEqual(receivedResponse, response)
1200         for an in receivedResponse.answer:
1201             self.assertEqual(an.ttl, self._staleCacheTTL)
1202
1203         total = 0
1204         for key in self._responsesCounter:
1205             total += self._responsesCounter[key]
1206
1207         self.assertEqual(total, misses)
1208
1209 class TestCachingStaleExpunged(DNSDistTest):
1210
1211     _consoleKey = DNSDistTest.generateConsoleKey()
1212     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1213     _staleCacheTTL = 60
1214     _config_params = ['_staleCacheTTL', '_consoleKeyB64', '_consolePort', '_testServerPort']
1215     _config_template = """
1216     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=%d})
1217     getPool(""):setCache(pc)
1218     setStaleCacheEntriesTTL(600)
1219     -- try to remove all expired entries
1220     setCacheCleaningPercentage(100)
1221     -- clean the cache every second
1222     setCacheCleaningDelay(1)
1223     setKey("%s")
1224     controlSocket("127.0.0.1:%d")
1225     newServer{address="127.0.0.1:%d"}
1226     """
1227     def testCacheStale(self):
1228         """
1229         Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, get no entry
1230         """
1231         misses = 0
1232         drops = 0
1233         ttl = 2
1234         name = 'stale-but-expunged.cache.tests.powerdns.com.'
1235         query = dns.message.make_query(name, 'A', 'IN')
1236         response = dns.message.make_response(query)
1237         rrset = dns.rrset.from_text(name,
1238                                     ttl,
1239                                     dns.rdataclass.IN,
1240                                     dns.rdatatype.A,
1241                                     '127.0.0.1')
1242         response.answer.append(rrset)
1243
1244         # Miss
1245         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1246         self.assertTrue(receivedQuery)
1247         self.assertTrue(receivedResponse)
1248         receivedQuery.id = query.id
1249         self.assertEqual(query, receivedQuery)
1250         self.assertEqual(response, receivedResponse)
1251         misses += 1
1252         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
1253
1254         # next queries should hit the cache
1255         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1256         self.assertEqual(receivedResponse, response)
1257         # the cache should have one entry
1258         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
1259         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
1260
1261         # ok, we mark the backend as down
1262         self.sendConsoleCommand("getServer(0):setDown()")
1263         # and we wait for the entry to expire
1264         time.sleep(ttl + 1)
1265         # wait a bit more to be sure that the cache cleaning algo has been run
1266         time.sleep(1)
1267         # the cache should be empty now
1268         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 0)
1269
1270         # we should get a DROP (backend is down, nothing in the cache anymore)
1271         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1272         self.assertEqual(receivedResponse, None)
1273         drops += 1
1274
1275         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), misses + drops)
1276         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
1277
1278         total = 0
1279         for key in self._responsesCounter:
1280             total += self._responsesCounter[key]
1281
1282         self.assertEqual(total, misses)
1283
1284 class TestCachingStaleExpungePrevented(DNSDistTest):
1285
1286     _consoleKey = DNSDistTest.generateConsoleKey()
1287     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1288     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
1289     _config_template = """
1290     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=0, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, ecsParsing=false, keepStaleData=true})
1291     getPool(""):setCache(pc)
1292     setStaleCacheEntriesTTL(600)
1293     -- try to remove all expired entries
1294     setCacheCleaningPercentage(100)
1295     -- clean the cache every second
1296     setCacheCleaningDelay(1)
1297     setKey("%s")
1298     controlSocket("127.0.0.1:%d")
1299     newServer{address="127.0.0.1:%d"}
1300     """
1301     def testCacheStale(self):
1302         """
1303         Cache: Cache entry, set backend down, wait for the cache cleaning to run and remove the entry, still get a cache HIT because the stale entry was not removed
1304         """
1305         misses = 0
1306         ttl = 2
1307         name = 'stale-not-expunged.cache.tests.powerdns.com.'
1308         query = dns.message.make_query(name, 'A', 'IN')
1309         response = dns.message.make_response(query)
1310         rrset = dns.rrset.from_text(name,
1311                                     ttl,
1312                                     dns.rdataclass.IN,
1313                                     dns.rdatatype.A,
1314                                     '127.0.0.1')
1315         response.answer.append(rrset)
1316
1317         # Miss
1318         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1319         self.assertTrue(receivedQuery)
1320         self.assertTrue(receivedResponse)
1321         receivedQuery.id = query.id
1322         self.assertEqual(query, receivedQuery)
1323         self.assertEqual(response, receivedResponse)
1324         misses += 1
1325         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
1326
1327         # next queries should hit the cache
1328         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1329         self.assertEqual(receivedResponse, response)
1330         # the cache should have one entry
1331         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
1332         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 1)
1333
1334         # ok, we mark the backend as down
1335         self.sendConsoleCommand("getServer(0):setDown()")
1336         # and we wait for the entry to expire
1337         time.sleep(ttl + 1)
1338         # wait a bit more to be sure that the cache cleaning algo has been run
1339         time.sleep(1)
1340         # the cache should NOT be empty because the removal of the expired entry should have been prevented
1341         # since all backends for this pool are down
1342         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"entries\"]").strip("\n")), 1)
1343
1344         # we should get a HIT
1345         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1346         self.assertEqual(receivedResponse, response)
1347
1348         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"misses\"]").strip("\n")), 1)
1349         self.assertEqual(int(self.sendConsoleCommand("getPool(\"\"):getCache():getStats()[\"hits\"]").strip("\n")), 2)
1350
1351         total = 0
1352         for key in self._responsesCounter:
1353             total += self._responsesCounter[key]
1354
1355         self.assertEqual(total, misses)
1356
1357 class TestCacheManagement(DNSDistTest):
1358
1359     _consoleKey = DNSDistTest.generateConsoleKey()
1360     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
1361     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
1362     _config_template = """
1363     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
1364     getPool(""):setCache(pc)
1365     setKey("%s")
1366     controlSocket("127.0.0.1:%d")
1367     newServer{address="127.0.0.1:%d"}
1368     """
1369     def testCacheExpunge(self):
1370         """
1371         Cache: Expunge
1372
1373         """
1374         misses = 0
1375         ttl = 600
1376         name = 'expunge.cache.tests.powerdns.com.'
1377         query = dns.message.make_query(name, 'A', 'IN')
1378         response = dns.message.make_response(query)
1379         rrset = dns.rrset.from_text(name,
1380                                     ttl,
1381                                     dns.rdataclass.IN,
1382                                     dns.rdatatype.A,
1383                                     '127.0.0.1')
1384         response.answer.append(rrset)
1385
1386         # Miss
1387         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1388         self.assertTrue(receivedQuery)
1389         self.assertTrue(receivedResponse)
1390         receivedQuery.id = query.id
1391         self.assertEqual(query, receivedQuery)
1392         self.assertEqual(response, receivedResponse)
1393         misses += 1
1394
1395         # next queries should hit the cache
1396         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1397         self.assertEqual(receivedResponse, response)
1398
1399         # remove cached entries
1400         self.sendConsoleCommand("getPool(\"\"):getCache():expunge(0)")
1401
1402         # Miss
1403         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1404         self.assertTrue(receivedQuery)
1405         self.assertTrue(receivedResponse)
1406         receivedQuery.id = query.id
1407         self.assertEqual(query, receivedQuery)
1408         self.assertEqual(response, receivedResponse)
1409         misses += 1
1410
1411         # next queries should hit the cache again
1412         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1413         self.assertEqual(receivedResponse, response)
1414
1415         total = 0
1416         for key in self._responsesCounter:
1417             total += self._responsesCounter[key]
1418
1419         self.assertEqual(total, misses)
1420
1421     def testCacheExpungeByName(self):
1422         """
1423         Cache: Expunge by name
1424
1425         """
1426         misses = 0
1427         ttl = 600
1428         name = 'expungebyname.cache.tests.powerdns.com.'
1429         query = dns.message.make_query(name, 'A', 'IN')
1430         response = dns.message.make_response(query)
1431         rrset = dns.rrset.from_text(name,
1432                                     ttl,
1433                                     dns.rdataclass.IN,
1434                                     dns.rdatatype.A,
1435                                     '127.0.0.1')
1436         response.answer.append(rrset)
1437
1438         name2 = 'expungebynameother.cache.tests.powerdns.com.'
1439         query2 = dns.message.make_query(name2, 'A', 'IN')
1440         response2 = dns.message.make_response(query2)
1441         rrset2 = dns.rrset.from_text(name2,
1442                                      ttl,
1443                                      dns.rdataclass.IN,
1444                                      dns.rdatatype.A,
1445                                      '127.0.0.1')
1446         response2.answer.append(rrset2)
1447
1448         # Miss
1449         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1450         self.assertTrue(receivedQuery)
1451         self.assertTrue(receivedResponse)
1452         receivedQuery.id = query.id
1453         self.assertEqual(query, receivedQuery)
1454         self.assertEqual(response, receivedResponse)
1455         misses += 1
1456
1457         # next queries should hit the cache
1458         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1459         self.assertEqual(receivedResponse, response)
1460
1461         # cache another entry
1462         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1463         self.assertTrue(receivedQuery)
1464         self.assertTrue(receivedResponse)
1465         receivedQuery.id = query2.id
1466         self.assertEqual(query2, receivedQuery)
1467         self.assertEqual(response2, receivedResponse)
1468         misses += 1
1469
1470         # queries for name and name 2 should hit the cache
1471         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1472         self.assertEqual(receivedResponse, response)
1473
1474         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1475         self.assertEqual(receivedResponse, response2)
1476
1477         # remove cached entries from name
1478         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"" + name + "\"))")
1479
1480         # Miss for name
1481         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1482         self.assertTrue(receivedQuery)
1483         self.assertTrue(receivedResponse)
1484         receivedQuery.id = query.id
1485         self.assertEqual(query, receivedQuery)
1486         self.assertEqual(response, receivedResponse)
1487         misses += 1
1488
1489         # next queries for name should hit the cache again
1490         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1491         self.assertEqual(receivedResponse, response)
1492
1493         # queries for name2 should still hit the cache
1494         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1495         self.assertEqual(receivedResponse, response2)
1496
1497         total = 0
1498         for key in self._responsesCounter:
1499             total += self._responsesCounter[key]
1500
1501         self.assertEqual(total, misses)
1502
1503     def testCacheExpungeByNameAndType(self):
1504         """
1505         Cache: Expunge by name and type
1506
1507         """
1508         misses = 0
1509         ttl = 600
1510         name = 'expungebynameandtype.cache.tests.powerdns.com.'
1511         query = dns.message.make_query(name, 'A', 'IN')
1512         response = dns.message.make_response(query)
1513         rrset = dns.rrset.from_text(name,
1514                                     ttl,
1515                                     dns.rdataclass.IN,
1516                                     dns.rdatatype.A,
1517                                     '127.0.0.1')
1518         response.answer.append(rrset)
1519
1520         query2 = dns.message.make_query(name, 'AAAA', 'IN')
1521         response2 = dns.message.make_response(query2)
1522         rrset2 = dns.rrset.from_text(name,
1523                                      ttl,
1524                                      dns.rdataclass.IN,
1525                                      dns.rdatatype.AAAA,
1526                                      '::1')
1527         response2.answer.append(rrset2)
1528
1529         # Miss
1530         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1531         self.assertTrue(receivedQuery)
1532         self.assertTrue(receivedResponse)
1533         receivedQuery.id = query.id
1534         self.assertEqual(query, receivedQuery)
1535         self.assertEqual(response, receivedResponse)
1536         misses += 1
1537
1538         # next queries should hit the cache
1539         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1540         self.assertEqual(receivedResponse, response)
1541
1542         # cache another entry
1543         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1544         self.assertTrue(receivedQuery)
1545         self.assertTrue(receivedResponse)
1546         receivedQuery.id = query2.id
1547         self.assertEqual(query2, receivedQuery)
1548         self.assertEqual(response2, receivedResponse)
1549         misses += 1
1550
1551         # queries for name A and AAAA should hit the cache
1552         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1553         self.assertEqual(receivedResponse, response)
1554
1555         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1556         self.assertEqual(receivedResponse, response2)
1557
1558         # remove cached entries from name A
1559         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"" + name + "\"), DNSQType.A)")
1560
1561         # Miss for name A
1562         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1563         self.assertTrue(receivedQuery)
1564         self.assertTrue(receivedResponse)
1565         receivedQuery.id = query.id
1566         self.assertEqual(query, receivedQuery)
1567         self.assertEqual(response, receivedResponse)
1568         misses += 1
1569
1570         # next queries for name A should hit the cache again
1571         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1572         self.assertEqual(receivedResponse, response)
1573
1574         # queries for name AAAA should still hit the cache
1575         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1576         self.assertEqual(receivedResponse, response2)
1577
1578         total = 0
1579         for key in self._responsesCounter:
1580             total += self._responsesCounter[key]
1581         self.assertEqual(total, misses)
1582
1583     def testCacheExpungeByNameAndSuffix(self):
1584         """
1585         Cache: Expunge by name
1586
1587         """
1588         misses = 0
1589         ttl = 600
1590         name = 'expungebyname.suffix.cache.tests.powerdns.com.'
1591         query = dns.message.make_query(name, 'A', 'IN')
1592         response = dns.message.make_response(query)
1593         rrset = dns.rrset.from_text(name,
1594                                     ttl,
1595                                     dns.rdataclass.IN,
1596                                     dns.rdatatype.A,
1597                                     '127.0.0.1')
1598         response.answer.append(rrset)
1599
1600         name2 = 'expungebyname.suffixother.cache.tests.powerdns.com.'
1601         query2 = dns.message.make_query(name2, 'A', 'IN')
1602         response2 = dns.message.make_response(query2)
1603         rrset2 = dns.rrset.from_text(name2,
1604                                      ttl,
1605                                      dns.rdataclass.IN,
1606                                      dns.rdatatype.A,
1607                                      '127.0.0.1')
1608         response2.answer.append(rrset2)
1609
1610         # Miss
1611         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1612         self.assertTrue(receivedQuery)
1613         self.assertTrue(receivedResponse)
1614         receivedQuery.id = query.id
1615         self.assertEqual(query, receivedQuery)
1616         self.assertEqual(response, receivedResponse)
1617         misses += 1
1618
1619         # next queries should hit the cache
1620         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1621         self.assertEqual(receivedResponse, response)
1622
1623         # cache another entry
1624         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1625         self.assertTrue(receivedQuery)
1626         self.assertTrue(receivedResponse)
1627         receivedQuery.id = query2.id
1628         self.assertEqual(query2, receivedQuery)
1629         self.assertEqual(response2, receivedResponse)
1630         misses += 1
1631
1632         # queries for name and name 2 should hit the cache
1633         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1634         self.assertEqual(receivedResponse, response)
1635
1636         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1637         self.assertEqual(receivedResponse, response2)
1638
1639         # remove cached entries from name
1640         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"suffix.cache.tests.powerdns.com.\"), DNSQType.ANY, true)")
1641
1642         # Miss for name
1643         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1644         self.assertTrue(receivedQuery)
1645         self.assertTrue(receivedResponse)
1646         receivedQuery.id = query.id
1647         self.assertEqual(query, receivedQuery)
1648         self.assertEqual(response, receivedResponse)
1649         misses += 1
1650
1651         # next queries for name should hit the cache again
1652         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1653         self.assertEqual(receivedResponse, response)
1654
1655         # queries for name2 should still hit the cache
1656         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1657         self.assertEqual(receivedResponse, response2)
1658
1659         total = 0
1660         for key in self._responsesCounter:
1661             total += self._responsesCounter[key]
1662
1663         self.assertEqual(total, misses)
1664
1665     def testCacheExpungeByNameAndTypeAndSuffix(self):
1666         """
1667         Cache: Expunge by name and type
1668
1669         """
1670         misses = 0
1671         ttl = 600
1672         name = 'expungebynameandtype.suffixtype.cache.tests.powerdns.com.'
1673         query = dns.message.make_query(name, 'A', 'IN')
1674         response = dns.message.make_response(query)
1675         rrset = dns.rrset.from_text(name,
1676                                     ttl,
1677                                     dns.rdataclass.IN,
1678                                     dns.rdatatype.A,
1679                                     '127.0.0.1')
1680         response.answer.append(rrset)
1681
1682         query2 = dns.message.make_query(name, 'AAAA', 'IN')
1683         response2 = dns.message.make_response(query2)
1684         rrset2 = dns.rrset.from_text(name,
1685                                      ttl,
1686                                      dns.rdataclass.IN,
1687                                      dns.rdatatype.AAAA,
1688                                      '::1')
1689         response2.answer.append(rrset2)
1690
1691         # Miss
1692         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1693         self.assertTrue(receivedQuery)
1694         self.assertTrue(receivedResponse)
1695         receivedQuery.id = query.id
1696         self.assertEqual(query, receivedQuery)
1697         self.assertEqual(response, receivedResponse)
1698         misses += 1
1699
1700         # next queries should hit the cache
1701         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1702         self.assertEqual(receivedResponse, response)
1703
1704         # cache another entry
1705         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
1706         self.assertTrue(receivedQuery)
1707         self.assertTrue(receivedResponse)
1708         receivedQuery.id = query2.id
1709         self.assertEqual(query2, receivedQuery)
1710         self.assertEqual(response2, receivedResponse)
1711         misses += 1
1712
1713         # queries for name A and AAAA should hit the cache
1714         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1715         self.assertEqual(receivedResponse, response)
1716
1717         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1718         self.assertEqual(receivedResponse, response2)
1719
1720         # remove cached entries from name A
1721         self.sendConsoleCommand("getPool(\"\"):getCache():expungeByName(newDNSName(\"suffixtype.cache.tests.powerdns.com.\"), DNSQType.A, true)")
1722
1723         # Miss for name A
1724         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1725         self.assertTrue(receivedQuery)
1726         self.assertTrue(receivedResponse)
1727         receivedQuery.id = query.id
1728         self.assertEqual(query, receivedQuery)
1729         self.assertEqual(response, receivedResponse)
1730         misses += 1
1731
1732         # next queries for name A should hit the cache again
1733         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1734         self.assertEqual(receivedResponse, response)
1735
1736         # queries for name AAAA should still hit the cache
1737         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
1738         self.assertEqual(receivedResponse, response2)
1739
1740         total = 0
1741         for key in self._responsesCounter:
1742             total += self._responsesCounter[key]
1743         self.assertEqual(total, misses)
1744
1745 class TestCachingTTL(DNSDistTest):
1746
1747     _maxCacheTTL = 86400
1748     _minCacheTTL = 600
1749     _config_params = ['_maxCacheTTL', '_minCacheTTL', '_testServerPort']
1750     _config_template = """
1751     pc = newPacketCache(1000, {maxTTL=%d, minTTL=%d})
1752     getPool(""):setCache(pc)
1753     newServer{address="127.0.0.1:%d"}
1754     """
1755     def testCacheShortTTL(self):
1756         """
1757         Cache: Entries with a TTL shorter than minTTL
1758
1759         """
1760         misses = 0
1761         ttl = 60
1762         name = 'ttltooshort.cache.tests.powerdns.com.'
1763         query = dns.message.make_query(name, 'A', 'IN')
1764         response = dns.message.make_response(query)
1765         rrset = dns.rrset.from_text(name,
1766                                     ttl,
1767                                     dns.rdataclass.IN,
1768                                     dns.rdatatype.A,
1769                                     '127.0.0.1')
1770         response.answer.append(rrset)
1771
1772         # Miss
1773         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1774         self.assertTrue(receivedQuery)
1775         self.assertTrue(receivedResponse)
1776         receivedQuery.id = query.id
1777         self.assertEqual(query, receivedQuery)
1778         self.assertEqual(response, receivedResponse)
1779         for an in receivedResponse.answer:
1780             self.assertEqual(an.ttl, ttl)
1781         misses += 1
1782
1783         # We should not have been cached
1784         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1785         self.assertTrue(receivedQuery)
1786         self.assertTrue(receivedResponse)
1787         receivedQuery.id = query.id
1788         self.assertEqual(query, receivedQuery)
1789         self.assertEqual(response, receivedResponse)
1790         for an in receivedResponse.answer:
1791             self.assertEqual(an.ttl, ttl)
1792         misses += 1
1793
1794         total = 0
1795         for key in self._responsesCounter:
1796             total += self._responsesCounter[key]
1797
1798         self.assertEqual(total, misses)
1799
1800     def testCacheNXWithNoRR(self):
1801         """
1802         Cache: NX with no RR
1803
1804         """
1805         misses = 0
1806         name = 'nxwithnorr.cache.tests.powerdns.com.'
1807         query = dns.message.make_query(name, 'A', 'IN')
1808         response = dns.message.make_response(query)
1809         response.set_rcode(dns.rcode.NXDOMAIN)
1810
1811         # Miss
1812         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1813         self.assertTrue(receivedQuery)
1814         self.assertTrue(receivedResponse)
1815         receivedQuery.id = query.id
1816         self.assertEqual(query, receivedQuery)
1817         self.assertEqual(response, receivedResponse)
1818         misses += 1
1819
1820         # We should not have been cached
1821         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1822         self.assertTrue(receivedQuery)
1823         self.assertTrue(receivedResponse)
1824         receivedQuery.id = query.id
1825         self.assertEqual(query, receivedQuery)
1826         self.assertEqual(response, receivedResponse)
1827         misses += 1
1828
1829         total = 0
1830         for key in self._responsesCounter:
1831             total += self._responsesCounter[key]
1832
1833         self.assertEqual(total, misses)
1834
1835 class TestCachingLongTTL(DNSDistTest):
1836
1837     _maxCacheTTL = 2
1838     _config_params = ['_maxCacheTTL', '_testServerPort']
1839     _config_template = """
1840     pc = newPacketCache(1000, {maxTTL=%d})
1841     getPool(""):setCache(pc)
1842     newServer{address="127.0.0.1:%d"}
1843     """
1844     def testCacheLongTTL(self):
1845         """
1846         Cache: Entries with a longer TTL than the maximum
1847
1848         """
1849         misses = 0
1850         ttl = 172800
1851         name = 'longttl.cache.tests.powerdns.com.'
1852         query = dns.message.make_query(name, 'A', 'IN')
1853         response = dns.message.make_response(query)
1854         rrset = dns.rrset.from_text(name,
1855                                     ttl,
1856                                     dns.rdataclass.IN,
1857                                     dns.rdatatype.A,
1858                                     '127.0.0.1')
1859         response.answer.append(rrset)
1860
1861         # Miss
1862         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1863         self.assertTrue(receivedQuery)
1864         self.assertTrue(receivedResponse)
1865         receivedQuery.id = query.id
1866         self.assertEqual(query, receivedQuery)
1867         self.assertEqual(response, receivedResponse)
1868         for an in receivedResponse.answer:
1869             self.assertEqual(an.ttl, ttl)
1870         misses += 1
1871
1872         # next queries should hit the cache
1873         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1874         self.assertEqual(receivedResponse, response)
1875         for an in receivedResponse.answer:
1876             self.assertTrue(an.ttl <= ttl)
1877
1878         time.sleep(self._maxCacheTTL + 1)
1879
1880         # we should not have cached for longer than max cache
1881         # so it should be a miss
1882         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1883         self.assertTrue(receivedQuery)
1884         self.assertTrue(receivedResponse)
1885         receivedQuery.id = query.id
1886         self.assertEqual(query, receivedQuery)
1887         self.assertEqual(response, receivedResponse)
1888         for an in receivedResponse.answer:
1889             self.assertEqual(an.ttl, ttl)
1890         misses += 1
1891
1892         total = 0
1893         for key in self._responsesCounter:
1894             total += self._responsesCounter[key]
1895
1896         self.assertEqual(total, misses)
1897
1898 class TestCachingFailureTTL(DNSDistTest):
1899
1900     _failureCacheTTL = 2
1901     _config_params = ['_failureCacheTTL', '_testServerPort']
1902     _config_template = """
1903     pc = newPacketCache(1000, {maxTTL=86400, minTTL=0, temporaryFailureTTL=%d, staleTTL=60})
1904     getPool(""):setCache(pc)
1905     newServer{address="127.0.0.1:%d"}
1906     """
1907     def testCacheServFailTTL(self):
1908         """
1909         Cache: ServFail TTL
1910
1911         """
1912         misses = 0
1913         name = 'servfail.failure.cache.tests.powerdns.com.'
1914         query = dns.message.make_query(name, 'A', 'IN')
1915         response = dns.message.make_response(query)
1916         response.set_rcode(dns.rcode.SERVFAIL)
1917
1918         # Miss
1919         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1920         self.assertTrue(receivedQuery)
1921         self.assertTrue(receivedResponse)
1922         receivedQuery.id = query.id
1923         self.assertEqual(query, receivedQuery)
1924         self.assertEqual(response, receivedResponse)
1925         misses += 1
1926
1927         # next queries should hit the cache
1928         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1929         self.assertEqual(receivedResponse, response)
1930
1931         time.sleep(self._failureCacheTTL + 1)
1932
1933         # we should not have cached for longer than failure cache
1934         # so it should be a miss
1935         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1936         self.assertTrue(receivedQuery)
1937         self.assertTrue(receivedResponse)
1938         receivedQuery.id = query.id
1939         self.assertEqual(query, receivedQuery)
1940         self.assertEqual(response, receivedResponse)
1941         misses += 1
1942
1943         total = 0
1944         for key in self._responsesCounter:
1945             total += self._responsesCounter[key]
1946
1947         self.assertEqual(total, misses)
1948
1949     def testCacheRefusedTTL(self):
1950         """
1951         Cache: Refused TTL
1952
1953         """
1954         misses = 0
1955         name = 'refused.failure.cache.tests.powerdns.com.'
1956         query = dns.message.make_query(name, 'A', 'IN')
1957         response = dns.message.make_response(query)
1958         response.set_rcode(dns.rcode.REFUSED)
1959
1960         # Miss
1961         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1962         self.assertTrue(receivedQuery)
1963         self.assertTrue(receivedResponse)
1964         receivedQuery.id = query.id
1965         self.assertEqual(query, receivedQuery)
1966         self.assertEqual(response, receivedResponse)
1967         misses += 1
1968
1969         # next queries should hit the cache
1970         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
1971         self.assertEqual(receivedResponse, response)
1972
1973         time.sleep(self._failureCacheTTL + 1)
1974
1975         # we should not have cached for longer than failure cache
1976         # so it should be a miss
1977         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
1978         self.assertTrue(receivedQuery)
1979         self.assertTrue(receivedResponse)
1980         receivedQuery.id = query.id
1981         self.assertEqual(query, receivedQuery)
1982         self.assertEqual(response, receivedResponse)
1983         misses += 1
1984
1985         total = 0
1986         for key in self._responsesCounter:
1987             total += self._responsesCounter[key]
1988
1989         self.assertEqual(total, misses)
1990
1991     def testCacheHeaderOnlyRefusedTTL(self):
1992         """
1993         Cache: Header-Only Refused TTL
1994
1995         """
1996         misses = 0
1997         name = 'header-only-refused.failure.cache.tests.powerdns.com.'
1998         query = dns.message.make_query(name, 'A', 'IN')
1999         response = dns.message.make_response(query)
2000         response.set_rcode(dns.rcode.REFUSED)
2001         response.question = []
2002
2003         # Miss
2004         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2005         self.assertTrue(receivedQuery)
2006         self.assertTrue(receivedResponse)
2007         receivedQuery.id = query.id
2008         self.assertEqual(query, receivedQuery)
2009         self.assertEqual(response, receivedResponse)
2010         misses += 1
2011
2012         # next queries should hit the cache
2013         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
2014         self.assertEqual(receivedResponse, response)
2015
2016         time.sleep(self._failureCacheTTL + 1)
2017
2018         # we should not have cached for longer than failure cache
2019         # so it should be a miss
2020         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2021         self.assertTrue(receivedQuery)
2022         self.assertTrue(receivedResponse)
2023         receivedQuery.id = query.id
2024         self.assertEqual(query, receivedQuery)
2025         self.assertEqual(response, receivedResponse)
2026         misses += 1
2027
2028         total = 0
2029         for key in self._responsesCounter:
2030             total += self._responsesCounter[key]
2031
2032         self.assertEqual(total, misses)
2033
2034 class TestCachingNegativeTTL(DNSDistTest):
2035
2036     _negCacheTTL = 2
2037     _config_params = ['_negCacheTTL', '_testServerPort']
2038     _config_template = """
2039     pc = newPacketCache(1000, {maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=%d})
2040     getPool(""):setCache(pc)
2041     newServer{address="127.0.0.1:%d"}
2042     """
2043
2044     def testCacheNegativeTTLNXDomain(self):
2045         """
2046         Cache: Negative TTL on NXDOMAIN
2047
2048         """
2049         misses = 0
2050         name = 'nxdomain.negativettl.cache.tests.powerdns.com.'
2051         query = dns.message.make_query(name, 'A', 'IN')
2052         response = dns.message.make_response(query)
2053         response.set_rcode(dns.rcode.NXDOMAIN)
2054         soa = dns.rrset.from_text(name,
2055                                   60,
2056                                   dns.rdataclass.IN,
2057                                   dns.rdatatype.SOA,
2058                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
2059         response.authority.append(soa)
2060
2061         # Miss
2062         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2063         self.assertTrue(receivedQuery)
2064         self.assertTrue(receivedResponse)
2065         receivedQuery.id = query.id
2066         self.assertEqual(query, receivedQuery)
2067         self.assertEqual(response, receivedResponse)
2068         misses += 1
2069
2070         # next queries should hit the cache
2071         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
2072         self.assertEqual(receivedResponse, response)
2073
2074         time.sleep(self._negCacheTTL + 1)
2075
2076         # we should not have cached for longer than the negative TTL
2077         # so it should be a miss
2078         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2079         self.assertTrue(receivedQuery)
2080         self.assertTrue(receivedResponse)
2081         receivedQuery.id = query.id
2082         self.assertEqual(query, receivedQuery)
2083         self.assertEqual(response, receivedResponse)
2084         misses += 1
2085
2086         total = 0
2087         for key in self._responsesCounter:
2088             total += self._responsesCounter[key]
2089
2090         self.assertEqual(total, misses)
2091
2092     def testCacheNegativeTTLNoData(self):
2093         """
2094         Cache: Negative TTL on NoData
2095
2096         """
2097         misses = 0
2098         name = 'nodata.negativettl.cache.tests.powerdns.com.'
2099         query = dns.message.make_query(name, 'A', 'IN')
2100         response = dns.message.make_response(query)
2101         response.set_rcode(dns.rcode.NOERROR)
2102         soa = dns.rrset.from_text(name,
2103                                   60,
2104                                   dns.rdataclass.IN,
2105                                   dns.rdatatype.SOA,
2106                                   'ns.' + name + ' hostmaster.' + name + ' 1 3600 3600 3600 60')
2107         response.authority.append(soa)
2108
2109         # Miss
2110         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2111         self.assertTrue(receivedQuery)
2112         self.assertTrue(receivedResponse)
2113         receivedQuery.id = query.id
2114         self.assertEqual(query, receivedQuery)
2115         self.assertEqual(response, receivedResponse)
2116         misses += 1
2117
2118         # next queries should hit the cache
2119         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
2120         self.assertEqual(receivedResponse, response)
2121
2122         time.sleep(self._negCacheTTL + 1)
2123
2124         # we should not have cached for longer than the negative TTL
2125         # so it should be a miss
2126         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2127         self.assertTrue(receivedQuery)
2128         self.assertTrue(receivedResponse)
2129         receivedQuery.id = query.id
2130         self.assertEqual(query, receivedQuery)
2131         self.assertEqual(response, receivedResponse)
2132         misses += 1
2133
2134         total = 0
2135         for key in self._responsesCounter:
2136             total += self._responsesCounter[key]
2137
2138         self.assertEqual(total, misses)
2139
2140 class TestCachingDontAge(DNSDistTest):
2141
2142     _config_template = """
2143     pc = newPacketCache(100, {maxTTL=86400, minTTL=0, temporaryFailureTTL=60, staleTTL=60, dontAge=true})
2144     getPool(""):setCache(pc)
2145     newServer{address="127.0.0.1:%d"}
2146     """
2147     def testCacheDoesntDecreaseTTL(self):
2148         """
2149         Cache: Cache doesn't decrease TTL with 'don't age' set
2150
2151         dnsdist is configured to cache entries but without aging the TTL,
2152         we are sending one request (cache miss) and verify that the cache
2153         hits don't have a decreasing TTL.
2154         """
2155         ttl = 600
2156         misses = 0
2157         name = 'cachedoesntdecreasettl.cache-dont-age.tests.powerdns.com.'
2158         query = dns.message.make_query(name, 'AAAA', 'IN')
2159         response = dns.message.make_response(query)
2160         rrset = dns.rrset.from_text(name,
2161                                     ttl,
2162                                     dns.rdataclass.IN,
2163                                     dns.rdatatype.AAAA,
2164                                     '::1')
2165         response.answer.append(rrset)
2166
2167         # first query to fill the cache
2168         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2169         self.assertTrue(receivedQuery)
2170         self.assertTrue(receivedResponse)
2171         receivedQuery.id = query.id
2172         self.assertEqual(query, receivedQuery)
2173         self.assertEqual(receivedResponse, response)
2174         misses += 1
2175
2176         # next queries should hit the cache
2177         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
2178         self.assertEqual(receivedResponse, response)
2179         for an in receivedResponse.answer:
2180             self.assertTrue(an.ttl == ttl)
2181
2182         # now we wait a bit for the TTL to decrease
2183         time.sleep(1)
2184
2185         # next queries should hit the cache
2186         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
2187         self.assertEqual(receivedResponse, response)
2188         for an in receivedResponse.answer:
2189             self.assertTrue(an.ttl == ttl)
2190
2191         total = 0
2192         for key in self._responsesCounter:
2193             total += self._responsesCounter[key]
2194
2195         self.assertEqual(total, misses)
2196
2197 class TestCachingECSWithoutPoolECS(DNSDistTest):
2198
2199     _consoleKey = DNSDistTest.generateConsoleKey()
2200     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
2201     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
2202     _config_template = """
2203     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
2204     getPool(""):setCache(pc)
2205     setKey("%s")
2206     controlSocket("127.0.0.1:%d")
2207     newServer{address="127.0.0.1:%d", useClientSubnet=true}
2208     """
2209
2210     def testCached(self):
2211         """
2212         Cache: Cached entry with ECS is a miss when no backend are available
2213         """
2214         ttl = 600
2215         name = 'cached.cache-ecs-without-pool-ecs.tests.powerdns.com.'
2216         query = dns.message.make_query(name, 'AAAA', 'IN')
2217         response = dns.message.make_response(query)
2218         rrset = dns.rrset.from_text(name,
2219                                     ttl,
2220                                     dns.rdataclass.IN,
2221                                     dns.rdatatype.AAAA,
2222                                     '::1')
2223         response.answer.append(rrset)
2224
2225         # first query to fill the cache
2226         for method in ("sendUDPQuery", "sendTCPQuery"):
2227             sender = getattr(self, method)
2228             (receivedQuery, receivedResponse) = sender(query, response)
2229             self.assertTrue(receivedQuery)
2230             self.assertTrue(receivedResponse)
2231             receivedQuery.id = query.id
2232             self.assertEqual(query, receivedQuery)
2233             self.assertEqual(receivedResponse, response)
2234
2235         # next queries should hit the cache
2236         for method in ("sendUDPQuery", "sendTCPQuery"):
2237             sender = getattr(self, method)
2238             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2239             self.assertEqual(receivedResponse, response)
2240
2241         # we mark the backend as down
2242         self.sendConsoleCommand("getServer(0):setDown()")
2243
2244         # we should NOT get a cached entry since it has ECS and we haven't asked the pool
2245         # to add ECS when no backend is up
2246         for method in ("sendUDPQuery", "sendTCPQuery"):
2247             sender = getattr(self, method)
2248             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2249             self.assertEqual(receivedResponse, None)
2250
2251 class TestCachingECSWithPoolECS(DNSDistTest):
2252
2253     _consoleKey = DNSDistTest.generateConsoleKey()
2254     _consoleKeyB64 = base64.b64encode(_consoleKey).decode('ascii')
2255     _config_params = ['_consoleKeyB64', '_consolePort', '_testServerPort']
2256     _config_template = """
2257     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
2258     getPool(""):setCache(pc)
2259     getPool(""):setECS(true)
2260     setKey("%s")
2261     controlSocket("127.0.0.1:%d")
2262     newServer{address="127.0.0.1:%d", useClientSubnet=true}
2263     """
2264
2265     def testCached(self):
2266         """
2267         Cache: Cached entry with ECS is a hit when no backend are available
2268         """
2269         ttl = 600
2270         name = 'cached.cache-ecs-with-pool-ecs.tests.powerdns.com.'
2271         query = dns.message.make_query(name, 'AAAA', 'IN')
2272         response = dns.message.make_response(query)
2273         rrset = dns.rrset.from_text(name,
2274                                     ttl,
2275                                     dns.rdataclass.IN,
2276                                     dns.rdatatype.AAAA,
2277                                     '::1')
2278         response.answer.append(rrset)
2279
2280         # first query to fill the cache
2281         for method in ("sendUDPQuery", "sendTCPQuery"):
2282             sender = getattr(self, method)
2283             (receivedQuery, receivedResponse) = sender(query, response)
2284             self.assertTrue(receivedQuery)
2285             self.assertTrue(receivedResponse)
2286             receivedQuery.id = query.id
2287             self.assertEqual(query, receivedQuery)
2288             self.assertEqual(receivedResponse, response)
2289
2290         # next queries should hit the cache
2291         for method in ("sendUDPQuery", "sendTCPQuery"):
2292             sender = getattr(self, method)
2293             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2294             self.assertEqual(receivedResponse, response)
2295
2296         # we mark the backend as down
2297         self.sendConsoleCommand("getServer(0):setDown()")
2298
2299         # we should STILL get a cached entry since it has ECS and we have asked the pool
2300         # to add ECS when no backend is up
2301         for method in ("sendUDPQuery", "sendTCPQuery"):
2302             sender = getattr(self, method)
2303             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2304             self.assertEqual(receivedResponse, response)
2305
2306 class TestCachingCollisionNoECSParsing(DNSDistTest):
2307
2308     _config_template = """
2309     pc = newPacketCache(100, {maxTTL=86400, minTTL=1})
2310     getPool(""):setCache(pc)
2311     newServer{address="127.0.0.1:%d"}
2312     """
2313
2314     def testCacheCollisionNoECSParsing(self):
2315         """
2316         Cache: Collision with no ECS parsing
2317         """
2318         name = 'collision-no-ecs-parsing.cache.tests.powerdns.com.'
2319         ecso = clientsubnetoption.ClientSubnetOption('10.0.226.63', 32)
2320         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2321         query.flags = dns.flags.RD
2322         response = dns.message.make_response(query)
2323         rrset = dns.rrset.from_text(name,
2324                                     3600,
2325                                     dns.rdataclass.IN,
2326                                     dns.rdatatype.AAAA,
2327                                     '::1')
2328         response.answer.append(rrset)
2329
2330         # first query should to fill the cache
2331         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2332         self.assertTrue(receivedQuery)
2333         self.assertTrue(receivedResponse)
2334         receivedQuery.id = query.id
2335         self.assertEqual(query, receivedQuery)
2336         self.assertEqual(receivedResponse, response)
2337
2338         # second query will hash to the same key, triggering a collision which
2339         # will not be detected because the qname, qtype, qclass and flags will
2340         # match and EDNS Client Subnet parsing has not been enabled
2341         ecso2 = clientsubnetoption.ClientSubnetOption('10.1.60.19', 32)
2342         query2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2343         query2.flags = dns.flags.RD
2344         (_, receivedResponse) = self.sendUDPQuery(query2, response=None, useQueue=False)
2345         receivedResponse.id = response.id
2346         self.assertEqual(receivedResponse, response)
2347
2348 class TestCachingCollisionWithECSParsing(DNSDistTest):
2349
2350     _config_template = """
2351     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, parseECS=true})
2352     getPool(""):setCache(pc)
2353     newServer{address="127.0.0.1:%d"}
2354     """
2355
2356     def testCacheCollisionWithECSParsing(self):
2357         """
2358         Cache: Collision with ECS parsing
2359         """
2360         name = 'collision-with-ecs-parsing.cache.tests.powerdns.com.'
2361         ecso = clientsubnetoption.ClientSubnetOption('10.0.150.206', 32)
2362         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2363         query.flags = dns.flags.RD
2364         response = dns.message.make_response(query)
2365         rrset = dns.rrset.from_text(name,
2366                                     3600,
2367                                     dns.rdataclass.IN,
2368                                     dns.rdatatype.AAAA,
2369                                     '::1')
2370         response.answer.append(rrset)
2371
2372         # first query should to fill the cache
2373         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
2374         self.assertTrue(receivedQuery)
2375         self.assertTrue(receivedResponse)
2376         receivedQuery.id = query.id
2377         self.assertEqual(query, receivedQuery)
2378         self.assertEqual(receivedResponse, response)
2379
2380         # second query will hash to the same key, triggering a collision which
2381         # _will_ be detected this time because the qname, qtype, qclass and flags will
2382         # match but EDNS Client Subnet parsing is now enabled and will detect the issue
2383         ecso2 = clientsubnetoption.ClientSubnetOption('10.0.212.51', 32)
2384         query2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2385         query2.flags = dns.flags.RD
2386         response2 = dns.message.make_response(query2)
2387         rrset = dns.rrset.from_text(name,
2388                                     3600,
2389                                     dns.rdataclass.IN,
2390                                     dns.rdatatype.AAAA,
2391                                     '2001:DB8::1')
2392         response2.answer.append(rrset)
2393         (receivedQuery, receivedResponse) = self.sendUDPQuery(query2, response2)
2394         self.assertEqual(receivedResponse, response2)
2395
2396 class TestCachingScopeZero(DNSDistTest):
2397
2398     _config_template = """
2399     -- Be careful to enable ECS parsing in the packet cache, otherwise scope zero is disabled
2400     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, parseECS=true})
2401     getPool(""):setCache(pc)
2402     newServer{address="127.0.0.1:%d", useClientSubnet=true}
2403     -- to simulate a second client coming from a different IP address,
2404     -- we will force the ECS value added to the query if RD is set (note that we need
2405     -- to unset it using rules before the first cache lookup)
2406     addAction(RDRule(), SetECSAction("192.0.2.1/32"))
2407     addAction(RDRule(), SetNoRecurseAction())
2408     """
2409
2410     def testScopeZero(self):
2411         """
2412         Cache: Test the scope-zero feature, backend returns a scope of zero
2413         """
2414         ttl = 600
2415         name = 'scope-zero.cache.tests.powerdns.com.'
2416         query = dns.message.make_query(name, 'AAAA', 'IN')
2417         query.flags &= ~dns.flags.RD
2418         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2419         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2420         expectedQuery.flags &= ~dns.flags.RD
2421         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 0)
2422         expectedResponse = dns.message.make_response(query)
2423         scopedResponse = dns.message.make_response(query)
2424         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2425         rrset = dns.rrset.from_text(name,
2426                                     ttl,
2427                                     dns.rdataclass.IN,
2428                                     dns.rdatatype.AAAA,
2429                                     '::1')
2430         scopedResponse.answer.append(rrset)
2431         expectedResponse.answer.append(rrset)
2432
2433         for method in ("sendUDPQuery", "sendTCPQuery"):
2434             sender = getattr(self, method)
2435             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2436             receivedQuery.id = expectedQuery.id
2437             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2438             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2439
2440         # next query should hit the cache, nothing special about that
2441         for method in ("sendUDPQuery", "sendTCPQuery"):
2442             sender = getattr(self, method)
2443             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2444             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2445
2446         query = dns.message.make_query(name, 'AAAA', 'IN')
2447         query.flags &= dns.flags.RD
2448         # next query FROM A DIFFERENT CLIENT since RD is now set should STILL hit the cache
2449         for method in ("sendUDPQuery", "sendTCPQuery"):
2450             sender = getattr(self, method)
2451             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2452             receivedResponse.id = expectedResponse.id
2453             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2454
2455         name = 'scope-zero-with-ecs.cache.tests.powerdns.com.'
2456         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24)
2457         query = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2458         query.flags &= ~dns.flags.RD
2459         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2460         expectedQuery.flags &= ~dns.flags.RD
2461         expectedResponse = dns.message.make_response(query)
2462         expectedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2463         expectedResponse.answer.append(rrset)
2464         scopedResponse = dns.message.make_response(query)
2465         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2466         scopedResponse.answer.append(rrset)
2467         # this query has ECS, it should NOT be able to use the scope-zero cached entry since the hash will be
2468         # different
2469         for method in ("sendUDPQuery", "sendTCPQuery"):
2470             sender = getattr(self, method)
2471             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2472             receivedQuery.id = expectedQuery.id
2473             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2474             self.checkMessageEDNSWithECS(receivedResponse, expectedResponse)
2475
2476         # it should still have been cached, though, so the next query should be a hit
2477         for method in ("sendUDPQuery", "sendTCPQuery"):
2478             sender = getattr(self, method)
2479             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2480             self.checkMessageEDNSWithECS(receivedResponse, expectedResponse)
2481
2482     def testScopeNotZero(self):
2483         """
2484         Cache: Test the scope-zero feature, backend returns a scope of non-zero
2485         """
2486         ttl = 600
2487         name = 'scope-not-zero.cache.tests.powerdns.com.'
2488         query = dns.message.make_query(name, 'AAAA', 'IN')
2489         query.flags &= ~dns.flags.RD
2490         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2491         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2492         expectedQuery.flags &= ~dns.flags.RD
2493         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
2494         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2495         expectedQuery2.flags &= ~dns.flags.RD
2496         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 24)
2497         ecsoResponse2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32, 24)
2498         rrset = dns.rrset.from_text(name,
2499                                     ttl,
2500                                     dns.rdataclass.IN,
2501                                     dns.rdatatype.AAAA,
2502                                     '::1')
2503         expectedResponse = dns.message.make_response(query)
2504         expectedResponse.answer.append(rrset)
2505         scopedResponse = dns.message.make_response(query)
2506         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2507         scopedResponse.answer.append(rrset)
2508         scopedResponse2 = dns.message.make_response(query)
2509         scopedResponse2.use_edns(edns=True, payload=4096, options=[ecsoResponse2])
2510         scopedResponse2.answer.append(rrset)
2511
2512         for method in ("sendUDPQuery", "sendTCPQuery"):
2513             sender = getattr(self, method)
2514             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2515             receivedQuery.id = expectedQuery.id
2516             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2517             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2518
2519         # next query should hit the cache, nothing special about that
2520         for method in ("sendUDPQuery", "sendTCPQuery"):
2521             sender = getattr(self, method)
2522             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2523             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2524
2525         query = dns.message.make_query(name, 'AAAA', 'IN')
2526         query.flags &= dns.flags.RD
2527         expectedResponse = dns.message.make_response(query)
2528         expectedResponse.answer.append(rrset)
2529         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2530         for method in ("sendUDPQuery", "sendTCPQuery"):
2531             sender = getattr(self, method)
2532             (receivedQuery, receivedResponse) = sender(query, scopedResponse2)
2533             receivedQuery.id = expectedQuery2.id
2534             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2535             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2536
2537     def testNoECS(self):
2538         """
2539         Cache: Test the scope-zero feature, backend returns no ECS at all
2540         """
2541         ttl = 600
2542         name = 'scope-zero-no-ecs.cache.tests.powerdns.com.'
2543         query = dns.message.make_query(name, 'AAAA', 'IN')
2544         query.flags &= ~dns.flags.RD
2545         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2546         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2547         expectedQuery.flags &= ~dns.flags.RD
2548         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
2549         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2550         expectedQuery2.flags &= ~dns.flags.RD
2551         rrset = dns.rrset.from_text(name,
2552                                     ttl,
2553                                     dns.rdataclass.IN,
2554                                     dns.rdatatype.AAAA,
2555                                     '::1')
2556         response = dns.message.make_response(query)
2557         response.answer.append(rrset)
2558
2559         for method in ("sendUDPQuery", "sendTCPQuery"):
2560             sender = getattr(self, method)
2561             (receivedQuery, receivedResponse) = sender(query, response)
2562             receivedQuery.id = expectedQuery.id
2563             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2564             self.checkMessageNoEDNS(receivedResponse, response)
2565
2566         # next query should hit the cache, nothing special about that
2567         for method in ("sendUDPQuery", "sendTCPQuery"):
2568             sender = getattr(self, method)
2569             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2570             self.checkMessageNoEDNS(receivedResponse, response)
2571
2572         query = dns.message.make_query(name, 'AAAA', 'IN')
2573         query.flags &= dns.flags.RD
2574         response = dns.message.make_response(query)
2575         response.answer.append(rrset)
2576         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2577         for method in ("sendUDPQuery", "sendTCPQuery"):
2578             sender = getattr(self, method)
2579             (receivedQuery, receivedResponse) = sender(query, response)
2580             receivedQuery.id = expectedQuery2.id
2581             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2582             self.checkMessageNoEDNS(receivedResponse, response)
2583
2584 class TestCachingScopeZeroButNoSubnetcheck(DNSDistTest):
2585
2586     _config_template = """
2587     -- We disable ECS parsing in the packet cache, meaning scope zero is disabled
2588     pc = newPacketCache(100, {maxTTL=86400, minTTL=1, temporaryFailureTTL=60, staleTTL=60, dontAge=false, numberOfShards=1, deferrableInsertLock=true, maxNegativeTTL=3600, parseECS=false})
2589     getPool(""):setCache(pc)
2590     newServer{address="127.0.0.1:%d", useClientSubnet=true}
2591     -- to simulate a second client coming from a different IP address,
2592     -- we will force the ECS value added to the query if RD is set (note that we need
2593     -- to unset it using rules before the first cache lookup)
2594     addAction(RDRule(), SetECSAction("192.0.2.1/32"))
2595     addAction(RDRule(), SetNoRecurseAction())
2596     """
2597
2598     def testScopeZero(self):
2599         """
2600         Cache: Test that the scope-zero feature is disabled when ECS parsing is not enabled in the cache
2601         """
2602         ttl = 600
2603         name = 'scope-zero-no-subnet.cache.tests.powerdns.com.'
2604         query = dns.message.make_query(name, 'AAAA', 'IN')
2605         query.flags &= ~dns.flags.RD
2606         ecso = clientsubnetoption.ClientSubnetOption('127.0.0.0', 24)
2607         expectedQuery = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso], payload=512)
2608         expectedQuery.flags &= ~dns.flags.RD
2609         ecso2 = clientsubnetoption.ClientSubnetOption('192.0.2.1', 32)
2610         expectedQuery2 = dns.message.make_query(name, 'AAAA', 'IN', use_edns=True, options=[ecso2], payload=512)
2611         expectedQuery2.flags &= ~dns.flags.RD
2612         ecsoResponse = clientsubnetoption.ClientSubnetOption('127.0.0.1', 24, 0)
2613         expectedResponse = dns.message.make_response(query)
2614         scopedResponse = dns.message.make_response(query)
2615         scopedResponse.use_edns(edns=True, payload=4096, options=[ecsoResponse])
2616         rrset = dns.rrset.from_text(name,
2617                                     ttl,
2618                                     dns.rdataclass.IN,
2619                                     dns.rdatatype.AAAA,
2620                                     '::1')
2621         scopedResponse.answer.append(rrset)
2622         expectedResponse.answer.append(rrset)
2623
2624         for method in ("sendUDPQuery", "sendTCPQuery"):
2625             sender = getattr(self, method)
2626             (receivedQuery, receivedResponse) = sender(query, scopedResponse)
2627             receivedQuery.id = expectedQuery.id
2628             self.checkMessageEDNSWithECS(expectedQuery, receivedQuery)
2629             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2630
2631         # next query should hit the cache, nothing special about that
2632         for method in ("sendUDPQuery", "sendTCPQuery"):
2633             sender = getattr(self, method)
2634             (_, receivedResponse) = sender(query, response=None, useQueue=False)
2635             self.checkMessageNoEDNS(receivedResponse, expectedResponse)
2636
2637         query = dns.message.make_query(name, 'AAAA', 'IN')
2638         query.flags &= dns.flags.RD
2639         response = dns.message.make_response(query)
2640         response.answer.append(rrset)
2641         # next query FROM A DIFFERENT CLIENT since RD is now set should NOT hit the cache
2642         for method in ("sendUDPQuery", "sendTCPQuery"):
2643             sender = getattr(self, method)
2644             (receivedQuery, receivedResponse) = sender(query, response)
2645             receivedQuery.id = expectedQuery2.id
2646             self.checkMessageEDNSWithECS(expectedQuery2, receivedQuery)
2647             self.checkMessageNoEDNS(receivedResponse, response)
2648
2649 class TestCachingAlteredHeader(DNSDistTest):
2650
2651     _config_template = """
2652     pc = newPacketCache(100)
2653     getPool(""):setCache(pc)
2654     addAction("cache-set-rd.tests.powerdns.com.", SetNoRecurseAction())
2655     newServer{address="127.0.0.1:%d"}
2656     """
2657
2658     def testCachingAlteredHeader(self):
2659         """
2660         Cache: The header has been altered via a rule
2661         """
2662         name = 'cache-set-rd.tests.powerdns.com.'
2663         query = dns.message.make_query(name, 'A', 'IN')
2664         # the query reaching the backend will never have the RD flag set
2665         expectedQuery = dns.message.make_query(name, 'A', 'IN')
2666         expectedQuery.flags &= ~dns.flags.RD
2667         response = dns.message.make_response(query)
2668         response.flags &= ~dns.flags.RD
2669         rrset = dns.rrset.from_text(name,
2670                                     60,
2671                                     dns.rdataclass.IN,
2672                                     dns.rdatatype.A,
2673                                     '192.0.2.1')
2674         response.answer.append(rrset)
2675
2676         # first query has RD=1
2677         query.flags |= dns.flags.RD
2678         expectedResponse = dns.message.make_response(query)
2679         rrset = dns.rrset.from_text(name,
2680                                     60,
2681                                     dns.rdataclass.IN,
2682                                     dns.rdatatype.A,
2683                                     '192.0.2.1')
2684         expectedResponse.answer.append(rrset)
2685
2686         for method in ("sendUDPQuery", "sendTCPQuery"):
2687             sender = getattr(self, method)
2688             (receivedQuery, receivedResponse) = sender(query, response)
2689             self.assertTrue(receivedQuery)
2690             self.assertTrue(receivedResponse)
2691             receivedQuery.id = expectedQuery.id
2692             self.assertEqual(expectedQuery, receivedQuery)
2693             self.assertEqual(receivedResponse, expectedResponse)
2694
2695         # next query should hit the cache
2696         for method in ("sendUDPQuery", "sendTCPQuery"):
2697             sender = getattr(self, method)
2698             (receivedQuery, receivedResponse) = sender(query, response=None, useQueue=False)
2699
2700         # same query with RD=0, should hit the cache as well
2701         query.flags &= ~dns.flags.RD
2702         expectedResponse = dns.message.make_response(query)
2703         rrset = dns.rrset.from_text(name,
2704                                     60,
2705                                     dns.rdataclass.IN,
2706                                     dns.rdatatype.A,
2707                                     '192.0.2.1')
2708         expectedResponse.answer.append(rrset)
2709         for method in ("sendUDPQuery", "sendTCPQuery"):
2710             sender = getattr(self, method)
2711             (receivedQuery, receivedResponse) = sender(query, response=None, useQueue=False)
2712             self.assertFalse(receivedQuery)
2713             self.assertTrue(receivedResponse)
2714             self.assertEqual(receivedResponse, expectedResponse)
2715
2716 class TestCachingBackendSettingRD(DNSDistTest):
2717
2718     _config_template = """
2719     pc = newPacketCache(100)
2720     getPool(""):setCache(pc)
2721     newServer{address="127.0.0.1:%d"}
2722     """
2723
2724     def testCachingBackendSetRD(self):
2725         """
2726         Cache: The backend sets RD=1 in the response even if the query had RD=0
2727         """
2728         name = 'backend-sets-rd.tests.powerdns.com.'
2729         query = dns.message.make_query(name, 'A', 'IN')
2730         query.flags &= ~dns.flags.RD
2731         expectedQuery = dns.message.make_query(name, 'A', 'IN')
2732         expectedQuery.flags &= ~dns.flags.RD
2733         response = dns.message.make_response(query)
2734         response.flags |= dns.flags.RD
2735         rrset = dns.rrset.from_text(name,
2736                                     60,
2737                                     dns.rdataclass.IN,
2738                                     dns.rdatatype.A,
2739                                     '192.0.2.1')
2740         response.answer.append(rrset)
2741
2742         expectedResponse = dns.message.make_response(query)
2743         expectedResponse.flags &= ~dns.flags.RD
2744         rrset = dns.rrset.from_text(name,
2745                                     60,
2746                                     dns.rdataclass.IN,
2747                                     dns.rdatatype.A,
2748                                     '192.0.2.1')
2749         expectedResponse.answer.append(rrset)
2750
2751         for method in ("sendUDPQuery", "sendTCPQuery"):
2752             sender = getattr(self, method)
2753             (receivedQuery, receivedResponse) = sender(query, response)
2754             self.assertTrue(receivedQuery)
2755             self.assertTrue(receivedResponse)
2756             receivedQuery.id = expectedQuery.id
2757             self.assertEqual(expectedQuery, receivedQuery)
2758             self.assertEqual(receivedResponse, expectedResponse)
2759
2760         # exact same query should be cached
2761         for method in ("sendUDPQuery", "sendTCPQuery"):
2762             sender = getattr(self, method)
2763             (receivedQuery, receivedResponse) = sender(query, response=None, useQueue=False)
2764             self.assertFalse(receivedQuery)
2765             self.assertTrue(receivedResponse)
2766             self.assertEqual(receivedResponse, expectedResponse)
2767
2768         # same query with RD=1, should NOT hit the cache
2769         query.flags |= dns.flags.RD
2770         expectedResponse = dns.message.make_response(query)
2771         rrset = dns.rrset.from_text(name,
2772                                     60,
2773                                     dns.rdataclass.IN,
2774                                     dns.rdatatype.A,
2775                                     '192.0.2.1')
2776         expectedResponse.answer.append(rrset)
2777
2778         for method in ("sendUDPQuery", "sendTCPQuery"):
2779             sender = getattr(self, method)
2780             (receivedQuery, receivedResponse) = sender(query, response)
2781             self.assertTrue(receivedQuery)
2782             self.assertTrue(receivedResponse)
2783             self.assertEqual(receivedResponse, expectedResponse)