Merge pull request #3355 from rgacogne/dnsdist-cache-clean

dnsdist: Add a simple Packet Cache

diff --cc pdns/README-dnsdist.md
Simple merge

diff --cc pdns/dnsdist-lua.cc
Simple merge

diff --cc pdns/dnsdistdist/Makefile.am
Simple merge

diff --cc pdns/dnsrulactions.hh
Simple merge

diff --cc regression-tests.dnsdist/test_Advanced.py
index 2b07fdb60ae4be0655f2715ff1c9f14dd3414ef6,392bbd395308577a4ae3040a81d86c0dff63cb56..de6f997a70139aa83b92e38970ac9edbb590ebf9
--- 1/regression-tests.dnsdist/test_Advanced.py
--- 2/regression-tests.dnsdist/test_Advanced.py
+++ b/regression-tests.dnsdist/test_Advanced.py
@@@ -821,4 -861,328 +821,326 @@@ class TestAdvancedOr(DNSDistTest)
          self.assertEquals(receivedResponse, expectedResponse)
  
          (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
 -        receivedResponse.id = expectedResponse.id
          self.assertEquals(receivedResponse, expectedResponse)
+ 
+ class TestAdvancedCaching(DNSDistTest):
+ 
+     _config_template = """
+     pc = newPacketCache(5, 86400, 1)
+     getPool(""):setCache(pc)
+     addAction(makeRule("nocache.tests.powerdns.com."), SkipCacheAction())
+     newServer{address="127.0.0.1:%s"}
+     """
+     def testCached(self):
+         """
+         Advanced: Served from cache
+ 
+         dnsdist is configured to cache entries, we are sending several
+         identical requests and checking that the backend only receive
+         the first one.
+         """
+         numberOfQueries = 10
+         name = 'cached.tests.powerdns.com.'
+         query = dns.message.make_query(name, 'AAAA', 'IN')
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     3600,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.AAAA,
+                                     '::1')
+         response.answer.append(rrset)
+ 
+         # first query to fill the cache
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(receivedResponse, response)
+ 
+         for idx in range(numberOfQueries):
+             (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+             receivedResponse.id = response.id
+             self.assertEquals(receivedResponse, response)
+ 
+             (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
+             receivedResponse.id = response.id
+             self.assertEquals(receivedResponse, response)
+ 
+         total = 0
+         for key in TestAdvancedCaching._responsesCounter:
+             total += TestAdvancedCaching._responsesCounter[key]
+ 
+         self.assertEquals(total, 1)
+ 
+     def testSkipCache(self):
+         """
+         Advanced: SkipCacheAction
+ 
+         dnsdist is configured to not cache entries for nocache.tests.powerdns.com.
+          we are sending several requests and checking that the backend get them all.
+         """
+         name = 'nocache.tests.powerdns.com.'
+         numberOfQueries = 10
+         query = dns.message.make_query(name, 'AAAA', 'IN')
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     3600,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.AAAA,
+                                     '::1')
+         response.answer.append(rrset)
+ 
+         for idx in range(numberOfQueries):
+             (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+             self.assertTrue(receivedQuery)
+             self.assertTrue(receivedResponse)
+             receivedQuery.id = query.id
+             receivedResponse.id = response.id
+             self.assertEquals(query, receivedQuery)
+             self.assertEquals(receivedResponse, response)
+ 
+             (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response)
+             self.assertTrue(receivedQuery)
+             self.assertTrue(receivedResponse)
+             receivedQuery.id = query.id
+             receivedResponse.id = response.id
+             self.assertEquals(query, receivedQuery)
+             self.assertEquals(receivedResponse, response)
+ 
+         for key in TestAdvancedCaching._responsesCounter:
+             value = TestAdvancedCaching._responsesCounter[key]
+             self.assertEquals(value, numberOfQueries)
+ 
+     def testCacheExpiration(self):
+         """
+         Advanced: Cache expiration
+ 
+         dnsdist is configured to cache entries, we are sending one request
+         (cache miss) with a very short TTL, checking that the next requests
+         are cached. Then we wait for the TTL to expire, check that the
+         next request is a miss but the following one a hit.
+         """
+         ttl = 2
+         misses = 0
+         name = 'cacheexpiration.tests.powerdns.com.'
+         query = dns.message.make_query(name, 'AAAA', 'IN')
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     ttl,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.AAAA,
+                                     '::1')
+         response.answer.append(rrset)
+ 
+         # first query to fill the cache
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(receivedResponse, response)
+         misses += 1
+ 
+         # next queries should hit the cache
+         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+ 
+         (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+ 
+         # now we wait a bit for the cache entry to expire
+         time.sleep(ttl + 1)
+ 
+         # next query should be a miss, fill the cache again
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(receivedResponse, response)
+         misses += 1
+ 
+         # following queries should hit the cache again
+         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+ 
+         (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+ 
+         total = 0
+         for key in TestAdvancedCaching._responsesCounter:
+             total += TestAdvancedCaching._responsesCounter[key]
+ 
+         self.assertEquals(total, misses)
+ 
+     def testCacheDecreaseTTL(self):
+         """
+         Advanced: Cache decreases TTL
+ 
+         dnsdist is configured to cache entries, we are sending one request
+         (cache miss) and verify that the cache hits have a decreasing TTL.
+         """
+         ttl = 600
+         misses = 0
+         name = 'cachedecreasettl.tests.powerdns.com.'
+         query = dns.message.make_query(name, 'AAAA', 'IN')
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     ttl,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.AAAA,
+                                     '::1')
+         response.answer.append(rrset)
+ 
+         # first query to fill the cache
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(receivedResponse, response)
+         misses += 1
+ 
+         # next queries should hit the cache
+         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+         for an in receivedResponse.answer:
+             self.assertTrue(an.ttl <= ttl)
+ 
+         (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+         for an in receivedResponse.answer:
+             self.assertTrue(an.ttl <= ttl)
+ 
+         # now we wait a bit for the TTL to decrease
+         time.sleep(1)
+ 
+         # next queries should hit the cache
+         (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+         for an in receivedResponse.answer:
+             self.assertTrue(an.ttl < ttl)
+ 
+         (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False)
+         receivedResponse.id = response.id
+         self.assertEquals(receivedResponse, response)
+         for an in receivedResponse.answer:
+             self.assertTrue(an.ttl < ttl)
+ 
+         total = 0
+         for key in TestAdvancedCaching._responsesCounter:
+             total += TestAdvancedCaching._responsesCounter[key]
+ 
+         self.assertEquals(total, misses)
+ 
+     def testCacheDifferentCase(self):
+         """
+         Advanced: Cache matches different case
+ 
+         dnsdist is configured to cache entries, we are sending one request
+         (cache miss) and verify that the same one with a different case
+         matches.
+         """
+         ttl = 600
+         name = 'cachedifferentcase.tests.powerdns.com.'
+         differentCaseName = 'CacheDifferentCASE.tests.powerdns.com.'
+         query = dns.message.make_query(name, 'AAAA', 'IN')
+         differentCaseQuery = dns.message.make_query(differentCaseName, 'AAAA', 'IN')
+         response = dns.message.make_response(query)
+         differentCaseResponse = dns.message.make_response(differentCaseQuery)
+         rrset = dns.rrset.from_text(name,
+                                     ttl,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.AAAA,
+                                     '::1')
+         response.answer.append(rrset)
+         differentCaseResponse.answer.append(rrset)
+ 
+         # first query to fill the cache
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(receivedResponse, response)
+ 
+         # different case query should still hit the cache
+         (_, receivedResponse) = self.sendUDPQuery(differentCaseQuery, response=None, useQueue=False)
+         receivedResponse.id = differentCaseResponse.id
+         self.assertEquals(receivedResponse, differentCaseResponse)
+ 
+         (_, receivedResponse) = self.sendTCPQuery(differentCaseQuery, response=None, useQueue=False)
+         receivedResponse.id = differentCaseResponse.id
+         self.assertEquals(receivedResponse, differentCaseResponse)
+ 
+ class TestAdvancedCachingWithExistingEDNS(DNSDistTest):
+ 
+     _config_template = """
+     pc = newPacketCache(5, 86400, 1)
+     getPool(""):setCache(pc)
+     newServer{address="127.0.0.1:%s"}
+     """
+     def testCacheWithEDNS(self):
+         """
+         Advanced: Cache should not match different EDNS value
+ 
+         dnsdist is configured to cache entries, we are sending one request
+         (cache miss) and verify that the same one with a different EDNS UDP
+         Payload size is not served from the cache.
+         """
+         misses = 0
+         name = 'cachedifferentedns.tests.powerdns.com.'
+         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=512)
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     3600,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.A,
+                                     '127.0.0.1')
+         response.answer.append(rrset)
+ 
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(response, receivedResponse)
+         misses += 1
+ 
+         query = dns.message.make_query(name, 'A', 'IN', use_edns=True, payload=4096)
+         response = dns.message.make_response(query)
+         rrset = dns.rrset.from_text(name,
+                                     3600,
+                                     dns.rdataclass.IN,
+                                     dns.rdatatype.A,
+                                     '127.0.0.1')
+         response.answer.append(rrset)
+ 
+         (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response)
+         self.assertTrue(receivedQuery)
+         self.assertTrue(receivedResponse)
+         receivedQuery.id = query.id
+         receivedResponse.id = response.id
+         self.assertEquals(query, receivedQuery)
+         self.assertEquals(response, receivedResponse)
+         misses += 1
+ 
+         total = 0
+         for key in TestAdvancedCaching._responsesCounter:
+             total += TestAdvancedCaching._responsesCounter[key]
+ 
+         self.assertEquals(total, misses)