From: Remi Gacogne Date: Mon, 19 Dec 2016 12:02:31 +0000 (+0100) Subject: dnsdist: Add cache hit response rules X-Git-Tag: rec-4.1.0-alpha1~298^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F4788%2Fhead;p=thirdparty%2Fpdns.git dnsdist: Add cache hit response rules It allows applying actions on a response coming from the cache, for example to be able to send a protobuf message. --- diff --git a/pdns/README-dnsdist.md b/pdns/README-dnsdist.md index 59d12320c1..4593686dd0 100644 --- a/pdns/README-dnsdist.md +++ b/pdns/README-dnsdist.md @@ -403,6 +403,10 @@ Response rules can be added via: * addResponseAction(DNS rule, DNS Response Action) * AddLuaResponseAction(DNS rule, Lua function) +Cache Hit Response rules, triggered on a cache hit, can be added via: + + * addCacheHitResponseAction(DNS rule, DNS Response Action) + A DNS rule can be: * an AllRule @@ -1344,16 +1348,21 @@ instantiate a server with additional parameters * Rule management related: * `clearRules()`: remove all current rules * `getAction(num)`: returns the Action associate with rule 'num'. + * `mvCacheHitResponseRule(from, to)`: move cache hit response rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule, + in which case the rule will be moved to the last position. * `mvResponseRule(from, to)`: move response rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule, in which case the rule will be moved to the last position. * `mvRule(from, to)`: move rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule, in which case the rule will be moved to the last position. * `newRuleAction(DNS Rule, DNS Action)`: return a pair of DNS Rule and DNS Action, to be used with `setRules()` + * `rmCacheHitResponseRule(n)`: remove cache hit response rule n * `rmResponseRule(n)`: remove response rule n * `rmRule(n)`: remove rule n * `setRules(list)`: replace the current rules with the supplied list of pairs of DNS Rules and DNS Actions (see `newRuleAction()`) + * `showCacheHitResponseRules()`: show all defined cache hit response rules * `showResponseRules()`: show all defined response rules * `showRules()`: show all defined rules + * `topCacheHitResponseRule()`: move the last cache hit response rule to the first position * `topResponseRule()`: move the last response rule to the first position * `topRule()`: move the last rule to the first position * Built-in Actions for Rules: diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc index 6e582a5ec9..ce642ad6b8 100644 --- a/pdns/dnsdist-console.cc +++ b/pdns/dnsdist-console.cc @@ -287,6 +287,7 @@ const std::vector g_consoleKeywords{ { "addPoolRule", true, "domain, pool", "send queries to this domain to that pool" }, { "addQPSLimit", true, "domain, n", "limit queries within that domain to n per second" }, { "addQPSPoolRule", true, "x, limit, pool", "like `addPoolRule`, but only select at most 'limit' queries/s for this pool, letting the subsequent rules apply otherwise" }, + { "addCacheHitResponseAction", true, "DNS rule, DNS response action", "add a cache hit response rule" }, { "addResponseAction", true, "DNS rule, DNS response action", "add a response rule" }, { "AllowAction", true, "", "let these packets go through" }, { "AllowResponseAction", true, "", "let these packets go through" }, @@ -325,6 +326,7 @@ const std::vector g_consoleKeywords{ { "makeKey", true, "", "generate a new server access key, emit configuration line ready for pasting" }, { "MaxQPSIPRule", true, "qps, v4Mask=32, v6Mask=64", "matches traffic exceeding the qps limit per subnet" }, { "MaxQPSRule", true, "qps", "matches traffic **not** exceeding this qps limit" }, + { "mvCacheHitResponseRule", true, "from, to", "move cache hit response rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule" }, { "mvResponseRule", true, "from, to", "move response rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule" }, { "mvRule", true, "from, to", "move rule 'from' to a position where it is in front of 'to'. 'to' can be one larger than the largest rule, in which case the rule will be moved to the last position" }, { "newDNSName", true, "name", "make a DNSName based on this .-terminated name" }, @@ -341,6 +343,7 @@ const std::vector g_consoleKeywords{ { "registerDynBPFFilter", true, "DynBPFFilter", "register this dynamic BPF filter into the web interface so that its counters are displayed" }, { "RemoteLogAction", true, "RemoteLogger [, alterFunction]", "send the content of this query to a remote logger via Protocol Buffer. `alterFunction` is a callback, receiving a DNSQuestion and a DNSDistProtoBufMessage, that can be used to modify the Protocol Buffer content, for example for anonymization purposes" }, { "RemoteLogResponseAction", true, "RemoteLogger [,alterFunction [,includeCNAME]]", "send the content of this response to a remote logger via Protocol Buffer. `alterFunction` is the same callback than the one in `RemoteLogAction` and `includeCNAME` indicates whether CNAME records inside the response should be parsed and exported. The default is to only exports A and AAAA records" }, + { "rmCacheHitResponseRule", true, "n", "remove cache hit response rule n" }, { "rmResponseRule", true, "n", "remove response rule n" }, { "rmRule", true, "n", "remove rule n" }, { "rmServer", true, "n", "remove server with index n" }, @@ -376,6 +379,7 @@ const std::vector g_consoleKeywords{ { "setVerboseHealthChecks", true, "bool", "set whether health check errors will be logged" }, { "show", true, "string", "outputs `string`" }, { "showACL", true, "", "show our ACL set" }, + { "showCacheHitResponseRules", true, "", "show all defined cache hit response rules" }, { "showDNSCryptBinds", true, "", "display the currently configured DNSCrypt binds" }, { "showDynBlocks", true, "", "show dynamic blocks in force" }, { "showResponseLatency", true, "", "show a plot of the response time latency distribution" }, @@ -390,6 +394,7 @@ const std::vector g_consoleKeywords{ { "TCAction", true, "", "create answer to query with TC and RD bits set, to move to TCP" }, { "testCrypto", true, "", "test of the crypto all works" }, { "topBandwidth", true, "top", "show top-`top` clients that consume the most bandwidth over length of ringbuffer" }, + { "topCacheHitResponseRule", true, "", "move the last cache hit response rule to the first position" }, { "topClients", true, "n", "show top-`n` clients sending the most queries over length of ringbuffer" }, { "topQueries", true, "n[, labels]", "show top 'n' queries, as grouped when optionally cut down to 'labels' labels" }, { "topResponses", true, "n, kind[, labels]", "show top 'n' responses with RCODE=kind (0=NO Error, 2=ServFail, 3=ServFail), as grouped when optionally cut down to 'labels' labels" }, diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index ceaafd6ef1..4cc53b2312 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -31,6 +31,7 @@ #include #include "dnswriter.hh" #include "lock.hh" +#include "dnsdist-lua.hh" #ifdef HAVE_SYSTEMD #include @@ -91,6 +92,7 @@ private: }; typedef boost::variant>, std::shared_ptr > luadnsrule_t; + std::shared_ptr makeRule(const luadnsrule_t& var) { if(auto src = boost::get>(&var)) @@ -1610,14 +1612,6 @@ vector> setupLua(bool client, const std::string& confi g_lua.writeFunction("setECSOverride", [](bool override) { g_ECSOverride=override; }); - g_lua.writeFunction("addResponseAction", [](luadnsrule_t var, std::shared_ptr ea) { - setLuaSideEffect(); - auto rule=makeRule(var); - g_resprulactions.modify([rule, ea](decltype(g_resprulactions)::value_type& rulactions){ - rulactions.push_back({rule, ea}); - }); - }); - g_lua.writeFunction("dumpStats", [] { setLuaNoSideEffect(); vector leftcolumn, rightcolumn; diff --git a/pdns/dnsdist-lua.hh b/pdns/dnsdist-lua.hh new file mode 100644 index 0000000000..5e88dfa91f --- /dev/null +++ b/pdns/dnsdist-lua.hh @@ -0,0 +1,25 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +typedef boost::variant>, std::shared_ptr > luadnsrule_t; +std::shared_ptr makeRule(const luadnsrule_t& var); diff --git a/pdns/dnsdist-lua2.cc b/pdns/dnsdist-lua2.cc index 507a99daac..e4894963cc 100644 --- a/pdns/dnsdist-lua2.cc +++ b/pdns/dnsdist-lua2.cc @@ -37,6 +37,8 @@ #include #include +#include "dnsdist-lua.hh" + boost::tribool g_noLuaSideEffect; static bool g_included{false}; @@ -810,6 +812,14 @@ void moreLua(bool client) g_lua.registerFunction("getStats", &DNSAction::getStats); + g_lua.writeFunction("addResponseAction", [](luadnsrule_t var, std::shared_ptr ea) { + setLuaSideEffect(); + auto rule=makeRule(var); + g_resprulactions.modify([rule, ea](decltype(g_resprulactions)::value_type& rulactions){ + rulactions.push_back({rule, ea}); + }); + }); + g_lua.writeFunction("showResponseRules", []() { setLuaNoSideEffect(); boost::format fmt("%-3d %9d %-50s %s\n"); @@ -863,6 +873,67 @@ void moreLua(bool client) g_resprulactions.setState(rules); }); + g_lua.writeFunction("addCacheHitResponseAction", [](luadnsrule_t var, std::shared_ptr ea) { + setLuaSideEffect(); + auto rule=makeRule(var); + g_cachehitresprulactions.modify([rule, ea](decltype(g_cachehitresprulactions)::value_type& rulactions){ + rulactions.push_back({rule, ea}); + }); + }); + + g_lua.writeFunction("showCacheHitResponseRules", []() { + setLuaNoSideEffect(); + boost::format fmt("%-3d %9d %-50s %s\n"); + g_outputBuffer += (fmt % "#" % "Matches" % "Rule" % "Action").str(); + int num=0; + for(const auto& lim : g_cachehitresprulactions.getCopy()) { + string name = lim.first->toString(); + g_outputBuffer += (fmt % num % lim.first->d_matches % name % lim.second->toString()).str(); + ++num; + } + }); + + g_lua.writeFunction("rmCacheHitResponseRule", [](unsigned int num) { + setLuaSideEffect(); + auto rules = g_cachehitresprulactions.getCopy(); + if(num >= rules.size()) { + g_outputBuffer = "Error: attempt to delete non-existing rule\n"; + return; + } + rules.erase(rules.begin()+num); + g_cachehitresprulactions.setState(rules); + }); + + g_lua.writeFunction("topCacheHitResponseRule", []() { + setLuaSideEffect(); + auto rules = g_cachehitresprulactions.getCopy(); + if(rules.empty()) + return; + auto subject = *rules.rbegin(); + rules.erase(std::prev(rules.end())); + rules.insert(rules.begin(), subject); + g_cachehitresprulactions.setState(rules); + }); + + g_lua.writeFunction("mvCacheHitResponseRule", [](unsigned int from, unsigned int to) { + setLuaSideEffect(); + auto rules = g_cachehitresprulactions.getCopy(); + if(from >= rules.size() || to > rules.size()) { + g_outputBuffer = "Error: attempt to move rules from/to invalid index\n"; + return; + } + auto subject = rules[from]; + rules.erase(rules.begin()+from); + if(to == rules.size()) + rules.push_back(subject); + else { + if(from < to) + --to; + rules.insert(rules.begin()+to, subject); + } + g_cachehitresprulactions.setState(rules); + }); + g_lua.writeFunction("showBinds", []() { setLuaNoSideEffect(); try { diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc index a66137ec9c..c90e79b3f4 100644 --- a/pdns/dnsdist-tcp.cc +++ b/pdns/dnsdist-tcp.cc @@ -222,6 +222,7 @@ void* tcpClientThread(int pipefd) auto localPolicy = g_policy.getLocal(); auto localRulactions = g_rulactions.getLocal(); auto localRespRulactions = g_resprulactions.getLocal(); + auto localCacheHitRespRulactions = g_cachehitresprulactions.getLocal(); auto localDynBlockNMG = g_dynblockNMG.getLocal(); auto localDynBlockSMT = g_dynblockSMT.getLocal(); auto localPools = g_pools.getLocal(); @@ -395,6 +396,14 @@ void* tcpClientThread(int pipefd) uint16_t cachedResponseSize = sizeof cachedResponse; uint32_t allowExpired = ds ? 0 : g_staleCacheEntriesTTL; if (packetCache->get(dq, (uint16_t) consumed, dq.dh->id, cachedResponse, &cachedResponseSize, &cacheKey, allowExpired)) { + DNSResponse dr(dq.qname, dq.qtype, dq.qclass, dq.local, dq.remote, (dnsheader*) cachedResponse, sizeof cachedResponse, cachedResponseSize, true, &queryRealTime); +#ifdef HAVE_PROTOBUF + dr.uniqueId = dq.uniqueId; +#endif + if (!processResponse(localCacheHitRespRulactions, dr, &delayMsec)) { + goto drop; + } + #ifdef HAVE_DNSCRYPT if (!encryptResponse(cachedResponse, &cachedResponseSize, sizeof cachedResponse, true, dnsCryptQuery)) { goto drop; diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc index 2002e6d0bd..9f9540bd89 100644 --- a/pdns/dnsdist.cc +++ b/pdns/dnsdist.cc @@ -124,6 +124,7 @@ GlobalStateHolder g_pools; GlobalStateHolder, std::shared_ptr > > > g_rulactions; GlobalStateHolder, std::shared_ptr > > > g_resprulactions; +GlobalStateHolder, std::shared_ptr > > > g_cachehitresprulactions; Rings g_rings; QueryCount g_qcount; @@ -1011,6 +1012,7 @@ try auto acl = g_ACL.getLocal(); auto localPolicy = g_policy.getLocal(); auto localRulactions = g_rulactions.getLocal(); + auto localCacheHitRespRulactions = g_cachehitresprulactions.getLocal(); auto localServers = g_dstates.getLocal(); auto localDynNMGBlock = g_dynblockNMG.getLocal(); auto localDynSMTBlock = g_dynblockSMT.getLocal(); @@ -1105,8 +1107,13 @@ try string poolname; int delayMsec=0; + /* we need an accurate ("real") value for the response and + to store into the IDS, but not for insertion into the + rings for example */ + struct timespec realTime; struct timespec now; gettime(&now); + gettime(&realTime, true); if (!processQuery(localDynNMGBlock, localDynSMTBlock, localRulactions, blockFilter, dq, poolname, &delayMsec, now)) { @@ -1154,6 +1161,14 @@ try uint16_t cachedResponseSize = sizeof cachedResponse; uint32_t allowExpired = ss ? 0 : g_staleCacheEntriesTTL; if (packetCache->get(dq, consumed, dh->id, cachedResponse, &cachedResponseSize, &cacheKey, allowExpired)) { + DNSResponse dr(dq.qname, dq.qtype, dq.qclass, dq.local, dq.remote, (dnsheader*) cachedResponse, sizeof cachedResponse, cachedResponseSize, false, &realTime); +#ifdef HAVE_PROTOBUF + dr.uniqueId = dq.uniqueId; +#endif + if (!processResponse(localCacheHitRespRulactions, dr, &delayMsec)) { + continue; + } + if (!cs->muted) { #ifdef HAVE_DNSCRYPT if (!encryptResponse(cachedResponse, &cachedResponseSize, sizeof cachedResponse, false, dnsCryptQuery)) { @@ -1209,7 +1224,7 @@ try ids->origFD = cs->udpFD; ids->origID = dh->id; ids->origRemote = remote; - ids->sentTime.start(); + ids->sentTime.set(realTime); ids->qname = qname; ids->qtype = dq.qtype; ids->qclass = dq.qclass; diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index ba2f8b72cf..46c7f25bf6 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -153,6 +153,10 @@ struct StopWatch unixDie("Getting timestamp"); } + + void set(const struct timespec& from) { + d_start = from; + } double udiff() const { struct timespec now; @@ -611,6 +615,7 @@ extern GlobalStateHolder g_dstates; extern GlobalStateHolder g_pools; extern GlobalStateHolder, std::shared_ptr > > > g_rulactions; extern GlobalStateHolder, std::shared_ptr > > > g_resprulactions; +extern GlobalStateHolder, std::shared_ptr > > > g_cachehitresprulactions; extern GlobalStateHolder g_ACL; extern ComboAddress g_serverControl; // not changed during runtime diff --git a/pdns/dnsdistdist/Makefile.am b/pdns/dnsdistdist/Makefile.am index c4a01202ca..2aa1c3979c 100644 --- a/pdns/dnsdistdist/Makefile.am +++ b/pdns/dnsdistdist/Makefile.am @@ -73,7 +73,7 @@ dnsdist_SOURCES = \ dnsdist-console.cc \ dnsdist-dnscrypt.cc \ dnsdist-ecs.cc dnsdist-ecs.hh \ - dnsdist-lua.cc \ + dnsdist-lua.hh dnsdist-lua.cc \ dnsdist-lua2.cc \ dnsdist-protobuf.cc dnsdist-protobuf.hh \ dnsdist-rings.cc \ diff --git a/pdns/dnsdistdist/dnsdist-lua.hh b/pdns/dnsdistdist/dnsdist-lua.hh new file mode 120000 index 0000000000..fab25c4c0c --- /dev/null +++ b/pdns/dnsdistdist/dnsdist-lua.hh @@ -0,0 +1 @@ +../dnsdist-lua.hh \ No newline at end of file diff --git a/regression-tests.dnsdist/test_CacheHitResponses.py b/regression-tests.dnsdist/test_CacheHitResponses.py new file mode 100644 index 0000000000..a6b8aafa6d --- /dev/null +++ b/regression-tests.dnsdist/test_CacheHitResponses.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import base64 +import time +import dns +from dnsdisttests import DNSDistTest + +class TestCacheHitResponses(DNSDistTest): + + _config_template = """ + pc = newPacketCache(100, 86400, 1) + getPool(""):setCache(pc) + addCacheHitResponseAction(makeRule("dropwhencached.cachehitresponses.tests.powerdns.com."), DropResponseAction()) + newServer{address="127.0.0.1:%s"} + """ + + def testDroppedWhenCached(self): + """ + CacheHitResponse: Drop when served from the cache + """ + ttl = 5 + name = 'dropwhencached.cachehitresponses.tests.powerdns.com.' + query = dns.message.make_query(name, 'AAAA', 'IN') + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + ttl, + dns.rdataclass.IN, + dns.rdatatype.AAAA, + '::1') + response.answer.append(rrset) + + # first query to fill the cache + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEquals(query, receivedQuery) + self.assertEquals(receivedResponse, response) + + # now the result should be cached, and so dropped + (_, receivedResponse) = self.sendUDPQuery(query, response=None, useQueue=False) + print(receivedResponse) + self.assertEquals(receivedResponse, None) + + time.sleep(ttl + 1) + + # should not be cached anymore and so valid + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEquals(query, receivedQuery) + self.assertEquals(receivedResponse, response) + + total = 0 + for key in self._responsesCounter: + total += self._responsesCounter[key] + TestCacheHitResponses._responsesCounter[key] = 0 + + self.assertEquals(total, 2) + + # TCP should not be cached + # first query to fill the cache + (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEquals(query, receivedQuery) + self.assertEquals(receivedResponse, response) + + # now the result should be cached, and so dropped + (_, receivedResponse) = self.sendTCPQuery(query, response=None, useQueue=False) + self.assertEquals(receivedResponse, None) + + time.sleep(ttl + 1) + + # should not be cached anymore and so valid + (receivedQuery, receivedResponse) = self.sendTCPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEquals(query, receivedQuery) + self.assertEquals(receivedResponse, response) + + total = 0 + for key in self._responsesCounter: + total += self._responsesCounter[key] + TestCacheHitResponses._responsesCounter[key] = 0 + + self.assertEquals(total, 2)