From: Remi Gacogne Date: Mon, 30 Jan 2023 17:15:05 +0000 (+0100) Subject: dnsdist: Add support for metadata in protobuf messages X-Git-Tag: dnsdist-1.8.0-rc1~15^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c02b67233460024de720620eb7ae990f236f9d04;p=thirdparty%2Fpdns.git dnsdist: Add support for metadata in protobuf messages --- diff --git a/contrib/ProtobufLogger.py b/contrib/ProtobufLogger.py index 27884d07ae..c6bb0269df 100644 --- a/contrib/ProtobufLogger.py +++ b/contrib/ProtobufLogger.py @@ -242,7 +242,7 @@ class PDNSPBConnHandler(object): requestorId = msg.requestorId nod = 0 - if (msg.HasField('newlyObservedDomain')): + if msg.HasField('newlyObservedDomain'): nod = msg.newlyObservedDomain print('[%s] %s of size %d: %s%s%s -> %s%s(%s) id: %d uuid: %s%s ' @@ -264,6 +264,15 @@ class PDNSPBConnHandler(object): serveridstr, nod)) + for mt in msg.meta: + values = '' + for entry in mt.value.stringVal: + values = ', '.join([values, entry]) if values != '' else entry + for entry in mt.value.intVal: + values = ', '.join([values, entry]) if values != '' else entry + + print('- %s -> %s' % (mt.key, values)) + def getRequestorSubnet(self, msg): requestorstr = None if msg.HasField('originalRequestorSubnet'): diff --git a/pdns/dnsdist-lua-actions.cc b/pdns/dnsdist-lua-actions.cc index 47f62a512d..175f365c42 100644 --- a/pdns/dnsdist-lua-actions.cc +++ b/pdns/dnsdist-lua-actions.cc @@ -1482,13 +1482,21 @@ private: boost::optional > d_alterFunc; }; +static void addMetaDataToProtobuf(DNSDistProtoBufMessage& message, const DNSQuestion& dq, const std::vector>& metas) +{ + for (const auto& [name, meta] : metas) { + message.addMeta(name, meta.getValue(dq)); + } +} + class RemoteLogAction : public DNSAction, public boost::noncopyable { public: // this action does not stop the processing - RemoteLogAction(std::shared_ptr& logger, boost::optional > alterFunc, const std::string& serverID, const std::string& ipEncryptKey): d_logger(logger), d_alterFunc(alterFunc), d_serverID(serverID), d_ipEncryptKey(ipEncryptKey) + RemoteLogAction(std::shared_ptr& logger, boost::optional > alterFunc, const std::string& serverID, const std::string& ipEncryptKey, std::vector>& metas): d_metas(std::move(metas)), d_logger(logger), d_alterFunc(alterFunc), d_serverID(serverID), d_ipEncryptKey(ipEncryptKey) { } + DNSAction::Action operator()(DNSQuestion* dq, std::string* ruleresult) const override { if (!dq->ids.uniqueId) { @@ -1507,6 +1515,8 @@ public: } #endif /* HAVE_IPCIPHER */ + addMetaDataToProtobuf(message, *dq, d_metas); + if (d_alterFunc) { auto lock = g_lua.lock(); (*d_alterFunc)(dq, &message); @@ -1524,6 +1534,7 @@ public: return "remote log to " + (d_logger ? d_logger->toString() : ""); } private: + std::vector> d_metas; std::shared_ptr d_logger; boost::optional > d_alterFunc; std::string d_serverID; @@ -1619,7 +1630,7 @@ class RemoteLogResponseAction : public DNSResponseAction, public boost::noncopya { public: // this action does not stop the processing - RemoteLogResponseAction(std::shared_ptr& logger, boost::optional > alterFunc, const std::string& serverID, const std::string& ipEncryptKey, bool includeCNAME): d_logger(logger), d_alterFunc(alterFunc), d_serverID(serverID), d_ipEncryptKey(ipEncryptKey), d_includeCNAME(includeCNAME) + RemoteLogResponseAction(std::shared_ptr& logger, boost::optional > alterFunc, const std::string& serverID, const std::string& ipEncryptKey, bool includeCNAME, std::vector>& metas): d_metas(std::move(metas)), d_logger(logger), d_alterFunc(alterFunc), d_serverID(serverID), d_ipEncryptKey(ipEncryptKey), d_includeCNAME(includeCNAME) { } DNSResponseAction::Action operator()(DNSResponse* dr, std::string* ruleresult) const override @@ -1640,6 +1651,8 @@ public: } #endif /* HAVE_IPCIPHER */ + addMetaDataToProtobuf(message, *dr, d_metas); + if (d_alterFunc) { auto lock = g_lua.lock(); (*d_alterFunc)(dr, &message); @@ -1657,6 +1670,7 @@ public: return "remote log response to " + (d_logger ? d_logger->toString() : ""); } private: + std::vector> d_metas; std::shared_ptr d_logger; boost::optional > d_alterFunc; std::string d_serverID; @@ -2448,7 +2462,7 @@ void setupLuaActions(LuaContext& luaCtx) }); #ifndef DISABLE_PROTOBUF - luaCtx.writeFunction("RemoteLogAction", [](std::shared_ptr logger, boost::optional > alterFunc, boost::optional> vars) { + luaCtx.writeFunction("RemoteLogAction", [](std::shared_ptr logger, boost::optional > alterFunc, boost::optional> vars, boost::optional> metas) { if (logger) { // avoids potentially-evaluated-expression warning with clang. RemoteLoggerInterface& rl = *logger.get(); @@ -2462,12 +2476,20 @@ void setupLuaActions(LuaContext& luaCtx) std::string ipEncryptKey; getOptionalValue(vars, "serverID", serverID); getOptionalValue(vars, "ipEncryptKey", ipEncryptKey); + + std::vector> metaOptions; + if (metas) { + for (const auto& [key, value] : *metas) { + metaOptions.push_back({key, ProtoBufMetaKey(value)}); + } + } + checkAllParametersConsumed("RemoteLogAction", vars); - return std::shared_ptr(new RemoteLogAction(logger, alterFunc, serverID, ipEncryptKey)); + return std::shared_ptr(new RemoteLogAction(logger, alterFunc, serverID, ipEncryptKey, metaOptions)); }); - luaCtx.writeFunction("RemoteLogResponseAction", [](std::shared_ptr logger, boost::optional > alterFunc, boost::optional includeCNAME, boost::optional> vars) { + luaCtx.writeFunction("RemoteLogResponseAction", [](std::shared_ptr logger, boost::optional > alterFunc, boost::optional includeCNAME, boost::optional> vars, boost::optional> metas) { if (logger) { // avoids potentially-evaluated-expression warning with clang. RemoteLoggerInterface& rl = *logger.get(); @@ -2481,9 +2503,17 @@ void setupLuaActions(LuaContext& luaCtx) std::string ipEncryptKey; getOptionalValue(vars, "serverID", serverID); getOptionalValue(vars, "ipEncryptKey", ipEncryptKey); + + std::vector> metaOptions; + if (metas) { + for (const auto& [key, value] : *metas) { + metaOptions.push_back({key, ProtoBufMetaKey(value)}); + } + } + checkAllParametersConsumed("RemoteLogResponseAction", vars); - return std::shared_ptr(new RemoteLogResponseAction(logger, alterFunc, serverID, ipEncryptKey, includeCNAME ? *includeCNAME : false)); + return std::shared_ptr(new RemoteLogResponseAction(logger, alterFunc, serverID, ipEncryptKey, includeCNAME ? *includeCNAME : false, metaOptions)); }); luaCtx.writeFunction("DnstapLogAction", [](const std::string& identity, std::shared_ptr logger, boost::optional > alterFunc) { diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index e77170d94f..f9aab09ce6 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -285,7 +285,7 @@ extern "C" } #endif -bool checkConfigurationTime(const std::string& name) +static bool checkConfigurationTime(const std::string& name) { if (!g_configurationDone) { return true; diff --git a/pdns/dnsdist-protobuf.cc b/pdns/dnsdist-protobuf.cc index 29e9b84c62..b20437bc61 100644 --- a/pdns/dnsdist-protobuf.cc +++ b/pdns/dnsdist-protobuf.cc @@ -22,6 +22,7 @@ #include "config.h" #ifndef DISABLE_PROTOBUF +#include "base64.hh" #include "dnsdist.hh" #include "dnsdist-protobuf.hh" #include "protozero.hh" @@ -103,6 +104,11 @@ void DNSDistProtoBufMessage::addTag(const std::string& strValue) d_additionalTags.push_back(strValue); } +void DNSDistProtoBufMessage::addMeta(const std::string& key, std::string&& value) +{ + d_metaTags.push_back({key, std::move(value)}); +} + void DNSDistProtoBufMessage::addRR(DNSName&& qname, uint16_t uType, uint16_t uClass, uint32_t uTTL, const std::string& strBlob) { d_additionalRRs.push_back({std::move(qname), strBlob, uTTL, uType, uClass}); @@ -186,6 +192,134 @@ void DNSDistProtoBufMessage::serialize(std::string& data) const } m.commitResponse(); + + for (const auto& [key, value] : d_metaTags) { + m.setMeta(key, {value}, {}); + } +} + +ProtoBufMetaKey::ProtoBufMetaKey(const std::string& key) +{ + auto& idx = s_types.get(); + auto it = idx.find(key); + if (it != idx.end()) { + d_type = it->d_type; + return; + } + else { + auto [prefix, variable] = splitField(key, ':'); + if (!variable.empty()) { + it = idx.find(prefix); + if (it != idx.end() && it->d_prefix) { + d_type = it->d_type; + if (it->d_numeric) { + try { + d_numericSubKey = std::stoi(variable); + } + catch (const std::exception& e) { + throw std::runtime_error("Unable to parse numeric ProtoBuf key '" + key + "'"); + } + } + else { + if (!it->d_caseSensitive) { + boost::algorithm::to_lower(variable); + } + d_subKey = variable; + } + return; + } + } + } + throw std::runtime_error("Invalid ProtoBuf key '" + key + "'"); +} + +std::string ProtoBufMetaKey::getValue(const DNSQuestion& dq) const +{ + auto& idx = s_types.get(); + auto it = idx.find(d_type); + if (it == idx.end()) { + throw std::runtime_error("Trying to get the value of an unsupported type: " + std::to_string(static_cast(d_type))); + } + return (it->d_func)(dq, d_subKey, d_numericSubKey); } +const std::string& ProtoBufMetaKey::getName() const +{ + auto& idx = s_types.get(); + auto it = idx.find(d_type); + if (it == idx.end()) { + throw std::runtime_error("Trying to get the name of an unsupported type: " + std::to_string(static_cast(d_type))); + } + return it->d_name; +} + +const ProtoBufMetaKey::TypeContainer ProtoBufMetaKey::s_types = { + ProtoBufMetaKey::KeyTypeDescription{ "sni", Type::SNI, [](const DNSQuestion& dq, const std::string&, uint8_t) -> std::string { return dq.sni; }, false }, + ProtoBufMetaKey::KeyTypeDescription{ "pool", Type::Pool, [](const DNSQuestion& dq, const std::string&, uint8_t) -> std::string { return dq.ids.poolName; }, false }, + ProtoBufMetaKey::KeyTypeDescription{ "b64-content", Type::B64Content, [](const DNSQuestion& dq, const std::string&, uint8_t) -> std::string { const auto& data = dq.getData(); return Base64Encode(std::string(data.begin(), data.end())); }, false }, + ProtoBufMetaKey::KeyTypeDescription{ "doh-header", Type::DoHHeader, [](const DNSQuestion& dq , const std::string& name, uint8_t) { + if (!dq.ids.du) { + return std::string(); + } + auto headers = dq.ids.du->getHTTPHeaders(); + auto it = headers.find(name); + if (it != headers.end()) { + return it->second; + } + return std::string(); + }, true, false }, + ProtoBufMetaKey::KeyTypeDescription{ "doh-host", Type::DoHHost, [](const DNSQuestion& dq, const std::string&, uint8_t) { return (dq.ids.du ? dq.ids.du->getHTTPHost() : ""); }, true, false }, + ProtoBufMetaKey::KeyTypeDescription{ "doh-path", Type::DoHPath, [](const DNSQuestion& dq, const std::string&, uint8_t) { return (dq.ids.du ? dq.ids.du->getHTTPPath() : ""); }, false }, + ProtoBufMetaKey::KeyTypeDescription{ "doh-query-string", Type::DoHQueryString, [](const DNSQuestion& dq, const std::string&, uint8_t) { return (dq.ids.du ? dq.ids.du->getHTTPQueryString() : ""); }, false }, + ProtoBufMetaKey::KeyTypeDescription{ "doh-scheme", Type::DoHScheme, [](const DNSQuestion& dq, const std::string&, uint8_t) { return (dq.ids.du ? dq.ids.du->getHTTPScheme() : ""); }, false, false }, + ProtoBufMetaKey::KeyTypeDescription{ "proxy-protocol-value", Type::ProxyProtocolValue, [](const DNSQuestion& dq, const std::string&, uint8_t numericSubKey) { + if (!dq.proxyProtocolValues) { + return std::string(); + } + for (const auto& value : *dq.proxyProtocolValues) { + if (value.type == numericSubKey) { + return value.content; + } + } + return std::string(); + }, true, false, true }, + ProtoBufMetaKey::KeyTypeDescription{ "proxy-protocol-values", Type::ProxyProtocolValues, [](const DNSQuestion& dq, const std::string&, uint8_t) { + if (!dq.proxyProtocolValues) { + return std::string(); + } + std::string result; + for (const auto& value : *dq.proxyProtocolValues) { + if (!result.empty()) { + result += ", "; + } + result += std::to_string(value.type) + ":" + value.content; + } + return result; + } }, + ProtoBufMetaKey::KeyTypeDescription{ "tag", Type::Tag, [](const DNSQuestion& dq, const std::string& subKey, uint8_t) { + if (!dq.ids.qTag) { + return std::string(); + } + for (const auto& [key, value] : *dq.ids.qTag) { + if (key == subKey) { + return value; + } + } + return std::string(); + }, true, true }, + ProtoBufMetaKey::KeyTypeDescription{ "tags", Type::Tags, [](const DNSQuestion& dq, const std::string&, uint8_t) { + if (!dq.ids.qTag) { + return std::string(); + } + std::string result; + for (const auto& [key, value] : *dq.ids.qTag) { + if (!result.empty()) { + result += ", "; + } + result += key + ":" + value; + } + return result; + } }, +}; + #endif /* DISABLE_PROTOBUF */ diff --git a/pdns/dnsdist-protobuf.hh b/pdns/dnsdist-protobuf.hh index 4a5cbd9914..1133f231f3 100644 --- a/pdns/dnsdist-protobuf.hh +++ b/pdns/dnsdist-protobuf.hh @@ -47,6 +47,7 @@ public: void setEDNSSubnet(const Netmask& nm); void addTag(const std::string& strValue); + void addMeta(const std::string& key, std::string&& value); void addRR(DNSName&& qname, uint16_t uType, uint16_t uClass, uint32_t uTTL, const std::string& data); void serialize(std::string& data) const; @@ -75,6 +76,7 @@ private: std::vector d_additionalRRs; std::vector d_additionalTags; + std::vector> d_metaTags; const DNSQuestion& d_dq; const DNSResponse* d_dr{nullptr}; @@ -94,4 +96,42 @@ private: bool d_includeCNAME{false}; }; +class ProtoBufMetaKey +{ + enum class Type : uint8_t { SNI, Pool, B64Content, DoHHeader, DoHHost, DoHPath, DoHQueryString, DoHScheme, ProxyProtocolValue, ProxyProtocolValues, Tag, Tags }; + + struct KeyTypeDescription + { + const std::string d_name; + const Type d_type; + const std::function d_func; + bool d_prefix{false}; + bool d_caseSensitive{true}; + bool d_numeric{false}; + }; + + struct NameTag {}; + struct TypeTag {}; + + typedef boost::multi_index_container< + KeyTypeDescription, + indexed_by < + hashed_unique, member>, + hashed_unique, member> + > + > TypeContainer; + + static const TypeContainer s_types; + +public: + ProtoBufMetaKey(const std::string& key); + + const std::string& getName() const; + std::string getValue(const DNSQuestion& dq) const; +private: + std::string d_subKey; + uint8_t d_numericSubKey{0}; + Type d_type; +}; + #endif /* DISABLE_PROTOBUF */ diff --git a/pdns/dnsdistdist/docs/rules-actions.rst b/pdns/dnsdistdist/docs/rules-actions.rst index 38bf7b564b..2209b0ecdb 100644 --- a/pdns/dnsdistdist/docs/rules-actions.rst +++ b/pdns/dnsdistdist/docs/rules-actions.rst @@ -1316,39 +1316,64 @@ The following actions exist. * ``ad``: bool - Set the AD bit to this value (true means the bit is set, false means it's cleared). Default is to clear it. * ``ra``: bool - Set the RA bit to this value (true means the bit is set, false means it's cleared). Default is to copy the value of the RD bit from the incoming query. -.. function:: RemoteLogAction(remoteLogger[, alterFunction [, options]]) +.. function:: RemoteLogAction(remoteLogger[, alterFunction [, options [, metas]]]) .. versionchanged:: 1.4.0 ``ipEncryptKey`` optional key added to the options table. + .. versionchanged:: 1.8.0 + ``metas`` optional parameter added. + Send the content of this query to a remote logger via Protocol Buffer. ``alterFunction`` is a callback, receiving a :class:`DNSQuestion` and a :class:`DNSDistProtoBufMessage`, that can be used to modify the Protocol Buffer content, for example for anonymization purposes. + Since 1.8.0 it is possible to add configurable meta-data fields to the Protocol Buffer message via the ``metas`` parameter, which takes a list of ``name``=``key`` pairs. For each entry in the list, a new value named ``name`` + will be added to the message with the value corresponding to the ``key``. Available keys are: + + * ``doh-header:
``: the content of the corresponding ``
`` HTTP header for DoH queries, empty otherwise + * ``doh-host``: the ``Host`` header for DoH queries, empty otherwise + * ``doh-path``: the HTTP path for DoH queries, empty otherwise + * ``doh-query-string``: the HTTP query string for DoH queries, empty otherwise + * ``doh-scheme``: the HTTP scheme for DoH queries, empty otherwise + * ``pool``: the currently selected pool of servers + * ``proxy-protocol-value:``: the content of the proxy protocol value of type ````, if any + * ``proxy-protocol-values``: the content of all proxy protocol values as a ":,...,:" string + * ``b64-content``: the base64-encoded DNS payload of the current query + * ``sni``: the Server Name Indication value for queries received over DoT or DoH. Empty otherwise. + * ``tag:``: the content of the corresponding ```` if any + * ``tags``: the list of all tags, and their values, as a ":,...,:" string + Subsequent rules are processed after this action. :param string remoteLogger: The :func:`remoteLogger ` object to write to :param string alterFunction: Name of a function to modify the contents of the logs before sending :param table options: A table with key: value pairs. + :param table metas: A list of name: key pairs, for meta-data to be added to Protocol Buffer message. Options: * ``serverID=""``: str - Set the Server Identity field. * ``ipEncryptKey=""``: str - A key, that can be generated via the :func:`makeIPCipherKey` function, to encrypt the IP address of the requestor for anonymization purposes. The encryption is done using ipcrypt for IPv4 and a 128-bit AES ECB operation for IPv6. -.. function:: RemoteLogResponseAction(remoteLogger[, alterFunction[, includeCNAME [, options]]]) +.. function:: RemoteLogResponseAction(remoteLogger[, alterFunction[, includeCNAME [, options [, metas]]]]) .. versionchanged:: 1.4.0 ``ipEncryptKey`` optional key added to the options table. + .. versionchanged:: 1.8.0 + ``metas`` optional parameter added. + Send the content of this response to a remote logger via Protocol Buffer. ``alterFunction`` is the same callback that receiving a :class:`DNSQuestion` and a :class:`DNSDistProtoBufMessage`, that can be used to modify the Protocol Buffer content, for example for anonymization purposes. ``includeCNAME`` indicates whether CNAME records inside the response should be parsed and exported. The default is to only exports A and AAAA records. + Since 1.8.0 it is possible to add configurable meta-data fields to the Protocol Buffer message via the ``metas`` parameter, which takes a list of ``name``=``key`` pairs. See :func:`RemoteLogAction` for the list of available keys. Subsequent rules are processed after this action. :param string remoteLogger: The :func:`remoteLogger ` object to write to :param string alterFunction: Name of a function to modify the contents of the logs before sending :param bool includeCNAME: Whether or not to parse and export CNAMEs. Default false :param table options: A table with key: value pairs. + :param table metas: A list of name: key pairs, for meta-data to be added to Protocol Buffer message. Options: diff --git a/regression-tests.dnsdist/test_Protobuf.py b/regression-tests.dnsdist/test_Protobuf.py index 7f0e7ba4f0..31dfc085da 100644 --- a/regression-tests.dnsdist/test_Protobuf.py +++ b/regression-tests.dnsdist/test_Protobuf.py @@ -1,10 +1,12 @@ #!/usr/bin/env python +import base64 import threading import socket import struct import sys import time from dnsdisttests import DNSDistTest, Queue +from proxyprotocol import ProxyProtocol import dns import dnsmessage_pb2 @@ -75,14 +77,20 @@ class DNSDistProtobufTest(DNSDistTest): msg.ParseFromString(data) return msg - def checkProtobufBase(self, msg, protocol, query, initiator, normalQueryResponse=True): + def checkProtobufBase(self, msg, protocol, query, initiator, normalQueryResponse=True, v6=False): self.assertTrue(msg) self.assertTrue(msg.HasField('timeSec')) self.assertTrue(msg.HasField('socketFamily')) - self.assertEqual(msg.socketFamily, dnsmessage_pb2.PBDNSMessage.INET) + if v6: + self.assertEqual(msg.socketFamily, dnsmessage_pb2.PBDNSMessage.INET6) + else: + self.assertEqual(msg.socketFamily, dnsmessage_pb2.PBDNSMessage.INET) self.assertTrue(msg.HasField('from')) fromvalue = getattr(msg, 'from') - self.assertEqual(socket.inet_ntop(socket.AF_INET, fromvalue), initiator) + if v6: + self.assertEqual(socket.inet_ntop(socket.AF_INET6, fromvalue), initiator) + else: + self.assertEqual(socket.inet_ntop(socket.AF_INET, fromvalue), initiator) self.assertTrue(msg.HasField('socketProtocol')) self.assertEqual(msg.socketProtocol, protocol) self.assertTrue(msg.HasField('messageId')) @@ -92,7 +100,7 @@ class DNSDistProtobufTest(DNSDistTest): self.assertTrue(msg.HasField('serverIdentity')) self.assertEqual(msg.serverIdentity, self._protobufServerID.encode('utf-8')) - if normalQueryResponse: + if normalQueryResponse and (protocol == dnsmessage_pb2.PBDNSMessage.UDP or protocol == dnsmessage_pb2.PBDNSMessage.TCP): # compare inBytes with length of query/response self.assertEqual(msg.inBytes, len(query.to_wire())) # dnsdist doesn't set the existing EDNS Subnet for now, @@ -101,13 +109,14 @@ class DNSDistProtobufTest(DNSDistTest): # self.assertEqual(len(msg.originalRequestorSubnet), 4) # self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.originalRequestorSubnet), '127.0.0.1') - def checkProtobufQuery(self, msg, protocol, query, qclass, qtype, qname, initiator='127.0.0.1'): + def checkProtobufQuery(self, msg, protocol, query, qclass, qtype, qname, initiator='127.0.0.1', v6=False): self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSQueryType) - self.checkProtobufBase(msg, protocol, query, initiator) + self.checkProtobufBase(msg, protocol, query, initiator, v6=v6) # dnsdist doesn't fill the responder field for responses # because it doesn't keep the information around. self.assertTrue(msg.HasField('to')) - self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.to), '127.0.0.1') + if not v6: + self.assertEqual(socket.inet_ntop(socket.AF_INET, msg.to), '127.0.0.1') self.assertTrue(msg.HasField('question')) self.assertTrue(msg.question.HasField('qClass')) self.assertEqual(msg.question.qClass, qclass) @@ -129,9 +138,9 @@ class DNSDistProtobufTest(DNSDistTest): self.assertTrue(msg.HasField('response')) self.assertTrue(msg.response.HasField('queryTimeSec')) - def checkProtobufResponse(self, msg, protocol, response, initiator='127.0.0.1'): + def checkProtobufResponse(self, msg, protocol, response, initiator='127.0.0.1', v6=False): self.assertEqual(msg.type, dnsmessage_pb2.PBDNSMessage.DNSResponseType) - self.checkProtobufBase(msg, protocol, response, initiator) + self.checkProtobufBase(msg, protocol, response, initiator, v6=v6) self.assertTrue(msg.HasField('response')) self.assertTrue(msg.response.HasField('queryTimeSec')) @@ -402,6 +411,205 @@ class TestProtobuf(DNSDistProtobufTest): self.checkProtobufResponseRecord(rr, dns.rdataclass.IN, dns.rdatatype.A, name, 3600) self.assertEqual(socket.inet_ntop(socket.AF_INET, rr.rdata), '127.0.0.1') +class TestProtobufMetaTags(DNSDistProtobufTest): + _config_params = ['_testServerPort', '_protobufServerPort'] + _config_template = """ + newServer{address="127.0.0.1:%s"} + rl = newRemoteLogger('127.0.0.1:%d') + + addAction(AllRule(), SetTagAction('my-tag-key', 'my-tag-value')) + addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1'}, {b64='b64-content', ['my-tag-export-name']='tag:my-tag-key'})) + addResponseAction(AllRule(), SetTagResponseAction('my-tag-key2', 'my-tag-value2')) + addResponseAction(AllRule(), RemoteLogResponseAction(rl, nil, false, {serverID='dnsdist-server-1'}, {['my-tag-export-name']='tags'})) + """ + + def testProtobufMeta(self): + """ + Protobuf: Meta values + """ + name = 'meta.protobuf.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + (receivedQuery, receivedResponse) = self.sendUDPQuery(query, response) + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + + # let the protobuf messages the time to get there + time.sleep(1) + + # check the protobuf message corresponding to the UDP query + msg = self.getFirstProtobufMessage() + + self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name) + self.assertEqual(len(msg.meta), 2) + self.assertEqual(msg.meta[0].key, 'b64') + b64EncodedQuery = base64.b64encode(query.to_wire()).decode('ascii') + self.assertEqual(msg.meta[0].value.stringVal[0], b64EncodedQuery) + self.assertEqual(msg.meta[1].key, 'my-tag-export-name') + self.assertEqual(msg.meta[1].value.stringVal[0], 'my-tag-value') + + # check the protobuf message corresponding to the UDP response + msg = self.getFirstProtobufMessage() + self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.UDP, response) + self.assertEqual(len(msg.meta), 1) + self.assertEqual(msg.meta[0].key, 'my-tag-export-name') + self.assertEqual(msg.meta[0].value.stringVal[0], 'my-tag-key2:my-tag-value2, my-tag-key:my-tag-value') + +class TestProtobufMetaDOH(DNSDistProtobufTest): + + _serverKey = 'server.key' + _serverCert = 'server.chain' + _serverName = 'tls.tests.dnsdist.org' + _caCert = 'ca.pem' + _dohServerPort = 8443 + _dohBaseURL = ("https://%s:%d/dns-query" % (_serverName, _dohServerPort)) + _config_params = ['_testServerPort', '_protobufServerPort', '_dohServerPort', '_serverCert', '_serverKey'] + _config_template = """ + newServer{address="127.0.0.1:%d"} + rl = newRemoteLogger('127.0.0.1:%d') + addDOHLocal("127.0.0.1:%s", "%s", "%s", { '/dns-query' }, { keepIncomingHeaders=true }) + + local mytags = {path='doh-path', host='doh-host', ['query-string']='doh-query-string', scheme='doh-scheme', agent='doh-header:user-agent'} + addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1'}, mytags)) + addResponseAction(AllRule(), RemoteLogResponseAction(rl, nil, false, {serverID='dnsdist-server-1'}, mytags)) + """ + + def testProtobufMetaDoH(self): + """ + Protobuf: Meta values - DoH + """ + name = 'meta-doh.protobuf.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + (receivedQuery, receivedResponse) = self.sendDOHQuery(self._dohServerPort, self._serverName, self._dohBaseURL, query, caFile=self._caCert, response=response) + + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + + # let the protobuf messages the time to get there + time.sleep(1) + + # check the protobuf message corresponding to the UDP query + msg = self.getFirstProtobufMessage() + + self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.DOH, query, dns.rdataclass.IN, dns.rdatatype.A, name) + self.assertEqual(len(msg.meta), 5) + tags = {} + for entry in msg.meta: + self.assertEqual(len(entry.value.stringVal), 1) + tags[entry.key] = entry.value.stringVal[0] + + self.assertIn('agent', tags) + self.assertIn('PycURL', tags['agent']) + self.assertIn('host', tags) + self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohServerPort)) + self.assertIn('path', tags) + self.assertEqual(tags['path'], '/dns-query') + self.assertIn('query-string', tags) + self.assertIn('?dns=', tags['query-string']) + self.assertIn('scheme', tags) + self.assertEqual(tags['scheme'], 'https') + + # check the protobuf message corresponding to the UDP response + msg = self.getFirstProtobufMessage() + self.checkProtobufResponse(msg, dnsmessage_pb2.PBDNSMessage.DOH, response) + self.assertEqual(len(msg.meta), 5) + tags = {} + for entry in msg.meta: + self.assertEqual(len(entry.value.stringVal), 1) + tags[entry.key] = entry.value.stringVal[0] + + self.assertIn('agent', tags) + self.assertIn('PycURL', tags['agent']) + self.assertIn('host', tags) + self.assertEqual(tags['host'], self._serverName + ':' + str(self._dohServerPort)) + self.assertIn('path', tags) + self.assertEqual(tags['path'], '/dns-query') + self.assertIn('query-string', tags) + self.assertIn('?dns=', tags['query-string']) + self.assertIn('scheme', tags) + self.assertEqual(tags['scheme'], 'https') + +class TestProtobufMetaProxy(DNSDistProtobufTest): + + _config_params = ['_testServerPort', '_protobufServerPort'] + _config_template = """ + setProxyProtocolACL( { "127.0.0.1/32" } ) + + newServer{address="127.0.0.1:%d"} + rl = newRemoteLogger('127.0.0.1:%d') + + local mytags = {pp='proxy-protocol-values', pp42='proxy-protocol-value:42'} + addAction(AllRule(), RemoteLogAction(rl, nil, {serverID='dnsdist-server-1'}, mytags)) + + -- proxy protocol values are NOT passed to the response + """ + + def testProtobufMetaProxy(self): + """ + Protobuf: Meta values - Proxy + """ + name = 'meta-proxy.protobuf.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 3600, + dns.rdataclass.IN, + dns.rdatatype.A, + '127.0.0.1') + response.answer.append(rrset) + + destAddr = "2001:db8::9" + destPort = 9999 + srcAddr = "2001:db8::8" + srcPort = 8888 + udpPayload = ProxyProtocol.getPayload(False, False, True, srcAddr, destAddr, srcPort, destPort, [ [ 2, b'foo'], [ 42, b'proxy'] ]) + (receivedQuery, receivedResponse) = self.sendUDPQuery(udpPayload + query.to_wire(), response, rawQuery=True) + + self.assertTrue(receivedQuery) + self.assertTrue(receivedResponse) + receivedQuery.id = query.id + self.assertEqual(query, receivedQuery) + self.assertEqual(response, receivedResponse) + + # let the protobuf messages the time to get there + time.sleep(1) + + # check the protobuf message corresponding to the UDP query + msg = self.getFirstProtobufMessage() + + self.checkProtobufQuery(msg, dnsmessage_pb2.PBDNSMessage.UDP, query, dns.rdataclass.IN, dns.rdatatype.A, name, initiator='2001:db8::8', v6=True) + self.assertEqual(len(msg.meta), 2) + tags = {} + for entry in msg.meta: + self.assertEqual(len(entry.value.stringVal), 1) + tags[entry.key] = entry.value.stringVal[0] + + self.assertIn('pp42', tags) + self.assertEqual(tags['pp42'], 'proxy') + self.assertIn('pp', tags) + self.assertEqual(tags['pp'], '2:foo, 42:proxy') + class TestProtobufIPCipher(DNSDistProtobufTest): _config_params = ['_testServerPort', '_protobufServerPort', '_protobufServerID', '_protobufServerID'] _config_template = """