From: Otto Moerbeek Date: Tue, 13 Oct 2020 15:21:20 +0000 (+0200) Subject: Things are starting to work, but I'll need to split the cached X-Git-Tag: dnsdist-1.6.0-alpha0~11^2~12 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=a44a8d6617e3062846e6092a3fd836f11be757a7;p=thirdparty%2Fpdns.git Things are starting to work, but I'll need to split the cached protobuf strings to allow for mod of the response. --- diff --git a/contrib/ProtobufLogger.py b/contrib/ProtobufLogger.py index a423c38f27..48a5763987 100644 --- a/contrib/ProtobufLogger.py +++ b/contrib/ProtobufLogger.py @@ -173,19 +173,28 @@ class PDNSPBConnHandler(object): if msg.HasField('initialRequestId'): initialrequestidstr = ', initial uuid: %s ' % (binascii.hexlify(bytearray(msg.initialRequestId))) - requestorstr = '' + requestorstr = '(N/A)' requestor = self.getRequestorSubnet(msg) if requestor: requestorstr = ' (' + requestor + ')' - deviceId = binascii.hexlify(bytearray(msg.deviceId)) - requestorId = msg.requestorId + deviceId = 'N/A' + if msg.HasField('deviceId'): + deviceId = binascii.hexlify(bytearray(msg.deviceId)) + deviceName = 'N/A' + if msg.HasField('deviceName'): + deviceName = msg.deviceName + + requestorId = 'N/A' + if msg.HasField('requestorId'): + requestorId = msg.requestorId + nod = 0 if (msg.HasField('newlyObservedDomain')): nod = msg.newlyObservedDomain - print('[%s] %s of size %d: %s%s%s -> %s%s (%s), id: %d, uuid: %s%s ' - 'requestorid: %s deviceid: %s serverid: %s nod: %d' % (datestr, + print('[%s] %s of size %d: %s%s%s -> %s%s(%s) id: %d uuid: %s%s ' + 'requestorid: %s deviceid: %s devicename: %s serverid: %s nod: %d' % (datestr, typestr, msg.inBytes, ipfromstr, @@ -199,6 +208,7 @@ class PDNSPBConnHandler(object): initialrequestidstr, requestorId, deviceId, + deviceName, serveridstr, nod)) diff --git a/pdns/dnsname.cc b/pdns/dnsname.cc index 3c360f18a0..ecb8e98d0d 100644 --- a/pdns/dnsname.cc +++ b/pdns/dnsname.cc @@ -161,16 +161,26 @@ void DNSName::packetParser(const char* qpos, int len, int offset, bool uncompres } std::string DNSName::toString(const std::string& separator, const bool trailing) const +{ + std::string ret; + toString(ret, separator, trailing); + return ret; +} + +void DNSName::toString(std::string& output, const std::string& separator, const bool trailing) const { if (empty()) { throw std::out_of_range("Attempt to print an unset dnsname"); } - if(isRoot()) - return trailing ? separator : ""; + if (isRoot()) { + output += (trailing ? separator : ""); + return; + } - std::string ret; - ret.reserve(d_storage.size()); + if (output.capacity() < (output.size() + d_storage.size())) { + output.reserve(output.size() + d_storage.size()); + } { // iterate over the raw labels @@ -178,15 +188,15 @@ std::string DNSName::toString(const std::string& separator, const bool trailing) const char* end = p + d_storage.size(); while (p < end && *p) { - appendEscapedLabel(ret, p + 1, static_cast(*p)); - ret += separator; + appendEscapedLabel(output, p + 1, static_cast(*p)); + output += separator; p += *p + 1; } } + if (!trailing) { - ret.resize(ret.size() - separator.size()); + output.resize(output.size() - separator.size()); } - return ret; } std::string DNSName::toLogString() const diff --git a/pdns/dnsname.hh b/pdns/dnsname.hh index 222edbaf38..1e146e831f 100644 --- a/pdns/dnsname.hh +++ b/pdns/dnsname.hh @@ -90,6 +90,7 @@ public: bool operator!=(const DNSName& other) const { return !(*this == other); } std::string toString(const std::string& separator=".", const bool trailing=true) const; //!< Our human-friendly, escaped, representation + void toString(std::string& output, const std::string& separator=".", const bool trailing=true) const; std::string toLogString() const; //!< like plain toString, but returns (empty) on empty names std::string toStringNoDot() const { return toString(".", false); } std::string toStringRootDot() const { if(isRoot()) return "."; else return toString(".", false); } diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index f552d569ad..6b81c3e2de 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -107,6 +107,7 @@ #ifdef HAVE_PROTOBUF #include "uuid-utils.hh" +#include "protozero.hh" #endif /* HAVE_PROTOBUF */ #include "xpf.hh" @@ -884,38 +885,35 @@ static void protobufLogQuery(uint8_t maskV4, uint8_t maskV6, const boost::uuids: Netmask requestorNM(remote, remote.sin4.sin_family == AF_INET ? maskV4 : maskV6); ComboAddress requestor = requestorNM.getMaskedNetwork(); requestor.setPort(remote.getPort()); - RecProtoBufMessage message(DNSProtoBufMessage::Query, uniqueId, &requestor, &local, qname, qtype, qclass, id, tcp, len); - message.setServerIdentity(SyncRes::s_serverID); - message.setEDNSSubnet(ednssubnet, ednssubnet.isIPv4() ? maskV4 : maskV6); - message.setRequestorId(requestorId); - message.setDeviceId(deviceId); - message.setDeviceName(deviceName); + + pdns::ProtoZero::Message m{128}; // guess at size + m.request(uniqueId, requestor, local, qname, qtype, qclass, id, tcp, len); + m.setServerIdentity(SyncRes::s_serverID); + m.setEDNSSubnet(ednssubnet, ednssubnet.isIPv4() ? maskV4 : maskV6); + m.setRequestorId(requestorId); + m.setDeviceId(deviceId); + m.setDeviceName(deviceName); if (!policyTags.empty()) { - message.setPolicyTags(policyTags); + m.startResponse(); + m.setPolicyTags(policyTags); + m.finishResponse(); } -// cerr <queueData(str); + server->queueData(m.getbuf()); } } -static void protobufLogResponse(const RecProtoBufMessage& message) +static void protobufLogResponse(const pdns::ProtoZero::Message& message) { if (!t_protobufServers) { return; } -// cerr <queueData(str); + server->queueData(data); } } #endif @@ -1407,15 +1405,16 @@ static void startDoResolve(void *p) auto luaconfsLocal = g_luaconfs.getLocal(); // Used to tell syncres later on if we should apply NSDNAME and NSIP RPZ triggers for this query bool wantsRPZ(true); - boost::optional pbMessage(boost::none); + std::unique_ptr pbMessage; + std::string::size_type pbUnmutablePart; // defines the (first) part of the protobuf message that is equal for everyone #ifdef HAVE_PROTOBUF if (checkProtobufExport(luaconfsLocal)) { - Netmask requestorNM(dc->d_source, dc->d_source.sin4.sin_family == AF_INET ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); - ComboAddress requestor = requestorNM.getMaskedNetwork(); - requestor.setPort(dc->d_source.getPort()); - pbMessage = RecProtoBufMessage(RecProtoBufMessage::Response, dc->d_uuid, &requestor, &dc->d_destination, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass, dc->d_mdp.d_header.id, dc->d_tcp, 0); + pbMessage = make_unique(128); // guess at size + pbMessage->response(dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass); pbMessage->setServerIdentity(SyncRes::s_serverID); - pbMessage->setEDNSSubnet(dc->d_ednssubnet.source, dc->d_ednssubnet.source.isIPv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); + + // RRSets added below + pbMessage->startResponse(); } #endif /* HAVE_PROTOBUF */ @@ -1862,7 +1861,7 @@ static void startDoResolve(void *p) #endif /* NOD_ENABLED */ #ifdef HAVE_PROTOBUF if (t_protobufServers && !(luaconfsLocal->protobufExportConfig.taggedOnly && appliedPolicy.getName().empty() && dc->d_policyTags.empty())) { - pbMessage->setBytes(packet.size()); + // Start constructing embedded DNSResponse object pbMessage->setResponseCode(pw.getHeader()->rcode); if (!appliedPolicy.getName().empty()) { pbMessage->setAppliedPolicy(appliedPolicy.getName()); @@ -1870,20 +1869,44 @@ static void startDoResolve(void *p) pbMessage->setAppliedPolicyTrigger(appliedPolicy.d_trigger); pbMessage->setAppliedPolicyHit(appliedPolicy.d_hit); } + // XXX if (nod) pbMessage->setPolicyTags(dc->d_policyTags); + pbMessage->finishResponse(); + pbMessage->setInBytes(packet.size()); + + // Done with embedded DNSResponse object and done with unmutable part. + pbUnmutablePart = pbMessage->getbuf().length(); + + // Below arte the fields that are not stored in the packet cache and will be appended here and on a cache hit + pbMessage->startResponse(); if (g_useKernelTimestamp && dc->d_kernelTimestamp.tv_sec) { pbMessage->setQueryTime(dc->d_kernelTimestamp.tv_sec, dc->d_kernelTimestamp.tv_usec); } else { pbMessage->setQueryTime(dc->d_now.tv_sec, dc->d_now.tv_usec); } + pbMessage->finishResponse(); + pbMessage->setMessageIdentity(dc->d_uuid); + pbMessage->setSocketFamily(dc->d_source.sin4.sin_family); + pbMessage->setSocketProtocol(dc->d_tcp); + Netmask requestorNM(dc->d_source, dc->d_source.sin4.sin_family == AF_INET ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); + ComboAddress requestor = requestorNM.getMaskedNetwork(); + pbMessage->setFrom(requestor); + pbMessage->setTo(dc->d_destination); + pbMessage->setId(dc->d_mdp.d_header.id); + + // Th exact semantics of timeSec/timeUsec vs queryTimeSec/queryTimeUsec are unclear to me ATM + pbMessage->setTime(); + pbMessage->setEDNSSubnet(dc->d_ednssubnet.source, dc->d_ednssubnet.source.isIPv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); pbMessage->setRequestorId(dq.requestorId); pbMessage->setDeviceId(dq.deviceId); pbMessage->setDeviceName(dq.deviceName); + pbMessage->setFromPort(dc->d_source.getPort()); + pbMessage->setToPort(dc->d_destination.getPort()); #ifdef NOD_ENABLED if (g_nodEnabled) { if (nod) { - pbMessage->setNOD(true); + pbMessage->setNewlyObservedDomain(true); pbMessage->addPolicyTag(g_nod_pbtag); } if (hasUDR) { @@ -1896,7 +1919,7 @@ static void startDoResolve(void *p) } #ifdef NOD_ENABLED if (g_nodEnabled) { - pbMessage->setNOD(false); + pbMessage->setNewlyObservedDomain(false); pbMessage->clearUDR(); if (nod) pbMessage->removePolicyTag(g_nod_pbtag); @@ -1932,7 +1955,7 @@ static void startDoResolve(void *p) pw.getHeader()->rcode == RCode::ServFail ? SyncRes::s_packetcacheservfailttl : min(minTTL,SyncRes::s_packetcachettl), dq.validationState, - std::move(pbMessage)); + pbMessage.get(), pbUnmutablePart); } // else cerr<<"Not putting in packet cache: "< pbMessage(boost::none); + string *pbString = nullptr; #ifdef HAVE_PROTOBUF if (t_protobufServers) { - pbMessage = RecProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType::Response); - pbMessage->setServerIdentity(SyncRes::s_serverID); if (logQuery && !(luaconfsLocal->protobufExportConfig.taggedOnly && policyTags.empty())) { protobufLogQuery(luaconfsLocal->protobufMaskV4, luaconfsLocal->protobufMaskV6, uniqueId, source, destination, ednssubnet.source, false, dh->id, question.size(), qname, qtype, qclass, policyTags, requestorId, deviceId, deviceName); } @@ -2685,10 +2706,10 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr as cacheable we would cache it with a wrong tag, so better safe than sorry. */ vState valState; if (qnameParsed) { - cacheHit = (!SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, qname, qtype, qclass, g_now.tv_sec, &response, &age, &valState, &qhash, pbMessage ? &(*pbMessage) : nullptr)); + cacheHit = !SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, qname, qtype, qclass, g_now.tv_sec, &response, &age, &valState, &qhash, &pbString); } else { - cacheHit = (!SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, qname, &qtype, &qclass, g_now.tv_sec, &response, &age, &valState, &qhash, pbMessage ? &(*pbMessage) : nullptr)); + cacheHit = !SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, qname, &qtype, &qclass, g_now.tv_sec, &response, &age, &valState, &qhash, &pbString); } if (cacheHit) { @@ -2700,21 +2721,41 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr } #ifdef HAVE_PROTOBUF - if(t_protobufServers && logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbMessage->getAppliedPolicy().empty() && pbMessage->getPolicyTags().empty())) { + if(t_protobufServers && logResponse /* && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbMessage->appliedPolicyIsSet() && pbMessage->policyTagsAreSet()) */) { + std::unique_ptr pbMessage; + if (pbString != nullptr) { + // We take the inmutable string form the cache an are appending a few values + pbMessage = make_unique(*pbString, 128); // The extra bytes we are going to add + } else { + pbMessage = make_unique(128); + pbMessage->setType(2); // Response + pbMessage->setServerIdentity(SyncRes::s_serverID); + } + //cerr << "from cache: " << pbMessage->getbuf().length() << endl; Netmask requestorNM(source, source.sin4.sin_family == AF_INET ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); ComboAddress requestor = requestorNM.getMaskedNetwork(); - requestor.setPort(source.getPort()); - pbMessage->update(uniqueId, &requestor, &destination, false, dh->id); - pbMessage->setEDNSSubnet(ednssubnet.source, ednssubnet.source.isIPv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); + pbMessage->setMessageIdentity(uniqueId); + pbMessage->setFrom(requestor); + pbMessage->setTo(destination); + pbMessage->setSocketProtocol(false); + pbMessage->setId(dh->id); + // Exact semantics of the various timestamps are unclear to me ATM + pbMessage->setTime(); + pbMessage->startResponse(); if (g_useKernelTimestamp && tv.tv_sec) { pbMessage->setQueryTime(tv.tv_sec, tv.tv_usec); } else { pbMessage->setQueryTime(g_now.tv_sec, g_now.tv_usec); } + pbMessage->finishResponse(); + pbMessage->setEDNSSubnet(ednssubnet.source, ednssubnet.source.isIPv4() ? luaconfsLocal->protobufMaskV4 : luaconfsLocal->protobufMaskV6); pbMessage->setRequestorId(requestorId); pbMessage->setDeviceId(deviceId); pbMessage->setDeviceName(deviceName); + pbMessage->setFromPort(source.getPort()); + pbMessage->setToPort(destination.getPort()); + //cerr << "affter adding: " << pbMessage->getbuf().length() << endl; protobufLogResponse(*pbMessage); } #endif /* HAVE_PROTOBUF */ diff --git a/pdns/protozero.cc b/pdns/protozero.cc new file mode 100644 index 0000000000..2be7f8736c --- /dev/null +++ b/pdns/protozero.cc @@ -0,0 +1,167 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include "protozero.hh" +#include "dnsrecords.hh" + + +void pdns::ProtoZero::Message::encodeComboAddress(const protozero::pbf_tag_type type, const ComboAddress& ca) +{ + if (ca.sin4.sin_family == AF_INET) { + d_pbf.add_bytes(type, reinterpret_cast(&ca.sin4.sin_addr.s_addr), sizeof(ca.sin4.sin_addr.s_addr)); + } + else if (ca.sin4.sin_family == AF_INET6) { + d_pbf.add_bytes(type, reinterpret_cast(&ca.sin6.sin6_addr.s6_addr), sizeof(ca.sin6.sin6_addr.s6_addr)); + } +} + +void pdns::ProtoZero::Message::encodeNetmask(const protozero::pbf_tag_type type, const Netmask& subnet, uint8_t mask) +{ + if (!subnet.empty()) { + ComboAddress ca(subnet.getNetwork()); + ca.truncate(mask); + if (ca.sin4.sin_family == AF_INET) { + d_pbf.add_bytes(type, reinterpret_cast(&ca.sin4.sin_addr.s_addr), sizeof(ca.sin4.sin_addr.s_addr)); + } + else if (ca.sin4.sin_family == AF_INET6) { + d_pbf.add_bytes(type, reinterpret_cast(&ca.sin6.sin6_addr.s6_addr), sizeof(ca.sin6.sin6_addr.s6_addr)); + } + } +} + +void pdns::ProtoZero::Message::encodeDNSName(protozero::pbf_writer& pbf, std::string& buffer, const protozero::pbf_tag_type type, const DNSName& name) +{ + // this will append the tag, mark the current position then reserve enough place to write the size + protozero::pbf_writer pbf_name{pbf, type}; + // we append the name to the buffer + name.toString(buffer); + // leaving the block will cause the sub writer to compute how much was written based on the new size and update the size accordingly +} + +void pdns::ProtoZero::Message::request(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, bool tcp, size_t len) +{ + setType(1); + setMessageIdentity(uniqueId); + setSocketFamily(requestor.sin4.sin_family); + setSocketProtocol(tcp); + setFrom(requestor); + setTo(local); + setInBytes(len); + setTime(); + setId(id); + setQuestion(qname, qtype, qclass); + setFromPort(requestor.getPort()); + setToPort(local.getPort()); +} + +void pdns::ProtoZero::Message::response(const DNSName& qname, uint16_t qtype, uint16_t qclass) +{ + setType(2); + setQuestion(qname, qtype, qclass); +} + + +#ifdef NOD_ENABLED +void pdns::ProtoZero::Message::addRR(const DNSRecord& record, const std::set& exportTypes, bool udr) +#else +void pdns::ProtoZero::Message::addRR(const DNSRecord& record, const std::set& exportTypes) +#endif /* NOD_ENABLED */ +{ + if (record.d_place != DNSResourceRecord::ANSWER || record.d_class != QClass::IN) { + return; + } + + if (exportTypes.count(record.d_type) == 0) { + return; + } + + protozero::pbf_writer pbf_rr{*d_response, 2}; + + encodeDNSName(pbf_rr, d_buffer, 1, record.d_name); + pbf_rr.add_uint32(2, record.d_type); + pbf_rr.add_uint32(3, record.d_class); + pbf_rr.add_uint32(4, record.d_ttl); + + switch(record.d_type) { + case QType::A: + { + const auto& content = dynamic_cast(*(record.d_content)); + ComboAddress data = content.getCA(); + pbf_rr.add_bytes(5, reinterpret_cast(&data.sin4.sin_addr.s_addr), sizeof(data.sin4.sin_addr.s_addr)); + break; + } + case QType::AAAA: + { + const auto& content = dynamic_cast(*(record.d_content)); + ComboAddress data = content.getCA(); + pbf_rr.add_bytes(5, reinterpret_cast(&data.sin6.sin6_addr.s6_addr), sizeof(data.sin6.sin6_addr.s6_addr)); + break; + } + case QType::CNAME: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.getTarget().toString()); + break; + } + case QType::TXT: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.d_text); + break; + } + case QType::NS: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.getNS().toString()); + break; + } + case QType::PTR: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.getContent().toString()); + break; + } + case QType::MX: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.d_mxname.toString()); + break; + } + case QType::SPF: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.getText()); + break; + } + case QType::SRV: + { + const auto& content = dynamic_cast(*(record.d_content)); + pbf_rr.add_string(5, content.d_target.toString()); + break; + } + default: + break; + } +#ifdef NOD_ENABLED + pbf_rr.add_bool(6, udr); +#endif +} diff --git a/pdns/protozero.hh b/pdns/protozero.hh new file mode 100644 index 0000000000..81681acbd2 --- /dev/null +++ b/pdns/protozero.hh @@ -0,0 +1,252 @@ +/* + * This file is part of PowerDNS or dnsdist. + * Copyright -- PowerDNS.COM B.V. and its contributors + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * In addition, for the avoidance of any doubt, permission is granted to + * link this program with OpenSSL and to (re)distribute the binaries + * produced as the result of such linking. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ +#pragma once + +#include +#include + +#include "config.h" +#include "iputils.hh" +#include "filterpo.hh" +#include "gettime.hh" +#include "uuid-utils.hh" + +namespace pdns { + namespace ProtoZero { + class Message { + public: + Message(size_t sz) : d_pbf{d_buffer} + { + d_buffer.reserve(sz); + } + Message(const std::string& buf, size_t sz) : d_buffer(buf), d_pbf(d_buffer) + { + // We expect to grow the buffwer + d_buffer.reserve(d_buffer.capacity() + sz); + } + const std::string& getbuf() const + { + return d_buffer; + } + std::string&& movebuf() + { + return std::move(d_buffer); + } + void encodeComboAddress(const protozero::pbf_tag_type type, const ComboAddress& ca); + void encodeNetmask(const protozero::pbf_tag_type type, const Netmask& subnet, uint8_t mask); + void encodeDNSName(protozero::pbf_writer& pbf, std::string& buffer, const protozero::pbf_tag_type type, const DNSName& name); + void request(const boost::uuids::uuid& uniqueId, const ComboAddress& requestor, const ComboAddress& local, const DNSName& qname, uint16_t qtype, uint16_t qclass, uint16_t id, bool tcp, size_t len); + void response(const DNSName& qname, uint16_t qtype, uint16_t qclass); + + void setType(int mtype) + { + d_pbf.add_enum(1, mtype); + } + void setMessageIdentity(const boost::uuids::uuid& uniqueId) + { + d_pbf.add_bytes(2, reinterpret_cast(uniqueId.begin()), uniqueId.size()); + } + void setServerIdentity(const std::string& serverIdentity) + { + d_pbf.add_bytes(3, serverIdentity.data(), serverIdentity.length()); + } + void setSocketFamily(int family) + { + d_pbf.add_enum(4, family == AF_INET ? 1 : 2); + } + void setSocketProtocol(bool tcp) + { + d_pbf.add_enum(5, tcp ? 2 : 1); + } + void setFrom(const ComboAddress& ca) + { + encodeComboAddress(6, ca); + } + void setTo(const ComboAddress& ca) + { + encodeComboAddress(7, ca); + } + void setInBytes(uint64_t len) + { + if (len) { + d_pbf.add_uint64(8, len); + } + } + void setTime() + { + struct timespec ts; + gettime(&ts, true); + // timeSec + d_pbf.add_uint32(9, ts.tv_sec); + // timeUsec + d_pbf.add_uint32(10, ts.tv_nsec / 1000); + } + void setId(uint16_t id) + { + d_pbf.add_uint32(11, ntohs(id)); + } + void setQuestion(const DNSName& qname, uint16_t qtype, uint16_t qclass) + { + protozero::pbf_writer pbf_question{d_pbf, 12}; + encodeDNSName(pbf_question, d_buffer, 1, qname); + pbf_question.add_uint32(2, qtype); + pbf_question.add_uint32(3, qclass); + } + void setEDNSSubnet(const Netmask& nm, uint8_t mask) + { + encodeNetmask(14, nm, mask); + } + void setRequestorId(const std::string& req) + { + if (true || !req.empty()) { + d_pbf.add_string(15, req); + } + } + void setInitialRequesId(const std::string& id) + { + if (!id.empty()) { + d_pbf.add_string(16, id); + } + } + void setDeviceId(const std::string& id) + { + if (true || !id.empty()) { + d_pbf.add_string(17, id); + } + } + void setNewlyObservedDomain(bool nod) + { + d_pbf.add_bool(18, nod); + } + void setDeviceName(const std::string& name) + { + if (true || !name.empty()) { + d_pbf.add_string(19, name); + } + } + void setFromPort(in_port_t port) + { + d_pbf.add_uint32(20, port); + } + void setToPort(in_port_t port) + { + d_pbf.add_uint32(21, port); + } + + // DNSResponse related fields below + void startResponse() + { + if (d_response != nullptr) { + throw new runtime_error("response already inited"); + } + d_response = new protozero::pbf_writer(d_pbf, 13); + } + void finishResponse() + { + delete d_response; + d_response = nullptr; + } + void setResponseCode(uint8_t rcode) + { + d_response->add_uint32(1, rcode); + } +#ifdef NOD_ENABLED + void addRR(const DNSRecord& record, const std::set& exportTypes, bool udr); +#else + void addRR(const DNSRecord& record, const std::set& exportTypes); +#endif + void clearUDR() + { + } + void setAppliedPolicy(const std::string& policy) + { + d_response->add_string(3, policy); + } + bool appliedPolicyIsSet() const + { + return false; + } + void setPolicyTags(const std::unordered_set& tags) + { + for (const auto& tag : tags) { + d_response->add_string(4, tag); + } + } + void addPolicyTag(const string& tag) + { + } + void removePolicyTag(const string& tag) + { + } + bool policyTagsAreSet() const + { + return false; + } + void setQueryTime(uint32_t sec, uint32_t usec) + { + d_response->add_uint32(5, sec); + d_response->add_uint32(6, usec); + } + void setAppliedPolicyType(const DNSFilterEngine::PolicyType type) + { + uint32_t p; + + switch(type) { + case DNSFilterEngine::PolicyType::None: + p = 1; + break; + case DNSFilterEngine::PolicyType::QName: + p = 2; + break; + case DNSFilterEngine::PolicyType::ClientIP: + p = 3; + break; + case DNSFilterEngine::PolicyType::ResponseIP: + p = 4; + break; + case DNSFilterEngine::PolicyType::NSDName: + p = 5; + break; + case DNSFilterEngine::PolicyType::NSIP: + p = 6; + break; + default: + throw std::runtime_error("Unsupported protobuf policy type"); + } + d_response->add_uint32(7, p); + } + void setAppliedPolicyTrigger(const DNSName& trigger) + { + encodeDNSName(*d_response, d_buffer, 8, trigger); + } + void setAppliedPolicyHit(const std::string& hit) + { + d_response->add_string(9, hit); + } + + private: + std::string d_buffer; + protozero::pbf_writer d_pbf; + protozero::pbf_writer *d_response{nullptr}; + }; + }; +}; diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc index 0f93c5d317..03758d1392 100644 --- a/pdns/recpacketcache.cc +++ b/pdns/recpacketcache.cc @@ -50,7 +50,7 @@ bool RecursorPacketCache::qrMatch(const packetCache_t::index::type::ite return queryMatches(iter->d_query, queryPacket, qname, optionsToSkip); } -bool RecursorPacketCache::checkResponseMatches(std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, RecProtoBufMessage* protobufMessage) +bool RecursorPacketCache::checkResponseMatches(std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, std::string** protobufMessage) { for(auto iter = range.first ; iter != range.second ; ++iter) { // the possibility is VERY real that we get hits that are not right - birthday paradox @@ -74,10 +74,10 @@ bool RecursorPacketCache::checkResponseMatches(std::paird_protobufMessage) { - protobufMessage->copyFrom(*(iter->d_protobufMessage)); + *protobufMessage = &*iter->d_protobufMessage; } else { - *protobufMessage = RecProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType::Response); + *protobufMessage = nullptr; } } #endif @@ -110,7 +110,7 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, - std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, RecProtoBufMessage* protobufMessage) + std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, std::string** protobufMessage) { *qhash = canHashPacket(queryPacket, true); const auto& idx = d_packetCache.get(); @@ -125,7 +125,7 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t* qtype, uint16_t* qclass, time_t now, - std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, RecProtoBufMessage* protobufMessage) + std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, std::string** protobufMessage) { *qhash = canHashPacket(queryPacket, true); const auto& idx = d_packetCache.get(); @@ -142,7 +142,7 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } -void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, boost::optional&& protobufMessage) +void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, pdns::ProtoZero::Message* protobufMessage, std::string::size_type pbUnmutablePart) { auto& idx = d_packetCache.get(); auto range = idx.equal_range(tie(tag,qhash)); @@ -161,7 +161,10 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, iter->d_vstate = valState; #ifdef HAVE_PROTOBUF if (protobufMessage) { - iter->d_protobufMessage = std::move(*protobufMessage); + std::string s(protobufMessage->movebuf()); + s.resize(pbUnmutablePart); + s.shrink_to_fit(); + iter->d_protobufMessage = boost::optional(s); } #endif @@ -179,7 +182,10 @@ void RecursorPacketCache::insertResponsePacket(unsigned int tag, uint32_t qhash, e.d_vstate = valState; #ifdef HAVE_PROTOBUF if (protobufMessage) { - e.d_protobufMessage = std::move(*protobufMessage); + std::string s(protobufMessage->movebuf()); + s.resize(pbUnmutablePart); + s.shrink_to_fit(); + e.d_protobufMessage = boost::optional(s); } #endif d_packetCache.insert(e); @@ -219,7 +225,7 @@ uint64_t RecursorPacketCache::doDump(int fd) for(auto i=sidx.cbegin(); i != sidx.cend(); ++i) { count++; try { - fprintf(fp.get(), "%s %" PRId64 " %s ; tag %d\n", i->d_name.toString().c_str(), static_cast(i->d_ttd - now), DNSRecordContent::NumberToType(i->d_type).c_str(), i->d_tag); + fprintf(fp.get(), "%s %" PRId64 " %s ; tag %d pb %zu\n", i->d_name.toString().c_str(), static_cast(i->d_ttd - now), DNSRecordContent::NumberToType(i->d_type).c_str(), i->d_tag, i->d_protobufMessage ? i->d_protobufMessage->length() : 0); } catch(...) { fprintf(fp.get(), "; error printing '%s'\n", i->d_name.empty() ? "EMPTY" : i->d_name.toString().c_str()); diff --git a/pdns/recpacketcache.hh b/pdns/recpacketcache.hh index 8ffc61ba83..7faf793fbe 100644 --- a/pdns/recpacketcache.hh +++ b/pdns/recpacketcache.hh @@ -33,6 +33,7 @@ #include "packetcache.hh" #include "validate.hh" +#include "protozero.hh" #ifdef HAVE_CONFIG_H #include "config.h" @@ -53,9 +54,10 @@ public: RecursorPacketCache(); bool getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age, uint32_t* qhash); bool getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, uint32_t* qhash); - bool getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, RecProtoBufMessage* protobufMessage); - bool getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t* qtype, uint16_t* qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, RecProtoBufMessage* protobufMessage); - void insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, boost::optional&& protobufMessage); + bool getResponsePacket(unsigned int tag, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, std::string** protobufMessage); + bool getResponsePacket(unsigned int tag, const std::string& queryPacket, DNSName& qname, uint16_t *qtype, uint16_t* qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, uint32_t* qhash, std::string** protobufMessage); + + void insertResponsePacket(unsigned int tag, uint32_t qhash, std::string&& query, const DNSName& qname, uint16_t qtype, uint16_t qclass, std::string&& responsePacket, time_t now, uint32_t ttl, const vState& valState, pdns::ProtoZero::Message* protobufMessage = nullptr, std::string::size_type pbUnmutablePart = 0); void doPruneTo(size_t maxSize=250000); uint64_t doDump(int fd); int doWipePacketCache(const DNSName& name, uint16_t qtype=0xffff, bool subtree=false); @@ -78,7 +80,7 @@ private: mutable std::string d_packet; // "I know what I am doing" mutable std::string d_query; #ifdef HAVE_PROTOBUF - mutable boost::optional d_protobufMessage; + mutable boost::optional d_protobufMessage; #endif mutable time_t d_ttd; mutable time_t d_creation; // so we can 'age' our packets @@ -108,7 +110,7 @@ private: packetCache_t d_packetCache; static bool qrMatch(const packetCache_t::index::type::iterator& iter, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass); - bool checkResponseMatches(std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, RecProtoBufMessage* protobufMessage); + bool checkResponseMatches(std::pair::type::iterator, packetCache_t::index::type::iterator> range, const std::string& queryPacket, const DNSName& qname, uint16_t qtype, uint16_t qclass, time_t now, std::string* responsePacket, uint32_t* age, vState* valState, std::string** protobufMessage); public: void preRemoval(const Entry& entry) diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index d32e3c0b2a..52c6c81eed 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -5,6 +5,7 @@ AM_CPPFLAGS = $(LUA_CFLAGS) $(YAHTTP_CFLAGS) $(BOOST_CPPFLAGS) $(LIBSODIUM_CFLAG AM_CPPFLAGS += \ -I$(top_srcdir)/ext/json11 \ + -I$(top_srcdir)/ext/protozero/include \ $(YAHTTP_CFLAGS) \ $(LIBCRYPTO_INCLUDES) @@ -67,6 +68,7 @@ EXTRA_DIST = \ opensslsigners.hh opensslsigners.cc \ portsmplexer.cc \ dnstap.proto dnstap.cc dnstap.hh fstrm_logger.cc fstrm_logger.hh rec-dnstap.hh \ + ext/protozero/include/* \ rrd/* \ html incfiles \ test_libcrypto \ @@ -190,7 +192,8 @@ pdns_recursor_SOURCES = \ ws-api.cc ws-api.hh \ ws-recursor.cc ws-recursor.hh \ xpf.cc xpf.hh \ - zoneparser-tng.cc zoneparser-tng.hh + zoneparser-tng.cc zoneparser-tng.hh \ + protozero.cc protozero.hh if !HAVE_LUA_HPP BUILT_SOURCES += lua.hpp diff --git a/pdns/recursordist/ext/protozero b/pdns/recursordist/ext/protozero new file mode 120000 index 0000000000..999735006e --- /dev/null +++ b/pdns/recursordist/ext/protozero @@ -0,0 +1 @@ +../../../ext/protozero \ No newline at end of file diff --git a/pdns/recursordist/protozero.cc b/pdns/recursordist/protozero.cc new file mode 120000 index 0000000000..b2f57f3c7d --- /dev/null +++ b/pdns/recursordist/protozero.cc @@ -0,0 +1 @@ +../protozero.cc \ No newline at end of file diff --git a/pdns/recursordist/protozero.hh b/pdns/recursordist/protozero.hh new file mode 120000 index 0000000000..edbde5a244 --- /dev/null +++ b/pdns/recursordist/protozero.hh @@ -0,0 +1 @@ +../protozero.hh \ No newline at end of file diff --git a/pdns/test-recpacketcache_cc.cc b/pdns/test-recpacketcache_cc.cc index fce6fcc823..8a4f309a2b 100644 --- a/pdns/test-recpacketcache_cc.cc +++ b/pdns/test-recpacketcache_cc.cc @@ -45,16 +45,16 @@ BOOST_AUTO_TEST_CASE(test_recPacketCacheSimple) { pw.commit(); string rpacket((const char*)&packet[0], packet.size()); - rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); rpc.doPruneTo(0); BOOST_CHECK_EQUAL(rpc.size(), 0U); - rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); rpc.doWipePacketCache(qname); BOOST_CHECK_EQUAL(rpc.size(), 0U); - rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag, qhash, string(qpacket), qname, QType::A, QClass::IN, string(rpacket), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); uint32_t qhash2 = 0; bool found = rpc.getResponsePacket(tag, qpacket, time(nullptr), &fpacket, &age, &qhash2); @@ -140,11 +140,11 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { BOOST_CHECK(r1packet != r2packet); /* inserting a response for tag1 */ - rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); /* inserting a different response for tag2, should not override the first one */ - rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 2U); /* remove all responses from the cache */ @@ -152,10 +152,10 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { BOOST_CHECK_EQUAL(rpc.size(), 0U); /* reinsert both */ - rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); - rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 2U); /* remove the responses by qname, should remove both */ @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { BOOST_CHECK_EQUAL(rpc.size(), 0U); /* insert the response for tag1 */ - rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag1, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r1packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 1U); /* we can retrieve it */ @@ -182,7 +182,7 @@ BOOST_AUTO_TEST_CASE(test_recPacketCache_Tags) { BOOST_CHECK_EQUAL(temphash, qhash); /* adding a response for the second tag */ - rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, boost::none); + rpc.insertResponsePacket(tag2, qhash, string(qpacket), qname, QType::A, QClass::IN, string(r2packet), time(0), ttd, vState::Indeterminate, nullptr); BOOST_CHECK_EQUAL(rpc.size(), 2U); /* We still get the correct response for the first tag */