From: Remi Gacogne Date: Fri, 24 Jun 2016 08:56:04 +0000 (+0200) Subject: Protobuf refactoring X-Git-Tag: rec-4.0.0~17^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d9d3f9c13c51c4df5a7863056cae7afd330f374c;p=thirdparty%2Fpdns.git Protobuf refactoring The main idea here is to have a object encapsulating the protobuf stuff, that exists but does nothing when protobuf support is off. This makes it a lot easier and cleanier for the rest of the code. This also: * unifies the protobuf code used in dnsdist, rec and dnspcap2protobuf * add CNAME values to responses --- diff --git a/contrib/ProtobufLogger.py b/contrib/ProtobufLogger.py index 848dd63fb9..eaddd5cc56 100644 --- a/contrib/ProtobufLogger.py +++ b/contrib/ProtobufLogger.py @@ -94,6 +94,8 @@ class PDNSPBConnHandler(object): if (rrclass == 1 or rrclass == 255) and rr.HasField('rdata'): if rrtype == 1: rdatastr = socket.inet_ntop(socket.AF_INET, rr.rdata) + elif rrtype == 5: + rdatastr = rr.rdata elif rrtype == 28: rdatastr = socket.inet_ntop(socket.AF_INET6, rr.rdata) diff --git a/pdns/Makefile.am b/pdns/Makefile.am index 07024b2815..cd3112f011 100644 --- a/pdns/Makefile.am +++ b/pdns/Makefile.am @@ -1064,9 +1064,11 @@ dnspcap2protobuf_SOURCES = \ dnspcap2protobuf.cc \ dnsrecords.cc \ dnswriter.cc dnswriter.hh \ + gettime.cc gettime.hh \ logger.cc \ misc.cc \ nsecrecords.cc \ + protobuf.cc protobuf.hh \ qtype.cc \ rcpgenerator.cc rcpgenerator.hh \ sillyrecords.cc \ diff --git a/pdns/dnsdist-protobuf.cc b/pdns/dnsdist-protobuf.cc index 93bab7f3ff..12b9463f3a 100644 --- a/pdns/dnsdist-protobuf.cc +++ b/pdns/dnsdist-protobuf.cc @@ -10,124 +10,15 @@ #ifdef HAVE_PROTOBUF #include "dnsmessage.pb.h" -static void protobufFillMessage(PBDNSMessage& message, const DNSQuestion& dq) +DNSDistProtoBufMessage::DNSDistProtoBufMessage(DNSProtoBufMessageType type, const DNSQuestion& dq): DNSProtoBufMessage(type, dq.uniqueId, dq.remote, dq.local, *dq.qname, dq.qtype, dq.qclass, dq.dh->id, dq.tcp, dq.len) { - std::string* messageId = message.mutable_messageid(); - messageId->resize(dq.uniqueId.size()); - std::copy(dq.uniqueId.begin(), dq.uniqueId.end(), messageId->begin()); - - message.set_socketfamily(dq.remote->sin4.sin_family == AF_INET ? PBDNSMessage_SocketFamily_INET : PBDNSMessage_SocketFamily_INET6); - message.set_socketprotocol(dq.tcp ? PBDNSMessage_SocketProtocol_TCP : PBDNSMessage_SocketProtocol_UDP); - if (dq.local->sin4.sin_family == AF_INET) { - message.set_to(&dq.local->sin4.sin_addr.s_addr, sizeof(dq.local->sin4.sin_addr.s_addr)); - } - else if (dq.local->sin4.sin_family == AF_INET6) { - message.set_to(&dq.local->sin6.sin6_addr.s6_addr, sizeof(dq.local->sin6.sin6_addr.s6_addr)); - } - if (dq.remote->sin4.sin_family == AF_INET) { - message.set_from(&dq.remote->sin4.sin_addr.s_addr, sizeof(dq.remote->sin4.sin_addr.s_addr)); - } - else if (dq.remote->sin4.sin_family == AF_INET6) { - message.set_from(&dq.remote->sin6.sin6_addr.s6_addr, sizeof(dq.remote->sin6.sin6_addr.s6_addr)); - } - - message.set_inbytes(dq.len); - - struct timespec ts; - gettime(&ts, true); - message.set_timesec(ts.tv_sec); - message.set_timeusec(ts.tv_nsec / 1000); - message.set_id(ntohs(dq.dh->id)); - - PBDNSMessage_DNSQuestion* question = message.mutable_question(); - question->set_qname(dq.qname->toString()); - question->set_qtype(dq.qtype); - question->set_qclass(dq.qclass); -} - -void protobufMessageFromQuestion(const DNSQuestion& dq, std::string& data) -{ - PBDNSMessage message; - message.set_type(PBDNSMessage_Type_DNSQueryType); - protobufFillMessage(message, dq); -// cerr <ancount) == 0) - return; - - if (ntohs(dh->qdcount) == 0) - return; - - vector content(len - sizeof(dnsheader)); - copy(packet + sizeof(dnsheader), packet + len, content.begin()); - PacketReader pr(content); - - size_t idx = 0; - DNSName rrname; - uint16_t qdcount = ntohs(dh->qdcount); - uint16_t ancount = ntohs(dh->ancount); - uint16_t rrtype; - uint16_t rrclass; - string blob; - struct dnsrecordheader ah; - - rrname = pr.getName(); - rrtype = pr.get16BitInt(); - rrclass = pr.get16BitInt(); - - /* consume remaining qd if any */ - if (qdcount > 1) { - for(idx = 1; idx < qdcount; idx++) { - rrname = pr.getName(); - rrtype = pr.get16BitInt(); - rrclass = pr.get16BitInt(); - (void) rrtype; - (void) rrclass; + if (type == Response) { + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (response) { + response->set_rcode(dq.dh->rcode); } + addRRsFromPacket((const char*) dq.dh, dq.len); } +}; - /* parse AN */ - for (idx = 0; idx < ancount; idx++) { - rrname = pr.getName(); - pr.getDnsrecordheader(ah); - - pr.xfrBlob(blob); - - if (ah.d_type == QType::A || ah.d_type == QType::AAAA) { - PBDNSMessage_DNSResponse_DNSRR* rr = response.add_rrs(); - if (rr) { - rr->set_name(rrname.toString()); - rr->set_type(ah.d_type); - rr->set_class_(ah.d_class); - rr->set_ttl(ah.d_ttl); - rr->set_rdata(blob.c_str(), blob.length()); - } - } - } -} - -void protobufMessageFromResponse(const DNSQuestion& dr, std::string& data) -{ - PBDNSMessage message; - message.set_type(PBDNSMessage_Type_DNSResponseType); - protobufFillMessage(message, dr); - - PBDNSMessage_DNSResponse response; - response.set_rcode(dr.dh->rcode); - addRRs((const char*) dr.dh, dr.len, response); - message.set_allocated_response(&response); - -// cerr < #include -#include "dnsmessage.pb.h" #include "iputils.hh" #include "misc.hh" #include "dns.hh" #include "dnspcap.hh" #include "dnsparser.hh" +#include "protobuf.hh" #include "statbag.hh" StatBag S; -static void addRRs(const char* packet, const size_t len, PBDNSMessage_DNSResponse* response) -try -{ - if (len < sizeof(struct dnsheader)) - return; - - const struct dnsheader* dh = (const struct dnsheader*) packet; - - if (ntohs(dh->ancount) == 0) - return; - - if (ntohs(dh->qdcount) == 0) - return; - - vector content(len - sizeof(dnsheader)); - copy(packet + sizeof(dnsheader), packet + len, content.begin()); - PacketReader pr(content); - - size_t idx = 0; - DNSName rrname; - uint16_t qdcount = ntohs(dh->qdcount); - uint16_t ancount = ntohs(dh->ancount); - uint16_t rrtype; - uint16_t rrclass; - string blob; - struct dnsrecordheader ah; - - rrname = pr.getName(); - rrtype = pr.get16BitInt(); - rrclass = pr.get16BitInt(); - - /* consume remaining qd if any */ - if (qdcount > 1) { - for(idx = 1; idx < qdcount; idx++) { - rrname = pr.getName(); - rrtype = pr.get16BitInt(); - rrclass = pr.get16BitInt(); - (void) rrtype; - (void) rrclass; - } - } - - /* parse AN */ - for (idx = 0; idx < ancount; idx++) { - rrname = pr.getName(); - pr.getDnsrecordheader(ah); - - pr.xfrBlob(blob); - if (ah.d_type == QType::A || ah.d_type == QType::AAAA) { - PBDNSMessage_DNSResponse_DNSRR* rr = response->add_rrs(); - if (rr) { - rr->set_name(rrname.toString()); - rr->set_type(ah.d_type); - rr->set_class_(ah.d_class); - rr->set_ttl(ah.d_ttl); - rr->set_rdata(blob.c_str(), blob.length()); - } - } - } -} -catch(const std::exception& e) -{ - cerr<<"Error parsing response records: "<id)); - message.set_type(dh->qr ? PBDNSMessage_Type_DNSResponseType : PBDNSMessage_Type_DNSQueryType); - const ComboAddress requestor = dh->qr ? pr.getDest() : pr.getSource(); - const ComboAddress responder = dh->qr ? pr.getSource() : pr.getDest(); - - *((char*)&requestor.sin4.sin_addr.s_addr)|=ind; - *((char*)&responder.sin4.sin_addr.s_addr)|=ind; - message.set_socketfamily(requestor.sin4.sin_family == AF_INET ? PBDNSMessage_SocketFamily_INET : PBDNSMessage_SocketFamily_INET6); - // we handle UDP packets only for now - message.set_socketprotocol(PBDNSMessage_SocketProtocol_UDP); - if (requestor.sin4.sin_family == AF_INET) { - message.set_from(&requestor.sin4.sin_addr.s_addr, sizeof(requestor.sin4.sin_addr.s_addr)); - } - else if (requestor.sin4.sin_family == AF_INET6) { - message.set_from(&requestor.sin6.sin6_addr.s6_addr, sizeof(requestor.sin6.sin6_addr.s6_addr)); - } - if (responder.sin4.sin_family == AF_INET) { - message.set_to(&responder.sin4.sin_addr.s_addr, sizeof(responder.sin4.sin_addr.s_addr)); - } - else if (responder.sin4.sin_family == AF_INET6) { - message.set_to(&responder.sin6.sin6_addr.s6_addr, sizeof(responder.sin6.sin6_addr.s6_addr)); - } - message.set_inbytes(pr.d_len); - - PBDNSMessage_DNSQuestion* question = message.mutable_question(); - PBDNSMessage_DNSResponse* response = message.mutable_response(); - + boost::uuids::uuid uniqueId; if (!dh->qr) { - boost::uuids::uuid uniqueId = uuidGenerator(); + uniqueId = uuidGenerator(); ids[dh->id] = uniqueId; - std::string* messageId = message.mutable_messageid(); - messageId->resize(uniqueId.size()); - std::copy(uniqueId.begin(), uniqueId.end(), messageId->begin()); } else { const auto& it = ids.find(dh->id); if (it != ids.end()) { - std::string* messageId = message.mutable_messageid(); - messageId->resize(it->second.size()); - std::copy(it->second.begin(), it->second.end(), messageId->begin()); + uniqueId = it->second; + } + else { + uniqueId = uuidGenerator(); } - - response->set_rcode(dh->rcode); - addRRs((const char*) dh, pr.d_len, response); } - question->set_qname(qname.toString()); - question->set_qtype(qtype); - question->set_qclass(qclass); + const ComboAddress requestor = dh->qr ? pr.getDest() : pr.getSource(); + const ComboAddress responder = dh->qr ? pr.getSource() : pr.getDest(); + *((char*)&requestor.sin4.sin_addr.s_addr)|=ind; + *((char*)&responder.sin4.sin_addr.s_addr)|=ind; + + DNSProtoBufMessage message(dh->qr ? DNSProtoBufMessage::DNSProtoBufMessageType::Response : DNSProtoBufMessage::DNSProtoBufMessageType::Query, uniqueId, &requestor, &responder, qname, qtype, qclass, dh->id, false, pr.d_len); + message.setTime(pr.d_pheader.ts.tv_sec, pr.d_pheader.ts.tv_usec); + + if (dh->qr) { + message.setResponseCode(dh->rcode); + try { + message.addRRsFromPacket((const char*) dh, pr.d_len); + } + catch(std::exception& e) + { + cerr<<"Error parsing response records: "<queueData(data); #endif /* HAVE_PROTOBUF */ return Action::None; @@ -936,8 +937,9 @@ public: DNSResponseAction::Action operator()(DNSQuestion* dq, string* ruleresult) const override { #ifdef HAVE_PROTOBUF + DNSDistProtoBufMessage message(DNSDistProtoBufMessage::Response, *dq); std::string data; - protobufMessageFromResponse(*dq, data); + message.serialize(data); d_logger->queueData(data); #endif /* HAVE_PROTOBUF */ return Action::None; diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 66b12aaffa..9ebf695c4b 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -83,11 +83,7 @@ extern SortList g_sortlist; #include "ednsoptions.hh" #include "gettime.hh" -#ifdef HAVE_PROTOBUF -#include -#include -#include "dnsmessage.pb.h" -#endif +#include "rec-protobuf.hh" #ifdef HAVE_SYSTEMD #include @@ -614,96 +610,29 @@ catch(...) } #ifdef HAVE_PROTOBUF -static void protobufUpdateMessage(PBDNSMessage& message, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id) -{ - std::string* messageId = message.mutable_messageid(); - messageId->resize(uniqueId.size()); - std::copy(uniqueId.begin(), uniqueId.end(), messageId->begin()); - - message.set_socketfamily(remote.sin4.sin_family == AF_INET ? PBDNSMessage_SocketFamily_INET : PBDNSMessage_SocketFamily_INET6); - message.set_socketprotocol(tcp ? PBDNSMessage_SocketProtocol_TCP : PBDNSMessage_SocketProtocol_UDP); - - if (local.sin4.sin_family == AF_INET) { - message.set_to(&local.sin4.sin_addr.s_addr, sizeof(local.sin4.sin_addr.s_addr)); - } - else if (local.sin4.sin_family == AF_INET6) { - message.set_to(&local.sin6.sin6_addr.s6_addr, sizeof(local.sin6.sin6_addr.s6_addr)); - } - if (remote.sin4.sin_family == AF_INET) { - message.set_from(&remote.sin4.sin_addr.s_addr, sizeof(remote.sin4.sin_addr.s_addr)); - } - else if (remote.sin4.sin_family == AF_INET6) { - message.set_from(&remote.sin6.sin6_addr.s6_addr, sizeof(remote.sin6.sin6_addr.s6_addr)); - } - if (!ednssubnet.empty()) { - const ComboAddress ca = ednssubnet.getNetwork(); - if (ca.sin4.sin_family == AF_INET) { - message.set_originalrequestorsubnet(&ca.sin4.sin_addr.s_addr, sizeof(ca.sin4.sin_addr.s_addr)); - } - else if (ca.sin4.sin_family == AF_INET6) { - message.set_originalrequestorsubnet(&ca.sin6.sin6_addr.s6_addr, sizeof(ca.sin6.sin6_addr.s6_addr)); - } - } - - struct timespec ts; - gettime(&ts, true); - message.set_timesec(ts.tv_sec); - message.set_timeusec(ts.tv_nsec / 1000); - - message.set_id(ntohs(id)); -} - -static void protobufFillMessage(PBDNSMessage& message, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, const DNSName& qname, uint16_t qtype, uint16_t qclass) -{ - protobufUpdateMessage(message, uniqueId, remote, local, ednssubnet, tcp, id); - - PBDNSMessage_DNSQuestion* question = message.mutable_question(); - question->set_qname(qname.toString()); - question->set_qtype(qtype); - question->set_qclass(qclass); -} - -static void protobufFillMessageFromDC(PBDNSMessage& message, const DNSComboWriter* dc) -{ - protobufFillMessage(message, dc->d_uuid, dc->d_remote, dc->d_local, dc->d_ednssubnet, dc->d_tcp, dc->d_mdp.d_header.id, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass); -} - static void protobufLogQuery(const std::shared_ptr& logger, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, size_t len, const DNSName& qname, uint16_t qtype, uint16_t qclass, const std::string appliedPolicy, const std::vector& policyTags) { - PBDNSMessage message; - message.set_type(PBDNSMessage_Type_DNSQueryType); - message.set_inbytes(len); - protobufFillMessage(message, uniqueId, remote, local, ednssubnet, tcp, id, qname, qtype, qclass); + RecProtoBufMessage message(DNSProtoBufMessage::Query, uniqueId, &remote, &local, qname, qtype, qclass, id, tcp, len); + message.setEDNSSubnet(ednssubnet); - /* just fake it up for now */ - PBDNSMessage_DNSResponse* response = message.mutable_response(); if (!appliedPolicy.empty()) { - response->set_appliedpolicy(appliedPolicy); + message.setAppliedPolicy(appliedPolicy); } if (!policyTags.empty()) { - for(const auto tag : policyTags) { - response->add_tags(tag); - } + message.setPolicyTags(policyTags); } -// cerr <queueData(str); } -static void protobufFillResponseFromDC(PBDNSMessage& message, const DNSComboWriter* dc, size_t responseSize) +static void protobufLogResponse(const std::shared_ptr& logger, const RecProtoBufMessage& message) { - message.set_type(PBDNSMessage_Type_DNSResponseType); - message.set_inbytes(responseSize); - protobufFillMessageFromDC(message, dc); -} - -static void protobufLogResponse(const std::shared_ptr& logger, const PBDNSMessage& message) -{ -// cerr <queueData(str); } #endif @@ -728,10 +657,14 @@ void startDoResolve(void *p) auto luaconfsLocal = g_luaconfs.getLocal(); std::string appliedPolicy; + RecProtoBufMessage pbMessage(RecProtoBufMessage::Response); #ifdef HAVE_PROTOBUF - PBDNSMessage pbMessage; - PBDNSMessage_DNSResponse* protobufResponse = pbMessage.mutable_response(); -#endif + if (luaconfsLocal->protobufServer) { + pbMessage.update(dc->d_uuid, &dc->d_remote, &dc->d_local, dc->d_tcp, dc->d_mdp.d_header.id); + pbMessage.setEDNSSubnet(dc->d_ednssubnet); + pbMessage.setQuestion(dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass); + } +#endif /* HAVE_PROTOBUF */ DNSPacketWriter pw(packet, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_mdp.d_qclass); @@ -1021,24 +954,8 @@ void startDoResolve(void *p) goto sendit; // need to jump over pw.commit } #ifdef HAVE_PROTOBUF - if(luaconfsLocal->protobufServer && protobufResponse && (i->d_type == QType::A || i->d_type == QType::AAAA)) { - PBDNSMessage_DNSResponse_DNSRR* pbRR = protobufResponse->add_rrs(); - if(pbRR) { - pbRR->set_name(i->d_name.toString()); - pbRR->set_type(i->d_type); - pbRR->set_class_(i->d_class); - pbRR->set_ttl(i->d_ttl); - if (i->d_type == QType::A) { - const ARecordContent& arc = dynamic_cast(*(i->d_content)); - ComboAddress data = arc.getCA(); - pbRR->set_rdata(&data.sin4.sin_addr.s_addr, sizeof(data.sin4.sin_addr.s_addr)); - } - else if (i->d_type == QType::AAAA) { - const AAAARecordContent& arc = dynamic_cast(*(i->d_content)); - ComboAddress data = arc.getCA(); - pbRR->set_rdata(&data.sin6.sin6_addr.s6_addr, sizeof(data.sin6.sin6_addr.s6_addr)); - } - } + if(luaconfsLocal->protobufServer && (i->d_type == QType::A || i->d_type == QType::AAAA || i->d_type == QType::CNAME)) { + pbMessage.addRR(*i); } #endif } @@ -1051,18 +968,10 @@ void startDoResolve(void *p) updateResponseStats(res, dc->d_remote, packet.size(), &dc->d_mdp.d_qname, dc->d_mdp.d_qtype); #ifdef HAVE_PROTOBUF if (luaconfsLocal->protobufServer) { - if (protobufResponse) { - protobufResponse->set_rcode(pw.getHeader()->rcode); - if (!appliedPolicy.empty()) { - protobufResponse->set_appliedpolicy(appliedPolicy); - } - if (!dc->d_policyTags.empty()) { - for(const auto tag : dc->d_policyTags) { - protobufResponse->add_tags(tag); - } - } - } - protobufFillResponseFromDC(pbMessage, dc, packet.size()); + pbMessage.setBytes(packet.size()); + pbMessage.setResponseCode(pw.getHeader()->rcode); + pbMessage.setAppliedPolicy(appliedPolicy); + pbMessage.setPolicyTags(dc->d_policyTags); protobufLogResponse(luaconfsLocal->protobufServer, pbMessage); } #endif @@ -1078,25 +987,12 @@ void startDoResolve(void *p) if(sendmsg(dc->d_socket, &msgh, 0) < 0 && g_logCommonErrors) L<d_remote.toStringWithPort()<<" failed with: "<protobufServer) { - t_packetCache->insertResponsePacket(dc->d_tag, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_query, - string((const char*)&*packet.begin(), packet.size()), - g_now.tv_sec, - pw.getHeader()->rcode == RCode::ServFail ? SyncRes::s_packetcacheservfailttl : - min(minTTL,SyncRes::s_packetcachettl), - &pbMessage); - } - else { -#endif - t_packetCache->insertResponsePacket(dc->d_tag, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_query, + t_packetCache->insertResponsePacket(dc->d_tag, dc->d_mdp.d_qname, dc->d_mdp.d_qtype, dc->d_query, string((const char*)&*packet.begin(), packet.size()), g_now.tv_sec, pw.getHeader()->rcode == RCode::ServFail ? SyncRes::s_packetcacheservfailttl : - min(minTTL,SyncRes::s_packetcachettl)); -#ifdef HAVE_PROTOBUF - } -#endif + min(minTTL,SyncRes::s_packetcachettl), + &pbMessage); } // else cerr<<"Not putting in packet cache: "< policyTags; #ifdef HAVE_PROTOBUF boost::uuids::uuid uniqueId; - PBDNSMessage pbMessage; auto luaconfsLocal = g_luaconfs.getLocal(); if (luaconfsLocal->protobufServer) { needECS = true; @@ -1459,23 +1354,22 @@ string* doProcessUDPQuestion(const std::string& question, const ComboAddress& fr } bool cacheHit = false; + RecProtoBufMessage pbMessage(DNSProtoBufMessage::DNSProtoBufMessageType::Response); #ifdef HAVE_PROTOBUF if(luaconfsLocal->protobufServer) { protobufLogQuery(luaconfsLocal->protobufServer, uniqueId, fromaddr, destaddr, ednssubnet, false, dh->id, question.size(), qname, qtype, qclass, std::string(), policyTags); + } +#endif /* HAVE_PROTOBUF */ - cacheHit = (!SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, g_now.tv_sec, &response, &age, &pbMessage)); - if (cacheHit) { - protobufUpdateMessage(pbMessage, uniqueId, fromaddr, destaddr, ednssubnet, false, dh->id); + cacheHit = (!SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, g_now.tv_sec, &response, &age, &pbMessage)); + if (cacheHit) { +#ifdef HAVE_PROTOBUF + if(luaconfsLocal->protobufServer) { + pbMessage.update(uniqueId, &fromaddr, &destaddr, false, dh->id); + pbMessage.setEDNSSubnet(ednssubnet); protobufLogResponse(luaconfsLocal->protobufServer, pbMessage); } - } - else { -#endif - cacheHit = (!SyncRes::s_nopacketcache && t_packetCache->getResponsePacket(ctag, question, g_now.tv_sec, &response, &age)); -#ifdef HAVE_PROTOBUF - } -#endif - if (cacheHit) { +#endif /* HAVE_PROTOBUF */ if(!g_quiet) L<d_tcp=false; dc->d_policyTags = policyTags; #ifdef HAVE_PROTOBUF - dc->d_uuid = uniqueId; + if (luaconfsLocal->protobufServer) { + dc->d_uuid = uniqueId; + } dc->d_ednssubnet = ednssubnet; #endif diff --git a/pdns/protobuf.cc b/pdns/protobuf.cc new file mode 100644 index 0000000000..304a88cb88 --- /dev/null +++ b/pdns/protobuf.cc @@ -0,0 +1,196 @@ + +#include "protobuf.hh" +#include "dnsparser.hh" +#include "gettime.hh" + +DNSProtoBufMessage::DNSProtoBufMessage(DNSProtoBufMessageType type) +{ +#ifdef HAVE_PROTOBUF + d_message.set_type(type == DNSProtoBufMessage::DNSProtoBufMessageType::Query ? PBDNSMessage_Type_DNSQueryType : PBDNSMessage_Type_DNSResponseType); +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::setQuestion(const DNSName& qname, uint16_t qtype, uint16_t qclass) +{ +#ifdef HAVE_PROTOBUF + PBDNSMessage_DNSQuestion* question = d_message.mutable_question(); + if (question) { + question->set_qname(qname.toString()); + question->set_qtype(qtype); + question->set_qclass(qclass); + } +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::setBytes(size_t bytes) +{ +#ifdef HAVE_PROTOBUF + d_message.set_inbytes(bytes); +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::setResponseCode(uint8_t rcode) +{ +#ifdef HAVE_PROTOBUF + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (response) { + response->set_rcode(rcode); + } +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::setTime(time_t sec, uint32_t usec) +{ +#ifdef HAVE_PROTOBUF + d_message.set_timesec(sec); + d_message.set_timeusec(usec); +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::setEDNSSubnet(const Netmask& subnet) +{ +#ifdef HAVE_PROTOBUF + if (!subnet.empty()) { + const ComboAddress ca = subnet.getNetwork(); + if (ca.sin4.sin_family == AF_INET) { + d_message.set_originalrequestorsubnet(&ca.sin4.sin_addr.s_addr, sizeof(ca.sin4.sin_addr.s_addr)); + } + else if (ca.sin4.sin_family == AF_INET6) { + d_message.set_originalrequestorsubnet(&ca.sin6.sin6_addr.s6_addr, sizeof(ca.sin6.sin6_addr.s6_addr)); + } + } +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::addRRsFromPacket(const char* packet, const size_t len) +{ +#ifdef HAVE_PROTOBUF + if (len < sizeof(struct dnsheader)) + return; + + const struct dnsheader* dh = (const struct dnsheader*) packet; + + if (ntohs(dh->ancount) == 0) + return; + + if (ntohs(dh->qdcount) == 0) + return; + + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (!response) + return; + + vector content(len - sizeof(dnsheader)); + copy(packet + sizeof(dnsheader), packet + len, content.begin()); + PacketReader pr(content); + + size_t idx = 0; + DNSName rrname; + uint16_t qdcount = ntohs(dh->qdcount); + uint16_t ancount = ntohs(dh->ancount); + uint16_t rrtype; + uint16_t rrclass; + string blob; + struct dnsrecordheader ah; + + rrname = pr.getName(); + rrtype = pr.get16BitInt(); + rrclass = pr.get16BitInt(); + + /* consume remaining qd if any */ + if (qdcount > 1) { + for(idx = 1; idx < qdcount; idx++) { + rrname = pr.getName(); + rrtype = pr.get16BitInt(); + rrclass = pr.get16BitInt(); + (void) rrtype; + (void) rrclass; + } + } + + /* parse AN */ + for (idx = 0; idx < ancount; idx++) { + rrname = pr.getName(); + pr.getDnsrecordheader(ah); + + pr.xfrBlob(blob); + + if (ah.d_type == QType::A || ah.d_type == QType::AAAA) { + PBDNSMessage_DNSResponse_DNSRR* rr = response->add_rrs(); + if (rr) { + rr->set_name(rrname.toString()); + rr->set_type(ah.d_type); + rr->set_class_(ah.d_class); + rr->set_ttl(ah.d_ttl); + rr->set_rdata(blob.c_str(), blob.length()); + } + } + } +#endif /* HAVE_PROTOBUF */ +} + +void DNSProtoBufMessage::serialize(std::string& data) const +{ +#ifdef HAVE_PROTOBUF + d_message.SerializeToString(&data); +#endif /* HAVE_PROTOBUF */ +} + +std::string DNSProtoBufMessage::toDebugString() const +{ +#ifdef HAVE_PROTOBUF + return d_message.DebugString(); +#endif /* HAVE_PROTOBUF */ +} + +#ifdef HAVE_PROTOBUF + +void DNSProtoBufMessage::setUUID(const boost::uuids::uuid& uuid) +{ + std::string* messageId = d_message.mutable_messageid(); + messageId->resize(uuid.size()); + std::copy(uuid.begin(), uuid.end(), messageId->begin()); +} + +void DNSProtoBufMessage::update(const boost::uuids::uuid& uuid, const ComboAddress* requestor, const ComboAddress* responder, bool isTCP, uint16_t id) +{ + struct timespec ts; + gettime(&ts, true); + setTime(ts.tv_sec, ts.tv_nsec / 1000); + + setUUID(uuid); + d_message.set_id(ntohs(id)); + + d_message.set_socketfamily((requestor && requestor->sin4.sin_family == AF_INET) ? PBDNSMessage_SocketFamily_INET : PBDNSMessage_SocketFamily_INET6); + d_message.set_socketprotocol(isTCP ? PBDNSMessage_SocketProtocol_TCP : PBDNSMessage_SocketProtocol_UDP); + + if (responder) { + if (responder->sin4.sin_family == AF_INET) { + d_message.set_to(&responder->sin4.sin_addr.s_addr, sizeof(responder->sin4.sin_addr.s_addr)); + } + else if (responder->sin4.sin_family == AF_INET6) { + d_message.set_to(&responder->sin6.sin6_addr.s6_addr, sizeof(responder->sin6.sin6_addr.s6_addr)); + } + } + if (requestor) { + if (requestor->sin4.sin_family == AF_INET) { + d_message.set_from(&requestor->sin4.sin_addr.s_addr, sizeof(requestor->sin4.sin_addr.s_addr)); + } + else if (requestor->sin4.sin_family == AF_INET6) { + d_message.set_from(&requestor->sin6.sin6_addr.s6_addr, sizeof(requestor->sin6.sin6_addr.s6_addr)); + } + } +} + + +DNSProtoBufMessage::DNSProtoBufMessage(DNSProtoBufMessageType type, const boost::uuids::uuid& uuid, const ComboAddress* requestor, const ComboAddress* to, const DNSName& domain, int qtype, uint16_t qclass, uint16_t qid, bool isTCP, size_t bytes) +{ + update(uuid, requestor, to, isTCP, qid); + + d_message.set_type(type == DNSProtoBufMessage::DNSProtoBufMessageType::Query ? PBDNSMessage_Type_DNSQueryType : PBDNSMessage_Type_DNSResponseType); + + setBytes(bytes); + setQuestion(domain, qtype, qclass); +} + +#endif /* HAVE_PROTOBUF */ diff --git a/pdns/protobuf.hh b/pdns/protobuf.hh new file mode 100644 index 0000000000..faf70c27db --- /dev/null +++ b/pdns/protobuf.hh @@ -0,0 +1,53 @@ + +#pragma once + +#include +#include + +#include "config.h" + +#include "dnsname.hh" +#include "iputils.hh" + +#ifdef HAVE_PROTOBUF +#include +#include +#include "dnsmessage.pb.h" +#endif /* HAVE_PROTOBUF */ + +class DNSProtoBufMessage +{ +public: + enum DNSProtoBufMessageType { + Query, + Response + }; + + DNSProtoBufMessage() + { + } + + DNSProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType type); + + ~DNSProtoBufMessage() + { + } + + void setQuestion(const DNSName& qname, uint16_t qtype, uint16_t qclass); + void setEDNSSubnet(const Netmask& subnet); + void setBytes(size_t bytes); + void setTime(time_t sec, uint32_t usec); + void setResponseCode(uint8_t rcode); + void addRRsFromPacket(const char* packet, const size_t len); + void serialize(std::string& data) const; + std::string toDebugString() const; + +#ifdef HAVE_PROTOBUF + DNSProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType type, const boost::uuids::uuid& uuid, const ComboAddress* requestor, const ComboAddress* responder, const DNSName& domain, int qtype, uint16_t qclass, uint16_t qid, bool isTCP, size_t bytes); + void update(const boost::uuids::uuid& uuid, const ComboAddress* requestor, const ComboAddress* responder, bool isTCP, uint16_t id); + void setUUID(const boost::uuids::uuid& uuid); + +protected: + PBDNSMessage d_message; +#endif /* HAVE_PROTOBUF */ +}; diff --git a/pdns/rec-protobuf.cc b/pdns/rec-protobuf.cc new file mode 100644 index 0000000000..77e05d5d67 --- /dev/null +++ b/pdns/rec-protobuf.cc @@ -0,0 +1,74 @@ + +#include "config.h" +#include "rec-protobuf.hh" + +void RecProtoBufMessage::addRR(const DNSRecord& record) +{ +#ifdef HAVE_PROTOBUF + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (!response) { + return; + } + + if (record.d_place != DNSResourceRecord::ANSWER || + record.d_class != QClass::IN || + (record.d_type != QType::A && + record.d_type != QType::AAAA && + record.d_type != QType::CNAME)) { + return; + } + + PBDNSMessage_DNSResponse_DNSRR* pbRR = response->add_rrs(); + if (!pbRR) { + return; + } + + pbRR->set_name(record.d_name.toString()); + pbRR->set_type(record.d_type); + pbRR->set_class_(record.d_class); + pbRR->set_ttl(record.d_ttl); + if (record.d_type == QType::A) { + const ARecordContent& arc = dynamic_cast(*(record.d_content)); + ComboAddress data = arc.getCA(); + pbRR->set_rdata(&data.sin4.sin_addr.s_addr, sizeof(data.sin4.sin_addr.s_addr)); + } + else if (record.d_type == QType::AAAA) { + const AAAARecordContent& arc = dynamic_cast(*(record.d_content)); + ComboAddress data = arc.getCA(); + pbRR->set_rdata(&data.sin6.sin6_addr.s6_addr, sizeof(data.sin6.sin6_addr.s6_addr)); + } else if (record.d_type == QType::CNAME) { + const CNAMERecordContent& crc = dynamic_cast(*(record.d_content)); + DNSName data = crc.getTarget(); + pbRR->set_rdata(data.toString()); + } +#endif /* HAVE_PROTOBUF */ +} + +void RecProtoBufMessage::addRRs(const std::vector& records) +{ + for (const auto& record : records) { + addRR(record); + } +} + +void RecProtoBufMessage::setAppliedPolicy(const std::string& policy) +{ +#ifdef HAVE_PROTOBUF + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (response && !policy.empty()) { + response->set_appliedpolicy(policy); + } +#endif /* HAVE_PROTOBUF */ +} + +void RecProtoBufMessage::setPolicyTags(const std::vector& policyTags) +{ +#ifdef HAVE_PROTOBUF + PBDNSMessage_DNSResponse* response = d_message.mutable_response(); + if (response) { + for (const auto& tag : policyTags) { + response->add_tags(tag); + } + } +#endif /* HAVE_PROTOBUF */ +} diff --git a/pdns/rec-protobuf.hh b/pdns/rec-protobuf.hh new file mode 100644 index 0000000000..99dcaa564f --- /dev/null +++ b/pdns/rec-protobuf.hh @@ -0,0 +1,29 @@ +#pragma once + +#include "protobuf.hh" + +#include "dnsrecords.hh" + +class RecProtoBufMessage: public DNSProtoBufMessage +{ +public: + RecProtoBufMessage(): DNSProtoBufMessage() + { + } + + RecProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType type): DNSProtoBufMessage(type) + { + } + +#ifdef HAVE_PROTOBUF + RecProtoBufMessage(DNSProtoBufMessage::DNSProtoBufMessageType type, const boost::uuids::uuid& uuid, const ComboAddress* requestor, const ComboAddress* responder, const DNSName& domain, int qtype, uint16_t qclass, uint16_t qid, bool isTCP, size_t bytes): DNSProtoBufMessage(type, uuid, requestor, responder, domain, qtype, qclass, qid, isTCP, bytes) + { + } +#endif /* HAVE_PROTOBUF */ + + void addRRs(const std::vector& records); + void addRR(const DNSRecord& record); + void setAppliedPolicy(const std::string& policy); + void setPolicyTags(const std::vector& policyTags); + +}; diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc index 8b45f7041a..811d0a763d 100644 --- a/pdns/recpacketcache.cc +++ b/pdns/recpacketcache.cc @@ -95,7 +95,6 @@ uint32_t RecursorPacketCache::canHashPacket(const std::string& origPacket) return ret; } -#ifdef HAVE_PROTOBUF bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age) { @@ -103,11 +102,7 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, - std::string* responsePacket, uint32_t* age, PBDNSMessage* protobufMessage) -#else -bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, - std::string* responsePacket, uint32_t* age) -#endif + std::string* responsePacket, uint32_t* age, RecProtoBufMessage* protobufMessage) { uint32_t h = canHashPacket(queryPacket); auto& idx = d_packetCache.get(); @@ -158,16 +153,12 @@ bool RecursorPacketCache::getResponsePacket(unsigned int tag, const std::string& } -#ifdef HAVE_PROTOBUF void RecursorPacketCache::insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttl) { insertResponsePacket(tag, qname, qtype, queryPacket, responsePacket, now, ttl, nullptr); } -void RecursorPacketCache::insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttl, const PBDNSMessage* protobufMessage) -#else -void RecursorPacketCache::insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttl) -#endif +void RecursorPacketCache::insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttl, const RecProtoBufMessage* protobufMessage) { auto qhash = canHashPacket(queryPacket); auto& idx = d_packetCache.get(); diff --git a/pdns/recpacketcache.hh b/pdns/recpacketcache.hh index e61dfefe3d..5cca3f8aaa 100644 --- a/pdns/recpacketcache.hh +++ b/pdns/recpacketcache.hh @@ -15,11 +15,7 @@ #ifdef HAVE_CONFIG_H #include "config.h" #endif -#ifdef HAVE_PROTOBUF -#include -#include -#include "dnsmessage.pb.h" -#endif +#include "rec-protobuf.hh" using namespace ::boost::multi_index; @@ -35,10 +31,8 @@ public: RecursorPacketCache(); bool getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age); void insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttd); -#ifdef HAVE_PROTOBUF - bool getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age, PBDNSMessage* protobufMessage); - void insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttd, const PBDNSMessage* protobufMessage); -#endif + bool getResponsePacket(unsigned int tag, const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age, RecProtoBufMessage* protobufMessage); + void insertResponsePacket(unsigned int tag, const DNSName& qname, uint16_t qtype, const std::string& queryPacket, const std::string& responsePacket, time_t now, uint32_t ttd, const RecProtoBufMessage* protobufMessage); void doPruneTo(unsigned int maxSize=250000); uint64_t doDump(int fd); int doWipePacketCache(const DNSName& name, uint16_t qtype=0xffff, bool subtree=false); @@ -59,7 +53,7 @@ private: uint16_t d_type; mutable std::string d_packet; // "I know what I am doing" #ifdef HAVE_PROTOBUF - mutable PBDNSMessage d_protobufMessage; + mutable RecProtoBufMessage d_protobufMessage; #endif uint32_t d_qhash; uint32_t d_tag; diff --git a/pdns/recursordist/Makefile.am b/pdns/recursordist/Makefile.am index 0a42a72147..8ff5623146 100644 --- a/pdns/recursordist/Makefile.am +++ b/pdns/recursordist/Makefile.am @@ -100,12 +100,14 @@ pdns_recursor_SOURCES = \ opensslsigners.cc opensslsigners.hh \ pdns_recursor.cc \ pdnsexception.hh \ + protobuf.cc protobuf.hh \ pubsuffix.hh pubsuffix.cc \ qtype.hh qtype.cc \ randomhelper.cc \ rcpgenerator.cc rcpgenerator.hh \ rec-carbon.cc \ rec-lua-conf.hh rec-lua-conf.cc \ + rec-protobuf.cc rec-protobuf.hh \ rec_channel.cc rec_channel.hh \ rec_channel_rec.cc \ recpacketcache.cc recpacketcache.hh \ diff --git a/pdns/recursordist/protobuf.cc b/pdns/recursordist/protobuf.cc new file mode 120000 index 0000000000..088ecc89e8 --- /dev/null +++ b/pdns/recursordist/protobuf.cc @@ -0,0 +1 @@ +../protobuf.cc \ No newline at end of file diff --git a/pdns/recursordist/protobuf.hh b/pdns/recursordist/protobuf.hh new file mode 120000 index 0000000000..c7def8e442 --- /dev/null +++ b/pdns/recursordist/protobuf.hh @@ -0,0 +1 @@ +../protobuf.hh \ No newline at end of file diff --git a/pdns/recursordist/rec-protobuf.cc b/pdns/recursordist/rec-protobuf.cc new file mode 120000 index 0000000000..1bab3e4e2e --- /dev/null +++ b/pdns/recursordist/rec-protobuf.cc @@ -0,0 +1 @@ +../rec-protobuf.cc \ No newline at end of file diff --git a/pdns/recursordist/rec-protobuf.hh b/pdns/recursordist/rec-protobuf.hh new file mode 120000 index 0000000000..93a7607b37 --- /dev/null +++ b/pdns/recursordist/rec-protobuf.hh @@ -0,0 +1 @@ +../rec-protobuf.hh \ No newline at end of file