From: Charles-Henri Bruyand Date: Thu, 10 Jun 2021 15:22:59 +0000 (+0200) Subject: recursor: lua ffi, add support for setting arbitrary key-value metadata X-Git-Tag: dnsdist-1.7.0-alpha1~106^2~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=634dd69e64ed982a4e047546d1a16f135bf0223a;p=thirdparty%2Fpdns.git recursor: lua ffi, add support for setting arbitrary key-value metadata --- diff --git a/pdns/dnsmessage.proto b/pdns/dnsmessage.proto index f6ba17ccf5..972bd8de38 100644 --- a/pdns/dnsmessage.proto +++ b/pdns/dnsmessage.proto @@ -115,6 +115,17 @@ message PBDNSMessage { optional string deviceName = 19; // Device name of the requestor optional uint32 fromPort = 20; // Source port of the DNS query (client) optional uint32 toPort = 21; // Destination port of the DNS query (server) + + message MetaValue { + repeated string stringVal = 1; + repeated int64 intVal = 2; + } + + message Meta { + required string key = 1; // MUST be unique, so if you have multiple values they must be aggregated into on Meta + required MetaValue value = 2; + } + repeated Meta meta = 22; // Arbitrary meta-data - to be used in future rather than adding new fields all the time } message PBDNSMessageList { diff --git a/pdns/lua-recursor4-ffi.hh b/pdns/lua-recursor4-ffi.hh index d4559a3d33..30ca676098 100644 --- a/pdns/lua-recursor4-ffi.hh +++ b/pdns/lua-recursor4-ffi.hh @@ -84,4 +84,6 @@ extern "C" { bool pdns_ffi_param_add_record(pdns_ffi_param_t *ref, const char* name, uint16_t type, uint32_t ttl, const char* content, size_t contentSize, pdns_record_place_t place) __attribute__ ((visibility ("default"))); void pdns_ffi_param_set_padding_disabled(pdns_ffi_param_t* ref, bool disabled) __attribute__ ((visibility ("default"))); + void pdns_ffi_param_add_meta_single_string_kv(pdns_ffi_param_t *ref, const char* key, const char* val) __attribute__ ((visibility ("default"))); + void pdns_ffi_param_add_meta_single_int64_kv(pdns_ffi_param_t *ref, const char* key, int64_t val) __attribute__ ((visibility ("default"))); } diff --git a/pdns/lua-recursor4.cc b/pdns/lua-recursor4.cc index 5aef9e9f33..c410cc1809 100644 --- a/pdns/lua-recursor4.cc +++ b/pdns/lua-recursor4.cc @@ -976,3 +976,13 @@ void pdns_ffi_param_set_padding_disabled(pdns_ffi_param_t* ref, bool disabled) { ref->params.disablePadding = disabled; } + +void pdns_ffi_param_add_meta_single_string_kv(pdns_ffi_param_t* ref, const char* key, const char* val) +{ + ref->params.meta[std::string(key)].stringVal.insert(std::string(val)); +} + +void pdns_ffi_param_add_meta_single_int64_kv(pdns_ffi_param_t* ref, const char* key, int64_t val) +{ + ref->params.meta[std::string(key)].intVal.insert(val); +} diff --git a/pdns/lua-recursor4.hh b/pdns/lua-recursor4.hh index 597191fb87..94e85a1d48 100644 --- a/pdns/lua-recursor4.hh +++ b/pdns/lua-recursor4.hh @@ -106,7 +106,7 @@ public: int rcode{0}; // struct dnsheader, packet length would be great vector records; - + string followupFunction; string followupPrefix; @@ -114,7 +114,7 @@ public: ComboAddress udpQueryDest; string udpAnswer; string udpCallback; - + LuaContext::LuaObject data; DNSName followupName; }; @@ -132,11 +132,16 @@ public: std::unordered_set* policyTags{nullptr}; std::unordered_map* discardedPolicies{nullptr}; }; + struct MetaValue + { + std::unordered_set stringVal; + std::unordered_set intVal; + }; struct FFIParams { public: - FFIParams(const DNSName& qname_, uint16_t qtype_, const ComboAddress& local_, const ComboAddress& remote_, const Netmask& ednssubnet_, LuaContext::LuaObject& data_, std::unordered_set& policyTags_, std::vector& records_, const EDNSOptionViewMap& ednsOptions_, const std::vector& proxyProtocolValues_, std::string& requestorId_, std::string& deviceId_, std::string& deviceName_, std::string& routingTag_, boost::optional& rcode_, uint32_t& ttlCap_, bool& variable_, bool tcp_, bool& logQuery_, bool& logResponse_, bool& followCNAMERecords_, boost::optional& extendedErrorCode_, std::string& extendedErrorExtra_, bool& disablePadding_): data(data_), qname(qname_), local(local_), remote(remote_), ednssubnet(ednssubnet_), policyTags(policyTags_), records(records_), ednsOptions(ednsOptions_), proxyProtocolValues(proxyProtocolValues_), requestorId(requestorId_), deviceId(deviceId_), deviceName(deviceName_), routingTag(routingTag_), extendedErrorExtra(extendedErrorExtra_), rcode(rcode_), extendedErrorCode(extendedErrorCode_), ttlCap(ttlCap_), variable(variable_), logQuery(logQuery_), logResponse(logResponse_), followCNAMERecords(followCNAMERecords_), disablePadding(disablePadding_), qtype(qtype_), tcp(tcp_) + FFIParams(const DNSName& qname_, uint16_t qtype_, const ComboAddress& local_, const ComboAddress& remote_, const Netmask& ednssubnet_, LuaContext::LuaObject& data_, std::unordered_set& policyTags_, std::vector& records_, const EDNSOptionViewMap& ednsOptions_, const std::vector& proxyProtocolValues_, std::string& requestorId_, std::string& deviceId_, std::string& deviceName_, std::string& routingTag_, boost::optional& rcode_, uint32_t& ttlCap_, bool& variable_, bool tcp_, bool& logQuery_, bool& logResponse_, bool& followCNAMERecords_, boost::optional& extendedErrorCode_, std::string& extendedErrorExtra_, bool& disablePadding_, std::map& meta_): data(data_), qname(qname_), local(local_), remote(remote_), ednssubnet(ednssubnet_), policyTags(policyTags_), records(records_), ednsOptions(ednsOptions_), proxyProtocolValues(proxyProtocolValues_), requestorId(requestorId_), deviceId(deviceId_), deviceName(deviceName_), routingTag(routingTag_), extendedErrorExtra(extendedErrorExtra_), rcode(rcode_), extendedErrorCode(extendedErrorCode_), ttlCap(ttlCap_), variable(variable_), logQuery(logQuery_), logResponse(logResponse_), followCNAMERecords(followCNAMERecords_), disablePadding(disablePadding_), qtype(qtype_), tcp(tcp_), meta(meta_) { } @@ -166,6 +171,8 @@ public: unsigned int tag{0}; uint16_t qtype; bool tcp; + + std::map& meta; }; unsigned int gettag(const ComboAddress& remote, const Netmask& ednssubnet, const ComboAddress& local, const DNSName& qname, uint16_t qtype, std::unordered_set* policyTags, LuaContext::LuaObject& data, const EDNSOptionViewMap&, bool tcp, std::string& requestorId, std::string& deviceId, std::string& deviceName, std::string& routingTag, const std::vector& proxyProtocolValues) const; @@ -212,4 +219,3 @@ private: typedef std::function policyEventFilter_t; policyEventFilter_t d_policyHitEventFilter; }; - diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index f1204784dd..f5db6328b2 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -362,6 +362,7 @@ struct DNSComboWriter { bool d_logResponse{false}; bool d_tcp{false}; bool d_responsePaddingDisabled{false}; + std::map d_meta; }; MT_t* getMT() @@ -1016,7 +1017,7 @@ catch(...) return "Exception making error message for exception"; } -static void protobufLogQuery(LocalStateHolder& luaconfsLocal, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, size_t len, const DNSName& qname, uint16_t qtype, uint16_t qclass, const std::unordered_set& policyTags, const std::string& requestorId, const std::string& deviceId, const std::string& deviceName) +static void protobufLogQuery(LocalStateHolder& luaconfsLocal, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, size_t len, const DNSName& qname, uint16_t qtype, uint16_t qclass, const std::unordered_set& policyTags, const std::string& requestorId, const std::string& deviceId, const std::string& deviceName, const std::map& meta) { if (!t_protobufServers) { return; @@ -1038,6 +1039,9 @@ static void protobufLogQuery(LocalStateHolder& luaconfsLocal, co if (!policyTags.empty()) { m.addPolicyTags(policyTags); } + if (!meta.empty()) { + m.setMeta(meta); + } std::string msg(m.finishAndMoveBuf()); for (auto& server : *t_protobufServers) { @@ -1062,7 +1066,7 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolder& meta) { pdns::ProtoZero::RecMessage pbMessage(pbData ? pbData->d_message : "", pbData ? pbData->d_response : "", 64, 10); // The extra bytes we are going to add // Normally we take the immutable string from the cache and append a few values, but if it's not there (can this happen?) @@ -1096,6 +1100,7 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolderd_gettag_ffi) { - RecursorLua4::FFIParams params(qname, qtype, dc->d_destination, dc->d_source, dc->d_ednssubnet.source, dc->d_data, dc->d_policyTags, dc->d_records, ednsOptions, dc->d_proxyProtocolValues, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_rcode, dc->d_ttlCap, dc->d_variable, true, logQuery, dc->d_logResponse, dc->d_followCNAMERecords, dc->d_extendedErrorCode, dc->d_extendedErrorExtra, dc->d_responsePaddingDisabled); + RecursorLua4::FFIParams params(qname, qtype, dc->d_destination, dc->d_source, dc->d_ednssubnet.source, dc->d_data, dc->d_policyTags, dc->d_records, ednsOptions, dc->d_proxyProtocolValues, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_rcode, dc->d_ttlCap, dc->d_variable, true, logQuery, dc->d_logResponse, dc->d_followCNAMERecords, dc->d_extendedErrorCode, dc->d_extendedErrorExtra, dc->d_responsePaddingDisabled, dc->d_meta); dc->d_tag = t_pdl->gettag_ffi(params); } else if (t_pdl->d_gettag) { @@ -2759,7 +2764,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) try { if (logQuery && !(luaconfsLocal->protobufExportConfig.taggedOnly && dc->d_policyTags.empty())) { - protobufLogQuery(luaconfsLocal, dc->d_uuid, dc->d_source, dc->d_destination, dc->d_ednssubnet.source, true, dh->id, conn->qlen, qname, qtype, qclass, dc->d_policyTags, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName); + protobufLogQuery(luaconfsLocal, dc->d_uuid, dc->d_source, dc->d_destination, dc->d_ednssubnet.source, true, dh->id, conn->qlen, qname, qtype, qclass, dc->d_policyTags, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta); } } catch (const std::exception& e) { @@ -2820,7 +2825,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var) if (cacheHit) { if (t_protobufServers && dc->d_logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { struct timeval tv{0, 0}; - protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName); + protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta); } if (!g_quiet) { @@ -2952,6 +2957,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr bool needECS = false; bool needXPF = g_XPFAcl.match(fromaddr); std::unordered_set policyTags; + std::map meta; LuaContext::LuaObject data; string requestorId; string deviceId; @@ -3017,7 +3023,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr if(t_pdl) { try { if (t_pdl->d_gettag_ffi) { - RecursorLua4::FFIParams params(qname, qtype, destination, source, ednssubnet.source, data, policyTags, records, ednsOptions, proxyProtocolValues, requestorId, deviceId, deviceName, routingTag, rcode, ttlCap, variable, false, logQuery, logResponse, followCNAMEs, extendedErrorCode, extendedErrorExtra, responsePaddingDisabled); + RecursorLua4::FFIParams params(qname, qtype, destination, source, ednssubnet.source, data, policyTags, records, ednsOptions, proxyProtocolValues, requestorId, deviceId, deviceName, routingTag, rcode, ttlCap, variable, false, logQuery, logResponse, followCNAMEs, extendedErrorCode, extendedErrorExtra, responsePaddingDisabled, meta); ctag = t_pdl->gettag_ffi(params); } @@ -3043,7 +3049,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr RecursorPacketCache::OptPBData pbData{boost::none}; if (t_protobufServers) { if (logQuery && !(luaconfsLocal->protobufExportConfig.taggedOnly && policyTags.empty())) { - protobufLogQuery(luaconfsLocal, uniqueId, source, destination, ednssubnet.source, false, dh->id, question.size(), qname, qtype, qclass, policyTags, requestorId, deviceId, deviceName); + protobufLogQuery(luaconfsLocal, uniqueId, source, destination, ednssubnet.source, false, dh->id, question.size(), qname, qtype, qclass, policyTags, requestorId, deviceId, deviceName, meta); } } @@ -3056,7 +3062,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr bool cacheHit = checkForCacheHit(qnameParsed, ctag, question, qname, qtype, qclass, g_now, response, qhash, pbData, false, source); if (cacheHit) { if (t_protobufServers && logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) { - protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName); + protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName, meta); } if (!g_quiet) { diff --git a/pdns/protozero.hh b/pdns/protozero.hh index 53547bf402..851cb8192c 100644 --- a/pdns/protozero.hh +++ b/pdns/protozero.hh @@ -27,13 +27,17 @@ #include "iputils.hh" #include "gettime.hh" #include "uuid-utils.hh" +#include "lua-recursor4.hh" namespace pdns { namespace ProtoZero { class Message { public: + + enum class MetaValueField : protozero::pbf_tag_type { stringVal = 1, intVal = 2 }; + enum class MetaField : protozero::pbf_tag_type { key = 1, value = 2 }; enum class MessageType : int32_t { DNSQueryType = 1, DNSResponseType = 2, DNSOutgoingQueryType = 3, DNSIncomingResponseType = 4 }; - enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21 }; + enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21, meta = 22}; enum class QuestionField : protozero::pbf_tag_type { qName = 1, qType = 2, qClass = 3}; enum class ResponseField : protozero::pbf_tag_type { rcode = 1, rrs = 2, appliedPolicy = 3, tags = 4, queryTimeSec = 5, queryTimeUsec = 6, appliedPolicyType = 7, appliedPolicyTrigger = 8, appliedPolicyHit = 9, appliedPolicyKind = 10, validationState = 11 }; enum class RRField : protozero::pbf_tag_type { name = 1, type = 2, class_ = 3, ttl = 4, rdata = 5, udr = 6 }; @@ -117,6 +121,21 @@ namespace pdns { pbf_question.add_uint32(static_cast(QuestionField::qClass), qclass); } + void setMeta(const std::map& meta) + { + for (const auto& m : meta) { + protozero::pbf_writer pbf_meta{d_message, static_cast(Field::meta)}; + pbf_meta.add_string(static_cast(MetaField::key), m.first); + protozero::pbf_writer pbf_meta_value{pbf_meta, static_cast(MetaField::value)}; + for (const auto& s: m.second.stringVal) { + pbf_meta_value.add_string(static_cast(MetaValueField::stringVal), s); + } + for (const auto& i: m.second.intVal) { + pbf_meta_value.add_uint64(static_cast(MetaValueField::intVal), i); + } + } + } + void setEDNSSubnet(const Netmask& nm, uint8_t mask) { encodeNetmask(static_cast(Field::originalRequestorSubnet), nm, mask);