recursor: lua ffi, add support for setting arbitrary key-value metadata

diff --git a/pdns/dnsmessage.proto b/pdns/dnsmessage.proto
index f6ba17ccf5adf7fc0b0fde9a0587211aec068d31..972bd8de38f23d307c757f3e8b77022a26e10fc3 100644 (file)
--- a/pdns/dnsmessage.proto
+++ b/pdns/dnsmessage.proto
@@ -115,6 +115,17 @@ message PBDNSMessage {
   optional string deviceName = 19;              // Device name of the requestor
   optional uint32 fromPort = 20;                // Source port of the DNS query (client)
   optional uint32 toPort = 21;                  // Destination port of the DNS query (server)
+
+  message MetaValue {
+    repeated string stringVal = 1;
+    repeated int64 intVal = 2;
+  }
+
+  message Meta {
+    required string key = 1;                    // MUST be unique, so if you have multiple values they must be aggregated into on Meta
+    required MetaValue value = 2;
+  }
+  repeated Meta meta = 22;                      // Arbitrary meta-data - to be used in future rather than adding new fields all the time
 }
 
 message PBDNSMessageList {

diff --git a/pdns/lua-recursor4-ffi.hh b/pdns/lua-recursor4-ffi.hh
index d4559a3d33a71eb9d4944d70cf7edd57b131214b..30ca6760987f7560c3c160bd1e523c9996145939 100644 (file)
--- a/pdns/lua-recursor4-ffi.hh
+++ b/pdns/lua-recursor4-ffi.hh
@@ -84,4 +84,6 @@ extern "C" {
   bool pdns_ffi_param_add_record(pdns_ffi_param_t *ref, const char* name, uint16_t type, uint32_t ttl, const char* content, size_t contentSize, pdns_record_place_t place) __attribute__ ((visibility ("default")));
 
   void pdns_ffi_param_set_padding_disabled(pdns_ffi_param_t* ref, bool disabled)  __attribute__ ((visibility ("default")));
+  void pdns_ffi_param_add_meta_single_string_kv(pdns_ffi_param_t *ref, const char* key, const char* val) __attribute__ ((visibility ("default")));
+  void pdns_ffi_param_add_meta_single_int64_kv(pdns_ffi_param_t *ref, const char* key, int64_t val) __attribute__ ((visibility ("default")));
 }

diff --git a/pdns/lua-recursor4.cc b/pdns/lua-recursor4.cc
index 5aef9e9f332f82801361e6d5f49eeca28c161f2a..c410cc180911e5078919e3872ba5f77419fc05de 100644 (file)
--- a/pdns/lua-recursor4.cc
+++ b/pdns/lua-recursor4.cc
@@ -976,3 +976,13 @@ void pdns_ffi_param_set_padding_disabled(pdns_ffi_param_t* ref, bool disabled)
 {
   ref->params.disablePadding = disabled;
 }
+
+void pdns_ffi_param_add_meta_single_string_kv(pdns_ffi_param_t* ref, const char* key, const char* val)
+{
+  ref->params.meta[std::string(key)].stringVal.insert(std::string(val));
+}
+
+void pdns_ffi_param_add_meta_single_int64_kv(pdns_ffi_param_t* ref, const char* key, int64_t val)
+{
+  ref->params.meta[std::string(key)].intVal.insert(val);
+}

diff --git a/pdns/lua-recursor4.hh b/pdns/lua-recursor4.hh
index 597191fb879315c48135beb4ba61377d0a88d764..94e85a1d485d1424dd6821e2419ab366d394ae65 100644 (file)
--- a/pdns/lua-recursor4.hh
+++ b/pdns/lua-recursor4.hh
@@ -106,7 +106,7 @@ public:
     int rcode{0};
     // struct dnsheader, packet length would be great
     vector<DNSRecord> records;
-    
+
     string followupFunction;
     string followupPrefix;
 
@@ -114,7 +114,7 @@ public:
     ComboAddress udpQueryDest;
     string udpAnswer;
     string udpCallback;
-    
+
     LuaContext::LuaObject data;
     DNSName followupName;
   };
@@ -132,11 +132,16 @@ public:
     std::unordered_set<std::string>* policyTags{nullptr};
     std::unordered_map<std::string, bool>* discardedPolicies{nullptr};
   };
+  struct MetaValue
+  {
+    std::unordered_set<std::string> stringVal;
+    std::unordered_set<int64_t> intVal;
+  };
 
   struct FFIParams
   {
   public:
-    FFIParams(const DNSName& qname_, uint16_t qtype_, const ComboAddress& local_, const ComboAddress& remote_, const Netmask& ednssubnet_, LuaContext::LuaObject& data_, std::unordered_set<std::string>& policyTags_, std::vector<DNSRecord>& records_, const EDNSOptionViewMap& ednsOptions_, const std::vector<ProxyProtocolValue>& proxyProtocolValues_, std::string& requestorId_, std::string& deviceId_, std::string& deviceName_, std::string& routingTag_, boost::optional<int>& rcode_, uint32_t& ttlCap_, bool& variable_, bool tcp_, bool& logQuery_, bool& logResponse_, bool& followCNAMERecords_, boost::optional<uint16_t>& extendedErrorCode_, std::string& extendedErrorExtra_, bool& disablePadding_): data(data_), qname(qname_), local(local_), remote(remote_), ednssubnet(ednssubnet_), policyTags(policyTags_), records(records_), ednsOptions(ednsOptions_), proxyProtocolValues(proxyProtocolValues_), requestorId(requestorId_), deviceId(deviceId_), deviceName(deviceName_), routingTag(routingTag_), extendedErrorExtra(extendedErrorExtra_), rcode(rcode_), extendedErrorCode(extendedErrorCode_), ttlCap(ttlCap_), variable(variable_), logQuery(logQuery_), logResponse(logResponse_), followCNAMERecords(followCNAMERecords_), disablePadding(disablePadding_), qtype(qtype_), tcp(tcp_)
+    FFIParams(const DNSName& qname_, uint16_t qtype_, const ComboAddress& local_, const ComboAddress& remote_, const Netmask& ednssubnet_, LuaContext::LuaObject& data_, std::unordered_set<std::string>& policyTags_, std::vector<DNSRecord>& records_, const EDNSOptionViewMap& ednsOptions_, const std::vector<ProxyProtocolValue>& proxyProtocolValues_, std::string& requestorId_, std::string& deviceId_, std::string& deviceName_, std::string& routingTag_, boost::optional<int>& rcode_, uint32_t& ttlCap_, bool& variable_, bool tcp_, bool& logQuery_, bool& logResponse_, bool& followCNAMERecords_, boost::optional<uint16_t>& extendedErrorCode_, std::string& extendedErrorExtra_, bool& disablePadding_, std::map<std::string, MetaValue>& meta_): data(data_), qname(qname_), local(local_), remote(remote_), ednssubnet(ednssubnet_), policyTags(policyTags_), records(records_), ednsOptions(ednsOptions_), proxyProtocolValues(proxyProtocolValues_), requestorId(requestorId_), deviceId(deviceId_), deviceName(deviceName_), routingTag(routingTag_), extendedErrorExtra(extendedErrorExtra_), rcode(rcode_), extendedErrorCode(extendedErrorCode_), ttlCap(ttlCap_), variable(variable_), logQuery(logQuery_), logResponse(logResponse_), followCNAMERecords(followCNAMERecords_), disablePadding(disablePadding_), qtype(qtype_), tcp(tcp_), meta(meta_)
     {
     }
 
@@ -166,6 +171,8 @@ public:
     unsigned int tag{0};
     uint16_t qtype;
     bool tcp;
+
+    std::map<std::string, MetaValue>& meta;
   };
 
   unsigned int gettag(const ComboAddress& remote, const Netmask& ednssubnet, const ComboAddress& local, const DNSName& qname, uint16_t qtype, std::unordered_set<std::string>* policyTags, LuaContext::LuaObject& data, const EDNSOptionViewMap&, bool tcp, std::string& requestorId, std::string& deviceId, std::string& deviceName, std::string& routingTag, const std::vector<ProxyProtocolValue>& proxyProtocolValues) const;
@@ -212,4 +219,3 @@ private:
   typedef std::function<bool(PolicyEvent&)> policyEventFilter_t;
   policyEventFilter_t d_policyHitEventFilter;
 };
-

diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index f1204784ddf1b534f873dc6fb1bd9ee9d3bb5272..f5db6328b226989aaedc77ae3c8d3a90061b645c 100644 (file)
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -362,6 +362,7 @@ struct DNSComboWriter {
   bool d_logResponse{false};
   bool d_tcp{false};
   bool d_responsePaddingDisabled{false};
+  std::map<std::string, RecursorLua4::MetaValue> d_meta;
 };
 
 MT_t* getMT()
@@ -1016,7 +1017,7 @@ catch(...)
   return "Exception making error message for exception";
 }
 
-static void protobufLogQuery(LocalStateHolder<LuaConfigItems>& luaconfsLocal, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, size_t len, const DNSName& qname, uint16_t qtype, uint16_t qclass, const std::unordered_set<std::string>& policyTags, const std::string& requestorId, const std::string& deviceId, const std::string& deviceName)
+static void protobufLogQuery(LocalStateHolder<LuaConfigItems>& luaconfsLocal, const boost::uuids::uuid& uniqueId, const ComboAddress& remote, const ComboAddress& local, const Netmask& ednssubnet, bool tcp, uint16_t id, size_t len, const DNSName& qname, uint16_t qtype, uint16_t qclass, const std::unordered_set<std::string>& policyTags, const std::string& requestorId, const std::string& deviceId, const std::string& deviceName, const std::map<std::string, RecursorLua4::MetaValue>& meta)
 {
   if (!t_protobufServers) {
     return;
@@ -1038,6 +1039,9 @@ static void protobufLogQuery(LocalStateHolder<LuaConfigItems>& luaconfsLocal, co
   if (!policyTags.empty()) {
     m.addPolicyTags(policyTags);
   }
+  if (!meta.empty()) {
+    m.setMeta(meta);
+  }
 
   std::string msg(m.finishAndMoveBuf());
   for (auto& server : *t_protobufServers) {
@@ -1062,7 +1066,7 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolder<Lua
                                 bool tcp, const ComboAddress& source, const ComboAddress& destination,
                                 const EDNSSubnetOpts& ednssubnet,
                                 const boost::uuids::uuid& uniqueId, const string& requestorId, const string& deviceId,
-                                const string& deviceName) 
+                                const string& deviceName, const std::map<std::string, RecursorLua4::MetaValue>& meta)
 {
   pdns::ProtoZero::RecMessage pbMessage(pbData ? pbData->d_message : "", pbData ? pbData->d_response : "", 64, 10); // The extra bytes we are going to add
   // Normally we take the immutable string from the cache and append a few values, but if it's not there (can this happen?)
@@ -1096,6 +1100,7 @@ static void protobufLogResponse(const struct dnsheader* dh, LocalStateHolder<Lua
   pbMessage.setDeviceName(deviceName);
   pbMessage.setFromPort(source.getPort());
   pbMessage.setToPort(destination.getPort());
+  pbMessage.setMeta(meta);
 #ifdef NOD_ENABLED
   if (g_nodEnabled) {
     pbMessage.setNewlyObservedDomain(false);
@@ -2720,7 +2725,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var)
           if(t_pdl) {
             try {
               if (t_pdl->d_gettag_ffi) {
-                RecursorLua4::FFIParams params(qname, qtype, dc->d_destination, dc->d_source, dc->d_ednssubnet.source, dc->d_data, dc->d_policyTags, dc->d_records, ednsOptions, dc->d_proxyProtocolValues, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_rcode, dc->d_ttlCap, dc->d_variable, true, logQuery, dc->d_logResponse, dc->d_followCNAMERecords, dc->d_extendedErrorCode, dc->d_extendedErrorExtra, dc->d_responsePaddingDisabled);
+                RecursorLua4::FFIParams params(qname, qtype, dc->d_destination, dc->d_source, dc->d_ednssubnet.source, dc->d_data, dc->d_policyTags, dc->d_records, ednsOptions, dc->d_proxyProtocolValues, requestorId, deviceId, deviceName, dc->d_routingTag, dc->d_rcode, dc->d_ttlCap, dc->d_variable, true, logQuery, dc->d_logResponse, dc->d_followCNAMERecords, dc->d_extendedErrorCode, dc->d_extendedErrorExtra, dc->d_responsePaddingDisabled, dc->d_meta);
                 dc->d_tag = t_pdl->gettag_ffi(params);
               }
               else if (t_pdl->d_gettag) {
@@ -2759,7 +2764,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var)
         try {
 
           if (logQuery && !(luaconfsLocal->protobufExportConfig.taggedOnly && dc->d_policyTags.empty())) {
-            protobufLogQuery(luaconfsLocal, dc->d_uuid, dc->d_source, dc->d_destination, dc->d_ednssubnet.source, true, dh->id, conn->qlen, qname, qtype, qclass, dc->d_policyTags, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName);
+            protobufLogQuery(luaconfsLocal, dc->d_uuid, dc->d_source, dc->d_destination, dc->d_ednssubnet.source, true, dh->id, conn->qlen, qname, qtype, qclass, dc->d_policyTags, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta);
           }
         }
         catch (const std::exception& e) {
@@ -2820,7 +2825,7 @@ static void handleRunningTCPQuestion(int fd, FDMultiplexer::funcparam_t& var)
         if (cacheHit) {
           if (t_protobufServers && dc->d_logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) {
             struct timeval tv{0, 0};
-            protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName);
+            protobufLogResponse(dh, luaconfsLocal, pbData, tv, true, dc->d_source, dc->d_destination, dc->d_ednssubnet, dc->d_uuid, dc->d_requestorId, dc->d_deviceId, dc->d_deviceName, dc->d_meta);
           }
 
           if (!g_quiet) {
@@ -2952,6 +2957,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr
   bool needECS = false;
   bool needXPF = g_XPFAcl.match(fromaddr);
   std::unordered_set<std::string> policyTags;
+  std::map<std::string, RecursorLua4::MetaValue> meta;
   LuaContext::LuaObject data;
   string requestorId;
   string deviceId;
@@ -3017,7 +3023,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr
         if(t_pdl) {
           try {
             if (t_pdl->d_gettag_ffi) {
-              RecursorLua4::FFIParams params(qname, qtype, destination, source, ednssubnet.source, data, policyTags, records, ednsOptions, proxyProtocolValues, requestorId, deviceId, deviceName, routingTag, rcode, ttlCap, variable, false, logQuery, logResponse, followCNAMEs, extendedErrorCode, extendedErrorExtra, responsePaddingDisabled);
+              RecursorLua4::FFIParams params(qname, qtype, destination, source, ednssubnet.source, data, policyTags, records, ednsOptions, proxyProtocolValues, requestorId, deviceId, deviceName, routingTag, rcode, ttlCap, variable, false, logQuery, logResponse, followCNAMEs, extendedErrorCode, extendedErrorExtra, responsePaddingDisabled, meta);
 
               ctag = t_pdl->gettag_ffi(params);
             }
@@ -3043,7 +3049,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr
     RecursorPacketCache::OptPBData pbData{boost::none};
     if (t_protobufServers) {
       if (logQuery && !(luaconfsLocal->protobufExportConfig.taggedOnly && policyTags.empty())) {
-        protobufLogQuery(luaconfsLocal, uniqueId, source, destination, ednssubnet.source, false, dh->id, question.size(), qname, qtype, qclass, policyTags, requestorId, deviceId, deviceName);
+        protobufLogQuery(luaconfsLocal, uniqueId, source, destination, ednssubnet.source, false, dh->id, question.size(), qname, qtype, qclass, policyTags, requestorId, deviceId, deviceName, meta);
       }
     }
 
@@ -3056,7 +3062,7 @@ static string* doProcessUDPQuestion(const std::string& question, const ComboAddr
     bool cacheHit = checkForCacheHit(qnameParsed, ctag, question, qname, qtype, qclass, g_now, response, qhash, pbData, false, source);
     if (cacheHit) {
       if (t_protobufServers && logResponse && !(luaconfsLocal->protobufExportConfig.taggedOnly && pbData && !pbData->d_tagged)) {
-        protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName);
+        protobufLogResponse(dh, luaconfsLocal, pbData, tv, false, source, destination, ednssubnet, uniqueId, requestorId, deviceId, deviceName, meta);
       }
 
       if (!g_quiet) {

diff --git a/pdns/protozero.hh b/pdns/protozero.hh
index 53547bf4027e9a5b7014d86a0d14797d0dcfd68a..851cb8192cbe39a7c6a847a8d523b69e3d8c6a2a 100644 (file)
--- a/pdns/protozero.hh
+++ b/pdns/protozero.hh
@@ -27,13 +27,17 @@
 #include "iputils.hh"
 #include "gettime.hh"
 #include "uuid-utils.hh"
+#include "lua-recursor4.hh"
 
 namespace pdns {
   namespace ProtoZero {
     class Message {
     public:
+
+      enum class MetaValueField : protozero::pbf_tag_type { stringVal = 1, intVal = 2 };
+      enum class MetaField : protozero::pbf_tag_type { key = 1, value = 2 };
       enum class MessageType : int32_t { DNSQueryType = 1, DNSResponseType = 2, DNSOutgoingQueryType = 3, DNSIncomingResponseType = 4 };
-      enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21 };
+      enum class Field : protozero::pbf_tag_type { type = 1, messageId = 2, serverIdentity = 3, socketFamily = 4, socketProtocol = 5, from = 6, to = 7, inBytes = 8, timeSec = 9, timeUsec = 10, id = 11, question = 12, response = 13, originalRequestorSubnet = 14, requestorId = 15, initialRequestId = 16, deviceId = 17, newlyObservedDomain = 18, deviceName = 19, fromPort = 20, toPort = 21, meta = 22};
       enum class QuestionField : protozero::pbf_tag_type { qName = 1, qType = 2, qClass = 3};
       enum class ResponseField : protozero::pbf_tag_type { rcode = 1, rrs = 2, appliedPolicy = 3, tags = 4, queryTimeSec = 5, queryTimeUsec = 6, appliedPolicyType = 7, appliedPolicyTrigger = 8, appliedPolicyHit = 9, appliedPolicyKind = 10, validationState = 11 };
       enum class RRField : protozero::pbf_tag_type { name = 1, type = 2, class_ = 3, ttl = 4, rdata = 5, udr = 6 };
@@ -117,6 +121,21 @@ namespace pdns {
         pbf_question.add_uint32(static_cast<protozero::pbf_tag_type>(QuestionField::qClass), qclass);
       }
 
+      void setMeta(const std::map<std::string, RecursorLua4::MetaValue>& meta)
+      {
+        for (const auto& m : meta) {
+          protozero::pbf_writer pbf_meta{d_message, static_cast<protozero::pbf_tag_type>(Field::meta)};
+          pbf_meta.add_string(static_cast<protozero::pbf_tag_type>(MetaField::key), m.first);
+          protozero::pbf_writer pbf_meta_value{pbf_meta, static_cast<protozero::pbf_tag_type>(MetaField::value)};
+          for (const auto& s: m.second.stringVal) {
+            pbf_meta_value.add_string(static_cast<protozero::pbf_tag_type>(MetaValueField::stringVal), s);
+          }
+          for (const auto& i: m.second.intVal) {
+            pbf_meta_value.add_uint64(static_cast<protozero::pbf_tag_type>(MetaValueField::intVal), i);
+          }
+        }
+      }
+
       void setEDNSSubnet(const Netmask& nm, uint8_t mask)
       {
         encodeNetmask(static_cast<protozero::pbf_tag_type>(Field::originalRequestorSubnet), nm, mask);