Rec: Store the RPZ policies in an unordered_map instead of a map

This speeds up the loading of huge zones a bit (~20%) while also nicely
improving the lookup speed.
Also adds a `zoneSizeHint` parameter to `rpzFile()` and `rpzMaster()` to be
able to reserve space before loading the zone, to prevent reallocation
and rehashing when possible.

(cherry picked from commit a2d0450ec9fa958308fe0c40499ce28228bb3f00)

diff --git a/docs/markdown/recursor/settings.md b/docs/markdown/recursor/settings.md
index 65064058d91b36d1eba56b6e2def396dc29b427d..495bfcdfd2c64adf80e2f9150d6800cc30108710 100644 (file)
--- a/docs/markdown/recursor/settings.md
+++ b/docs/markdown/recursor/settings.md
@@ -491,6 +491,7 @@ Settings for `rpzFile` and `rpzMaster` can contain:
 * defcontent = CNAME field to return in case of defpol=Policy.Custom
 * defttl = the TTL of the CNAME field to be synthesized. The default is to use the zone's TTL
 * policyName = the name logged as 'appliedPolicy' in protobuf messages when this policy is applied
+* zoneSizeHint = an indication of the number of expected entries in the zone, speeding up the loading of huge zones by reserving space in advance
 
 In addition to those, `rpzMaster` accepts:
 

diff --git a/pdns/filterpo.cc b/pdns/filterpo.cc
index d32e8c150c7b80a6919901503f53604ec80a9c16..509418208bbaca9d0a62a0b9360a3c66592f0a18 100644 (file)
--- a/pdns/filterpo.cc
+++ b/pdns/filterpo.cc
@@ -28,10 +28,8 @@ DNSFilterEngine::DNSFilterEngine()
 {
 }
 
-bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
+static bool findNamedPolicy(const std::unordered_map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
 {
-  DNSName s(qname);
-
   /* for www.powerdns.com, we need to check:
      www.powerdns.com.
        *.powerdns.com.
@@ -39,14 +37,15 @@ bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const
                     *.
    */
 
-  map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
-  iter = polmap.find(s);
+  std::unordered_map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
+  iter = polmap.find(qname);
 
   if(iter != polmap.end()) {
     pol=iter->second;
     return true;
   }
 
+  DNSName s(qname);
   while(s.chopOff()){
     iter = polmap.find(DNSName("*")+s);
     if(iter != polmap.end()) {

diff --git a/pdns/filterpo.hh b/pdns/filterpo.hh
index 283ea5bec71d1b9118876701b9cb671020f880bd..018cb7d77f1db004cc17377b5ea361502e2e43df 100644 (file)
--- a/pdns/filterpo.hh
+++ b/pdns/filterpo.hh
@@ -83,6 +83,10 @@ public:
   DNSFilterEngine();
   void clear();
   void clear(size_t zone);
+  void reserve(size_t zone, size_t entriesCount) {
+    assureZones(zone);
+    d_zones[zone].qpolName.reserve(entriesCount);
+  }
   void addClientTrigger(const Netmask& nm, Policy pol, size_t zone);
   void addQNameTrigger(const DNSName& nm, Policy pol, size_t zone);
   void addNSTrigger(const DNSName& dn, Policy pol, size_t zone);
@@ -112,13 +116,12 @@ public:
 private:
   void assureZones(size_t zone);
   struct Zone {
-    std::map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
+    std::unordered_map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
     NetmaskTree<Policy> qpolAddr;         // Source address
-    std::map<DNSName, Policy> propolName; // NSDNAME (RPZ)
+    std::unordered_map<DNSName, Policy> propolName; // NSDNAME (RPZ)
     NetmaskTree<Policy> propolNSAddr;     // NSIP (RPZ)
     NetmaskTree<Policy> postpolAddr;      // IP trigger (RPZ)
     std::shared_ptr<std::string> name;
   };
   vector<Zone> d_zones;
-
 };

diff --git a/pdns/rec-lua-conf.cc b/pdns/rec-lua-conf.cc
index 3f1ad6616da3e5f70c4b9b478f7f157a1ed50fac..a97a61b38fe2a9fb30786e54779b9eb9cb4b8c34 100644 (file)
--- a/pdns/rec-lua-conf.cc
+++ b/pdns/rec-lua-conf.cc
@@ -92,6 +92,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       try {
        boost::optional<DNSFilterEngine::Policy> defpol;
        std::string polName("rpzFile");
+       const size_t zoneIdx = lci.dfe.size();
        if(options) {
          auto& have = *options;
          if(have.count("policyName")) {
@@ -115,8 +116,10 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
                defpol->d_ttl = -1; // get it from the zone
            }
          }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
        }
-        const size_t zoneIdx = lci.dfe.size();
         theL()<<Logger::Warning<<"Loading RPZ from file '"<<filename<<"'"<<endl;
         lci.dfe.setPolicyName(zoneIdx, polName);
         loadRPZFromFile(filename, lci.dfe, defpol, zoneIdx);
@@ -135,7 +138,8 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
         int refresh=0;
        std::string polName;
        size_t maxReceivedXFRMBytes = 0;
-        ComboAddress localAddress;
+       ComboAddress localAddress;
+       const size_t zoneIdx = lci.dfe.size();
        if(options) {
          auto& have = *options;
           polName = zone_;
@@ -160,6 +164,9 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
                defpol->d_ttl = -1; // get it from the zone
            }
          }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
          if(have.count("tsigname")) {
             tt.name=DNSName(toLower(boost::get<string>(constGet(have, "tsigname"))));
             tt.algo=DNSName(toLower(boost::get<string>(constGet(have, "tsigalgo"))));
@@ -181,7 +188,6 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
           // We were passed a localAddress, check if its AF matches the master's
           throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
        DNSName zone(zone_);
-        const size_t zoneIdx = lci.dfe.size();
         lci.dfe.setPolicyName(zoneIdx, polName);
 
         if (!checkOnly) {