From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Mon, 20 Feb 2017 14:45:22 +0000 (+0100)
Subject: Rec: Store the RPZ policies in an unordered_map instead of a map
X-Git-Tag: rec-4.0.5-rc1~6^2
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F5165%2Fhead;p=thirdparty%2Fpdns.git

Rec: Store the RPZ policies in an unordered_map instead of a map

This speeds up the loading of huge zones a bit (~20%) while also nicely
improving the lookup speed.
Also adds a `zoneSizeHint` parameter to `rpzFile()` and `rpzMaster()` to be
able to reserve space before loading the zone, to prevent reallocation
and rehashing when possible.

(cherry picked from commit a2d0450ec9fa958308fe0c40499ce28228bb3f00)
---

diff --git a/docs/markdown/recursor/settings.md b/docs/markdown/recursor/settings.md
index 65064058d9..495bfcdfd2 100644
--- a/docs/markdown/recursor/settings.md
+++ b/docs/markdown/recursor/settings.md
@@ -491,6 +491,7 @@ Settings for `rpzFile` and `rpzMaster` can contain:
 * defcontent = CNAME field to return in case of defpol=Policy.Custom
 * defttl = the TTL of the CNAME field to be synthesized. The default is to use the zone's TTL
 * policyName = the name logged as 'appliedPolicy' in protobuf messages when this policy is applied
+* zoneSizeHint = an indication of the number of expected entries in the zone, speeding up the loading of huge zones by reserving space in advance
 
 In addition to those, `rpzMaster` accepts:
 
diff --git a/pdns/filterpo.cc b/pdns/filterpo.cc
index d32e8c150c..509418208b 100644
--- a/pdns/filterpo.cc
+++ b/pdns/filterpo.cc
@@ -28,10 +28,8 @@ DNSFilterEngine::DNSFilterEngine()
 {
 }
 
-bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
+static bool findNamedPolicy(const std::unordered_map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
 {
-  DNSName s(qname);
-
   /* for www.powerdns.com, we need to check:
      www.powerdns.com.
        *.powerdns.com.
@@ -39,14 +37,15 @@ bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const
                     *.
    */
 
-  map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
-  iter = polmap.find(s);
+  std::unordered_map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
+  iter = polmap.find(qname);
 
   if(iter != polmap.end()) {
     pol=iter->second;
     return true;
   }
 
+  DNSName s(qname);
   while(s.chopOff()){
     iter = polmap.find(DNSName("*")+s);
     if(iter != polmap.end()) {
diff --git a/pdns/filterpo.hh b/pdns/filterpo.hh
index 283ea5bec7..018cb7d77f 100644
--- a/pdns/filterpo.hh
+++ b/pdns/filterpo.hh
@@ -83,6 +83,10 @@ public:
   DNSFilterEngine();
   void clear();
   void clear(size_t zone);
+  void reserve(size_t zone, size_t entriesCount) {
+    assureZones(zone);
+    d_zones[zone].qpolName.reserve(entriesCount);
+  }
   void addClientTrigger(const Netmask& nm, Policy pol, size_t zone);
   void addQNameTrigger(const DNSName& nm, Policy pol, size_t zone);
   void addNSTrigger(const DNSName& dn, Policy pol, size_t zone);
@@ -112,13 +116,12 @@ public:
 private:
   void assureZones(size_t zone);
   struct Zone {
-    std::map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
+    std::unordered_map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
     NetmaskTree<Policy> qpolAddr;         // Source address
-    std::map<DNSName, Policy> propolName; // NSDNAME (RPZ)
+    std::unordered_map<DNSName, Policy> propolName; // NSDNAME (RPZ)
     NetmaskTree<Policy> propolNSAddr;     // NSIP (RPZ)
     NetmaskTree<Policy> postpolAddr;      // IP trigger (RPZ)
     std::shared_ptr<std::string> name;
   };
   vector<Zone> d_zones;
-
 };
diff --git a/pdns/rec-lua-conf.cc b/pdns/rec-lua-conf.cc
index 3f1ad6616d..a97a61b38f 100644
--- a/pdns/rec-lua-conf.cc
+++ b/pdns/rec-lua-conf.cc
@@ -92,6 +92,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       try {
 	boost::optional<DNSFilterEngine::Policy> defpol;
 	std::string polName("rpzFile");
+	const size_t zoneIdx = lci.dfe.size();
 	if(options) {
 	  auto& have = *options;
 	  if(have.count("policyName")) {
@@ -115,8 +116,10 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
 		defpol->d_ttl = -1; // get it from the zone
 	    }
 	  }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
 	}
-        const size_t zoneIdx = lci.dfe.size();
         theL()<<Logger::Warning<<"Loading RPZ from file '"<<filename<<"'"<<endl;
         lci.dfe.setPolicyName(zoneIdx, polName);
         loadRPZFromFile(filename, lci.dfe, defpol, zoneIdx);
@@ -135,7 +138,8 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
         int refresh=0;
 	std::string polName;
 	size_t maxReceivedXFRMBytes = 0;
-        ComboAddress localAddress;
+	ComboAddress localAddress;
+	const size_t zoneIdx = lci.dfe.size();
 	if(options) {
 	  auto& have = *options;
           polName = zone_;
@@ -160,6 +164,9 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
 		defpol->d_ttl = -1; // get it from the zone
 	    }
 	  }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
 	  if(have.count("tsigname")) {
             tt.name=DNSName(toLower(boost::get<string>(constGet(have, "tsigname"))));
             tt.algo=DNSName(toLower(boost::get<string>(constGet(have, "tsigalgo"))));
@@ -181,7 +188,6 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
           // We were passed a localAddress, check if its AF matches the master's
           throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
 	DNSName zone(zone_);
-        const size_t zoneIdx = lci.dfe.size();
         lci.dfe.setPolicyName(zoneIdx, polName);
 
         if (!checkOnly) {