Rec: Store the RPZ policies in an unordered_map instead of a map

This speeds up the loading of huge zones a bit (~20%) while also nicely
improving the lookup speed.
Also adds a `zoneSizeHint` parameter to `rpzFile()` and `rpzMaster()` to be
able to reserve space before loading the zone, to prevent reallocation
and rehashing when possible.

diff --git a/docs/markdown/recursor/settings.md b/docs/markdown/recursor/settings.md
index 1a039b4b7ad7878b7295935cea738c9c56489cd9..22b6b54c91fe1271ae09cbf9bc76f59a577c897d 100644 (file)
--- a/docs/markdown/recursor/settings.md
+++ b/docs/markdown/recursor/settings.md
@@ -508,6 +508,7 @@ Settings for `rpzFile` and `rpzMaster` can contain:
 * defcontent = CNAME field to return in case of defpol=Policy.Custom
 * defttl = the TTL of the CNAME field to be synthesized. The default is to use the zone's TTL
 * policyName = the name logged as 'appliedPolicy' in protobuf messages when this policy is applied
+* zoneSizeHint = an indication of the number of expected entries in the zone, speeding up the loading of huge zones by reserving space in advance
 
 In addition to those, `rpzMaster` accepts:
 

diff --git a/pdns/filterpo.cc b/pdns/filterpo.cc
index 4bcfab854c9524656a9cca0fcccd3ad088b03b50..88e5320a8206e8adc62d066da9d4b3b101078ae5 100644 (file)
--- a/pdns/filterpo.cc
+++ b/pdns/filterpo.cc
@@ -28,10 +28,8 @@ DNSFilterEngine::DNSFilterEngine()
 {
 }
 
-bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
+static bool findNamedPolicy(const std::unordered_map<DNSName, DNSFilterEngine::Policy>& polmap, const DNSName& qname, DNSFilterEngine::Policy& pol)
 {
-  DNSName s(qname);
-
   /* for www.powerdns.com, we need to check:
      www.powerdns.com.
        *.powerdns.com.
@@ -39,14 +37,15 @@ bool findNamedPolicy(const map<DNSName, DNSFilterEngine::Policy>& polmap, const
                     *.
    */
 
-  map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
-  iter = polmap.find(s);
+  std::unordered_map<DNSName, DNSFilterEngine::Policy>::const_iterator iter;
+  iter = polmap.find(qname);
 
   if(iter != polmap.end()) {
     pol=iter->second;
     return true;
   }
 
+  DNSName s(qname);
   while(s.chopOff()){
     iter = polmap.find(g_wildcarddnsname+s);
     if(iter != polmap.end()) {

diff --git a/pdns/filterpo.hh b/pdns/filterpo.hh
index 283ea5bec71d1b9118876701b9cb671020f880bd..018cb7d77f1db004cc17377b5ea361502e2e43df 100644 (file)
--- a/pdns/filterpo.hh
+++ b/pdns/filterpo.hh
@@ -83,6 +83,10 @@ public:
   DNSFilterEngine();
   void clear();
   void clear(size_t zone);
+  void reserve(size_t zone, size_t entriesCount) {
+    assureZones(zone);
+    d_zones[zone].qpolName.reserve(entriesCount);
+  }
   void addClientTrigger(const Netmask& nm, Policy pol, size_t zone);
   void addQNameTrigger(const DNSName& nm, Policy pol, size_t zone);
   void addNSTrigger(const DNSName& dn, Policy pol, size_t zone);
@@ -112,13 +116,12 @@ public:
 private:
   void assureZones(size_t zone);
   struct Zone {
-    std::map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
+    std::unordered_map<DNSName, Policy> qpolName;   // QNAME trigger (RPZ)
     NetmaskTree<Policy> qpolAddr;         // Source address
-    std::map<DNSName, Policy> propolName; // NSDNAME (RPZ)
+    std::unordered_map<DNSName, Policy> propolName; // NSDNAME (RPZ)
     NetmaskTree<Policy> propolNSAddr;     // NSIP (RPZ)
     NetmaskTree<Policy> postpolAddr;      // IP trigger (RPZ)
     std::shared_ptr<std::string> name;
   };
   vector<Zone> d_zones;
-
 };

diff --git a/pdns/rec-lua-conf.cc b/pdns/rec-lua-conf.cc
index 5b7052a5e016758ed699fabd6f0be29482d6dcac..f06575bd57763c71517342dd6b5fa08884d647bf 100644 (file)
--- a/pdns/rec-lua-conf.cc
+++ b/pdns/rec-lua-conf.cc
@@ -93,6 +93,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       try {
        boost::optional<DNSFilterEngine::Policy> defpol;
        std::string polName("rpzFile");
+       const size_t zoneIdx = lci.dfe.size();
        if(options) {
          auto& have = *options;
          if(have.count("policyName")) {
@@ -116,8 +117,10 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
                defpol->d_ttl = -1; // get it from the zone
            }
          }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
        }
-        const size_t zoneIdx = lci.dfe.size();
         theL()<<Logger::Warning<<"Loading RPZ from file '"<<filename<<"'"<<endl;
         lci.dfe.setPolicyName(zoneIdx, polName);
         loadRPZFromFile(filename, lci.dfe, defpol, zoneIdx);
@@ -136,7 +139,8 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
         int refresh=0;
        std::string polName;
        size_t maxReceivedXFRMBytes = 0;
-        ComboAddress localAddress;
+       ComboAddress localAddress;
+       const size_t zoneIdx = lci.dfe.size();
        if(options) {
          auto& have = *options;
           polName = zone_;
@@ -161,6 +165,9 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
                defpol->d_ttl = -1; // get it from the zone
            }
          }
+          if(have.count("zoneSizeHint")) {
+            lci.dfe.reserve(zoneIdx, static_cast<size_t>(boost::get<int>(constGet(have, "zoneSizeHint"))));
+          }
          if(have.count("tsigname")) {
             tt.name=DNSName(toLower(boost::get<string>(constGet(have, "tsigname"))));
             tt.algo=DNSName(toLower(boost::get<string>(constGet(have, "tsigalgo"))));
@@ -182,7 +189,6 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
           // We were passed a localAddress, check if its AF matches the master's
           throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
        DNSName zone(zone_);
-        const size_t zoneIdx = lci.dfe.size();
         lci.dfe.setPolicyName(zoneIdx, polName);
 
         if (!checkOnly) {