From: bert hubert <bert.hubert@netherlabs.nl>
Date: Fri, 30 Oct 2015 19:36:45 +0000 (+0100)
Subject: make all caches that can be cleaned in the recursor canonical ordered so we can do... 
X-Git-Tag: dnsdist-1.0.0-alpha1~230^2~10
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=86f3ca51b1792d418076c8f58cdfbd391c61a964;p=pdns

make all caches that can be cleaned in the recursor canonical ordered so we can do subtree cleaning.
Before this commit, packet cache cleaning actually did not work in 4.x!
---

diff --git a/pdns/dns.cc b/pdns/dns.cc
index 46da40b1c..b52bb8ae9 100644
--- a/pdns/dns.cc
+++ b/pdns/dns.cc
@@ -66,53 +66,6 @@ private:
   const unsigned int d_length;
 };
 
-//! compares two dns packets, skipping the header, but including the question and the qtype
-bool dnspacketLessThan(const std::string& a, const std::string& b)
-{
-  if(a.length() <= 12 || b.length() <= 12) 
-    return a.length() < b.length();
-//    throw runtime_error("Error parsing question in dnspacket comparison: packet too short");
-    
-  // we find: 3www4ds9a2nl0XXYY, where XX and YY are each 2 bytes describing class and type
-  
-  BoundsCheckingPointer aSafe(a), bSafe(b);
-  int aPos=12, bPos=12;
-  
-  unsigned char aLabelLen, bLabelLen;
-
-  do {  
-    aLabelLen = aSafe[aPos++]; bLabelLen = bSafe[bPos++];
-    // cerr<<"aLabelLen: "<<(int)aLabelLen<<", bLabelLen: "<< (int)bLabelLen<<endl;
-    
-    int result=0;
-    unsigned int n;
-    for(n = 0; n < aLabelLen && n < bLabelLen; ++n) 
-      if((result = dns_tolower(aSafe[aPos + n]) - dns_tolower(bSafe[bPos +n])))
-        break;
-    // cerr<<"Done loop, result="<<result<<", n = "<<n<<", aLabelLen="<<aLabelLen<<", bLabelLen="<<bLabelLen<<endl;
-    if(result < 0)
-      return true;
-    if(result > 0)
-      return false;
-    if(n == aLabelLen && n != bLabelLen)
-      return true; // a is shorter, shortest wins
-    if(n != aLabelLen && n == bLabelLen)
-      return false; // a is longer
-    //~ cerr<<"did not return\n";
-    aPos += aLabelLen; bPos += bLabelLen;
-  } while(aLabelLen && bLabelLen);
-  
-  if(aLabelLen || bLabelLen) //
-    throw runtime_error("Error in label comparison routine, should not happen");
-      
-  uint16_t aQtype = aSafe[aPos]*256 + aSafe[aPos + 1];
-  uint16_t bQtype = bSafe[bPos]*256 + bSafe[bPos + 1];
-  
-  uint16_t aQclass = aSafe[aPos+2]*256 + aSafe[aPos + 3];
-  uint16_t bQclass = bSafe[bPos+2]*256 + bSafe[bPos + 3];
-  
-  return boost::tie(aQtype, aQclass) < boost::tie(bQtype, bQclass);
-}
 
 // goal is to hash based purely on the question name, and turn error into 'default'
 uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init)
diff --git a/pdns/dns.hh b/pdns/dns.hh
index 7db16162a..b013beb24 100644
--- a/pdns/dns.hh
+++ b/pdns/dns.hh
@@ -222,7 +222,88 @@ struct dnsheader {
 extern time_t s_starttime;
 
 uint32_t hashQuestion(const char* packet, uint16_t len, uint32_t init);
-bool dnspacketLessThan(const std::string& a, const std::string& b);
+
+//! compares two dns packets in canonical order, skipping the header, but including the question and the qtype
+inline bool dnspacketLessThan(const std::string& a, const std::string& b)
+{
+  /* BEFORE YOU ATTEMPT TO MERGE THIS WITH DNSNAME::CANONICALCOMPARE! 
+     Please note that that code is subtly different, and for example only has
+     to deal with a string of labels, and not a trailing packet. Also, the comparison
+     rules are different since we also have to take into account qname and qtype.
+     So just grin and bear it.
+   */
+
+  if(a.length() <= 12 || b.length() <= 12) 
+    return a.length() < b.length();
+
+  uint8_t ourpos[64], rhspos[64];
+  uint8_t ourcount=0, rhscount=0;
+  //cout<<"Asked to compare "<<toString()<<" to "<<rhs.toString()<<endl;
+  const unsigned char* p;
+  for(p = (const unsigned char*)a.c_str()+12; p < (const unsigned char*)a.c_str() + a.size() && *p && ourcount < sizeof(ourpos); p+=*p+1)
+    ourpos[ourcount++]=(p-(const unsigned char*)a.c_str());
+  if(p>=(const unsigned char*)a.c_str() + a.size())
+    return true;
+
+  uint16_t aQtype = *(p+1)*256 + *(p+2);
+  uint16_t aQclass =*(p+3)*256 + *(p+4);
+
+  for(p = (const unsigned char*)b.c_str()+12; p < (const unsigned char*)b.c_str() + b.size() && *p && rhscount < sizeof(rhspos); p+=*p+1)
+    rhspos[rhscount++]=(p-(const unsigned char*)b.c_str());
+
+  if(p>=(const unsigned char*)b.c_str() + b.size())
+    return false;
+
+  uint16_t bQtype = *(p+1)*256 + *(p+2);
+  uint16_t bQclass =*(p+3)*256 + *(p+4);
+
+  if(ourcount == sizeof(ourpos) || rhscount==sizeof(rhspos)) {
+    DNSName aname(a.c_str(), a.size(), 12, false, &aQtype, &aQclass);
+    DNSName bname(b.c_str(), b.size(), 12, false, &bQtype, &bQclass);
+
+    if(aname.slowCanonCompare(bname))
+      return true;
+    if(aname!=bname)
+      return false;
+    return boost::tie(aQtype, aQclass) < boost::tie(bQtype, bQclass);
+  }
+  for(;;) {
+    if(ourcount == 0 && rhscount != 0)
+      return true;
+    if(ourcount == 0 && rhscount == 0)
+      break;
+    if(ourcount !=0 && rhscount == 0)
+      return false;
+    ourcount--;
+    rhscount--;
+
+    bool res=std::lexicographical_compare(
+					  a.c_str() + ourpos[ourcount] + 1, 
+					  a.c_str() + ourpos[ourcount] + 1 + *(a.c_str() + ourpos[ourcount]),
+					  b.c_str() + rhspos[rhscount] + 1, 
+					  b.c_str() + rhspos[rhscount] + 1 + *(b.c_str() + rhspos[rhscount]),
+					  [](const char& a, const char& b) {
+					    return dns2_tolower(a) < dns2_tolower(b); 
+					  });
+    
+    //    cout<<"Forward: "<<res<<endl;
+    if(res)
+      return true;
+
+    res=std::lexicographical_compare(	  b.c_str() + rhspos[rhscount] + 1, 
+					  b.c_str() + rhspos[rhscount] + 1 + *(b.c_str() + rhspos[rhscount]),
+					  a.c_str() + ourpos[ourcount] + 1, 
+					  a.c_str() + ourpos[ourcount] + 1 + *(a.c_str() + ourpos[ourcount]),
+					  [](const char& a, const char& b) {
+					    return dns2_tolower(a) < dns2_tolower(b); 
+					  });
+    //    cout<<"Reverse: "<<res<<endl;
+    if(res)
+      return false;
+  }
+  
+  return boost::tie(aQtype, aQclass) < boost::tie(bQtype, bQclass);
+}
 
 
 /** for use by DNSPacket, converts a SOAData class to a ascii line again */
diff --git a/pdns/rec_channel_rec.cc b/pdns/rec_channel_rec.cc
index 10d0a19d0..ca922a9bc 100644
--- a/pdns/rec_channel_rec.cc
+++ b/pdns/rec_channel_rec.cc
@@ -227,32 +227,57 @@ string doDumpEDNSStatus(T begin, T end)
   return "done\n";
 }
 
-uint64_t* pleaseWipeCache(const DNSName& canon)
+uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree)
 {
-  // clear packet cache too
-  return new uint64_t(t_RC->doWipeCache(canon) + t_packetCache->doWipePacketCache(canon));
+  return new uint64_t(t_RC->doWipeCache(canon, subtree));
 }
 
+uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree)
+{
+  return new uint64_t(t_packetCache->doWipePacketCache(canon,0xffff, subtree));
+}
 
-uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon)
+
+uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree)
 {
-  uint64_t res = t_sstorage->negcache.count(tie(canon));
-  pair<SyncRes::negcache_t::iterator, SyncRes::negcache_t::iterator> range=t_sstorage->negcache.equal_range(tie(canon));
-  t_sstorage->negcache.erase(range.first, range.second);
-  return new uint64_t(res);
+  if(!subtree) {
+    uint64_t res = t_sstorage->negcache.count(tie(canon));
+    auto range=t_sstorage->negcache.equal_range(tie(canon));
+    t_sstorage->negcache.erase(range.first, range.second);
+    return new uint64_t(res);
+  }
+  else {
+    unsigned int erased=0;
+    for(auto iter = t_sstorage->negcache.lower_bound(tie(canon)); iter != t_sstorage->negcache.end(); ) {
+      if(!iter->d_qname.isPartOf(canon))
+	break;
+      t_sstorage->negcache.erase(iter++);
+      erased++;
+    }
+    return new uint64_t(erased);
+  }
 }
 
 template<typename T>
 string doWipeCache(T begin, T end)
 {
-  int count=0, countNeg=0;
+  int count=0, pcount=0, countNeg=0;
   for(T i=begin; i != end; ++i) {
-    DNSName canon=DNSName(*i);
-    count+= broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, canon));
-    countNeg+=broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeAndCountNegCache, canon));
+    DNSName canon;
+    bool subtree=false;
+    if(boost::ends_with(*i, "$")) {
+      canon=DNSName(i->substr(0, i->size()-1));
+      subtree=true;
+    }
+    else 
+      canon=DNSName(*i);
+    
+    count+= broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, canon, subtree));
+    pcount+= broadcastAccFunction<uint64_t>(boost::bind(pleaseWipePacketCache, canon, subtree));
+    countNeg+=broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeAndCountNegCache, canon, subtree));
   }
 
-  return "wiped "+lexical_cast<string>(count)+" records, "+lexical_cast<string>(countNeg)+" negative records\n";
+  return "wiped "+std::to_string(count)+" records, "+std::to_string(countNeg)+" negative records, "+std::to_string(pcount)+" packets\n";
 }
 
 template<typename T>
@@ -281,7 +306,7 @@ string setMinimumTTL(T begin, T end)
   if(end-begin != 1) 
     return "Need to supply new minimum TTL number\n";
   SyncRes::s_minimumTTL = atoi(begin->c_str());
-  return "New minimum TTL: " + lexical_cast<string>(SyncRes::s_minimumTTL) + "\n";
+  return "New minimum TTL: " + std::to_string(SyncRes::s_minimumTTL) + "\n";
 }
 
 
diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc
index 8274a5aee..a102ca85f 100644
--- a/pdns/recpacketcache.cc
+++ b/pdns/recpacketcache.cc
@@ -15,7 +15,7 @@ RecursorPacketCache::RecursorPacketCache()
   d_hits = d_misses = 0;
 }
 
-int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype)
+int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool subtree)
 {
   vector<uint8_t> packet;
   DNSPacketWriter pw(packet, name, 0);
@@ -33,9 +33,17 @@ int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype)
     uint16_t t;
 
     DNSName found(iter->d_packet.c_str(), iter->d_packet.size(), 12, false, &t);
-    if(found==name) {   // this is case insensitive
-      break;
+    //    cout<<"At record "<<found<<" while searching for "<<name<<", subtree= "<<subtree<<endl;
+    if(subtree) {
+      if(!found.isPartOf(name)) {   // this is case insensitive
+	break;
+      }
+    }
+    else {
+      if(found != name)
+	break;
     }
+
     if(t==qtype || qtype==0xffff) {
       iter=d_packetCache.erase(iter);
       count++;
@@ -43,6 +51,7 @@ int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype)
     else
       ++iter;
   }
+  //  cout<<"Wiped "<<count<<" packets from cache"<<endl;
   return count;
 }
 
diff --git a/pdns/recpacketcache.hh b/pdns/recpacketcache.hh
index 53f5554e4..2cfaff4fc 100644
--- a/pdns/recpacketcache.hh
+++ b/pdns/recpacketcache.hh
@@ -22,7 +22,7 @@ public:
   bool getResponsePacket(const std::string& queryPacket, time_t now, std::string* responsePacket, uint32_t* age);
   void insertResponsePacket(const std::string& responsePacket, time_t now, uint32_t ttd);
   void doPruneTo(unsigned int maxSize=250000);
-  int doWipePacketCache(const DNSName& name, uint16_t qtype=0xffff);
+  int doWipePacketCache(const DNSName& name, uint16_t qtype=0xffff, bool subtree=false);
   
   void prune();
   uint64_t d_hits, d_misses;
diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc
index 6656f9ac5..8ec82fc66 100644
--- a/pdns/recursor_cache.cc
+++ b/pdns/recursor_cache.cc
@@ -197,19 +197,33 @@ void MemRecursorCache::replace(time_t now, const DNSName &qname, const QType& qt
   d_cache.replace(stored, ce);
 }
 
-int MemRecursorCache::doWipeCache(const DNSName& name, uint16_t qtype)
+int MemRecursorCache::doWipeCache(const DNSName& name, bool sub, uint16_t qtype)
 {
   int count=0;
   d_cachecachevalid=false;
   pair<cache_t::iterator, cache_t::iterator> range;
-  if(qtype==0xffff)
-    range=d_cache.equal_range(tie(name));
-  else
-    range=d_cache.equal_range(tie(name, qtype));
 
-  for(cache_t::const_iterator i=range.first; i != range.second; ) {
-    count++;
-    d_cache.erase(i++);
+  if(!sub) {
+    if(qtype==0xffff)
+      range=d_cache.equal_range(tie(name));
+    else
+      range=d_cache.equal_range(tie(name, qtype));
+    for(cache_t::const_iterator i=range.first; i != range.second; ) {
+      count++;
+      d_cache.erase(i++);
+    }
+  }
+  else {
+    for(auto iter = d_cache.lower_bound(tie(name)); iter != d_cache.end(); ) {
+      if(!iter->d_qname.isPartOf(name))
+	break;
+      if(iter->d_qtype == qtype || qtype == 0xffff) {
+	count++;
+	d_cache.erase(iter++);
+      }
+      else 
+	iter++;
+    }
   }
   return count;
 }
@@ -277,12 +291,12 @@ uint64_t MemRecursorCache::doDump(int fd)
     return 0;
   }
   fprintf(fp, "; main record cache dump from thread follows\n;\n");
-  typedef cache_t::nth_index<1>::type sequence_t;
-  sequence_t& sidx=d_cache.get<1>();
+
+  auto& sidx=d_cache.get<0>();
 
   uint64_t count=0;
   time_t now=time(0);
-  for(sequence_t::const_iterator i=sidx.begin(); i != sidx.end(); ++i) {
+  for(auto i=sidx.cbegin(); i != sidx.cend(); ++i) {
     for(auto j=i->d_records.cbegin(); j != i->d_records.cend(); ++j) {
       count++;
       try {
diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh
index c814d0f18..020787391 100644
--- a/pdns/recursor_cache.hh
+++ b/pdns/recursor_cache.hh
@@ -40,7 +40,7 @@ public:
   uint64_t doDump(int fd);
   uint64_t doDumpNSSpeeds(int fd);
 
-  int doWipeCache(const DNSName& name, uint16_t qtype=0xffff);
+  int doWipeCache(const DNSName& name, bool sub, uint16_t qtype=0xffff);
   bool doAgeCache(time_t now, const DNSName& name, uint16_t qtype, int32_t newTTL);
   uint64_t cacheHits, cacheMisses;
 
@@ -76,7 +76,7 @@ private:
                         member<CacheEntry,DNSName,&CacheEntry::d_qname>,
                         member<CacheEntry,uint16_t,&CacheEntry::d_qtype>
                       >,
-                      composite_key_compare<std::less<DNSName>, std::less<uint16_t> >
+                      composite_key_compare<CanonDNSNameCompare, std::less<uint16_t> >
                 >,
                sequenced<>
                >
diff --git a/pdns/reczones.cc b/pdns/reczones.cc
index 5c14a4abd..ed4fcbf85 100644
--- a/pdns/reczones.cc
+++ b/pdns/reczones.cc
@@ -54,6 +54,7 @@ void primeHints(void)
     };
     
     DNSRecord arr, aaaarr, nsrr;
+    nsrr.d_name=DNSName(".");
     arr.d_type=QType::A;
     aaaarr.d_type=QType::AAAA;
     nsrr.d_type=QType::NS;
@@ -254,7 +255,7 @@ string reloadAuthAndForwards()
   
     for(SyncRes::domainmap_t::const_iterator i = t_sstorage->domainmap->begin(); i != t_sstorage->domainmap->end(); ++i) {
       for(SyncRes::AuthDomain::records_t::const_iterator j = i->second.d_records.begin(); j != i->second.d_records.end(); ++j) 
-        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, j->d_name));
+        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, j->d_name, false));
     }
 
     string configname=::arg()["config-dir"]+"/recursor.conf";
@@ -294,12 +295,11 @@ string reloadAuthAndForwards()
     
     // purge again - new zones need to blank out the cache
     for(SyncRes::domainmap_t::const_iterator i = newDomainMap->begin(); i != newDomainMap->end(); ++i) {
-      for(SyncRes::AuthDomain::records_t::const_iterator j = i->second.d_records.begin(); j != i->second.d_records.end(); ++j) 
-        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, j->d_name));
+        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, i->first, true));
+        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipePacketCache, i->first, true));
+        broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeAndCountNegCache, i->first, true));
     }
 
-    // this is pretty blunt
-    broadcastFunction(pleaseWipeNegCache);
     broadcastFunction(boost::bind(pleaseUseNewSDomainsMap, newDomainMap)); 
     delete original;
     return "ok\n";
diff --git a/pdns/syncres.hh b/pdns/syncres.hh
index 99a10256c..618240ae7 100644
--- a/pdns/syncres.hh
+++ b/pdns/syncres.hh
@@ -324,7 +324,7 @@ public:
                     member<NegCacheEntry, DNSName, &NegCacheEntry::d_name>,
                     member<NegCacheEntry, QType, &NegCacheEntry::d_qtype>
            >,
-           composite_key_compare<std::less<DNSName>, std::less<QType> >
+           composite_key_compare<CanonDNSNameCompare, std::less<QType> >
        >,
        sequenced<>
     >
@@ -651,7 +651,8 @@ uint64_t* pleaseGetConcurrentQueries();
 uint64_t* pleaseGetThrottleSize();
 uint64_t* pleaseGetPacketCacheHits();
 uint64_t* pleaseGetPacketCacheSize();
-uint64_t* pleaseWipeCache(const DNSName& canon);
-uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon);
+uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree=false);
+uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree);
+uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree=false);
 void doCarbonDump(void*);
 #endif
diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc
index 49da6d1cf..af39651c6 100644
--- a/pdns/ws-recursor.cc
+++ b/pdns/ws-recursor.cc
@@ -418,8 +418,8 @@ static void apiServerFlushCache(HttpRequest* req, HttpResponse* resp) {
     throw HttpMethodNotAllowedException();
 
   DNSName canon(req->getvars["domain"]);
-  int count = broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, canon));
-  count += broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeAndCountNegCache, canon));
+  int count = broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeCache, canon, false));
+  count += broadcastAccFunction<uint64_t>(boost::bind(pleaseWipeAndCountNegCache, canon, false));
   map<string, string> object;
   object["count"] = lexical_cast<string>(count);
   object["result"] = "Flushed cache.";