fix pruning of negcache

specialize the common case where we see identical information coming in, and only update the ttd

git-svn-id: svn://svn.powerdns.com/pdns/trunk/pdns@535 d19b8d6e-7fed-0310-83ef-9ca221ded41b

diff --git a/pdns/Makefile.am b/pdns/Makefile.am
index cafd66c63d7d0bb9180cb96884bb035596abb8fa..ef40c71e2f4fb01d2236630f8313de4ce832f877 100644 (file)
--- a/pdns/Makefile.am
+++ b/pdns/Makefile.am
@@ -47,7 +47,8 @@ sdig_SOURCES=sdig.cc sstuff.hh dnsparser.cc dnsparser.hh dnsrecords.cc dnswriter
        logger.cc statbag.cc
 
 dnspbench_SOURCES=dnspbench.cc sstuff.hh dnsparser.cc dnsparser.hh dnsrecords.cc dnswriter.cc dnswriter.hh \
-       misc.cc misc.hh rcpgenerator.cc rcpgenerator.hh base64.cc base64.hh
+       misc.cc misc.hh rcpgenerator.cc rcpgenerator.hh base64.cc base64.hh unix_utility.cc logger.cc \
+       statbag.cc
 
 
 qgen_SOURCES=resolver.cc resolver.hh misc.cc unix_utility.cc qtype.cc \

diff --git a/pdns/dnspbench.cc b/pdns/dnspbench.cc
index 2056330041b92ae698406e757d8b1a3ae82276d9..8fc69942a20f509455abf83a94d714dd75089981 100644 (file)
--- a/pdns/dnspbench.cc
+++ b/pdns/dnspbench.cc
@@ -3,22 +3,32 @@
 #include "misc.hh"
 #include "dnswriter.hh"
 #include "dnsrecords.hh"
+#include "logger.hh"
+#include "statbag.hh"
 
+Logger L("dnspbench");
+StatBag S;
 
 
 int main(int argc, char** argv)
 try
 {
-  reportAllTypes();
-
-  cerr<<"sizeof(optString): "<<sizeof(struct optString)<<endl;
-
-  optString os("hallo!");
+  dnsheader dnsheader;
+  dnsheader.qdcount=htons(1);
+  dnsheader.ancount=htons(1);
+  Socket s(InterNetwork, Datagram);
+  string spacket;
+  char* p=(char*)&dnsheader;
+  spacket.assign(p, p+sizeof(dnsheader));
+  IPEndpoint rem("127.0.0.1",5300);
+  s.sendTo(spacket, rem);
+  
+  return 0;
 
-  cerr<<"optString: '"<<(string)os<<"'\n";
+  reportAllTypes();
 
   vector<uint8_t> packet;
-  
+
   uint16_t type=DNSRecordContent::TypeToNumber(argv[2]);
 
   DNSRecordContent* drc=DNSRecordContent::mastermake(type, 1, argv[3]);

diff --git a/pdns/dnsreplay-mindex.cc b/pdns/dnsreplay-mindex.cc
index 7e39920c65e9126270baf69848ee35b7558056a9..4db7479bb83ca13cd8862447314dd40ba0226095 100644 (file)
--- a/pdns/dnsreplay-mindex.cc
+++ b/pdns/dnsreplay-mindex.cc
@@ -511,7 +511,7 @@ void sendPacketFromPR(PcapPacketReader& pr, const IPEndpoint& remote)
          useconds+=1000000;
        }
        
-       double factor=20;
+       double factor=10;
        
        seconds/=factor;
        useconds/=factor;
@@ -623,7 +623,7 @@ try
 
     gettimeofday(&now, 0);
 
-    mental_time= mental_time + 2*(now-then);
+    mental_time= mental_time + 1*(now-then);
   }
  out:;
 }

diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc
index 5ffbd877811beeb4f13750720dff9dfb347f6d9c..cfa6c20462b930d1eb7b14d3cae3a14e8d3d7b17 100644 (file)
--- a/pdns/pdns_recursor.cc
+++ b/pdns/pdns_recursor.cc
@@ -500,8 +500,9 @@ void doStats(void)
 {
   if(qcounter) {
     L<<Logger::Error<<"stats: "<<qcounter<<" questions, "<<RC.size()<<" cache entries, "<<SyncRes::s_negcache.size()<<" negative entries, "
-     <<(int)((RC.cacheHits*100.0)/(RC.cacheHits+RC.cacheMisses))<<"% cache hits";
-    L<<Logger::Error<<", outpacket/query ratio "<<(int)(SyncRes::s_outqueries*100.0/SyncRes::s_queries)<<"%";
+     <<(int)((RC.cacheHits*100.0)/(RC.cacheHits+RC.cacheMisses))<<"% cache hits"<<endl;
+    L<<Logger::Error<<"stats: throttle map: "<<SyncRes::s_throttle.size()<<", ns speeds: "<<SyncRes::s_nsSpeeds.size()<<endl;
+    L<<Logger::Error<<"stats: outpacket/query ratio "<<(int)(SyncRes::s_outqueries*100.0/SyncRes::s_queries)<<"%";
     L<<Logger::Error<<", "<<(int)(SyncRes::s_throttledqueries*100.0/(SyncRes::s_outqueries+SyncRes::s_throttledqueries))<<"% throttled, "
      <<SyncRes::s_nodelegated<<" no-delegation drops"<<endl;
     L<<Logger::Error<<"stats: "<<SyncRes::s_tcpoutqueries<<" outgoing tcp connections, "<<MT->numProcesses()<<" queries running, "<<SyncRes::s_outgoingtimeouts<<" outgoing timeouts"<<endl;
@@ -518,6 +519,15 @@ void houseKeeping(void *)
   time_t now=time(0);
   if(now - last_prune > 60) { 
     RC.doPrune();
+    int pruned=0;
+    for(map<string, NegCacheEntry>::iterator i = SyncRes::s_negcache.begin(); i != SyncRes::s_negcache.end();) 
+      if(i->second.ttd > now) {
+       SyncRes::s_negcache.erase(i++);
+       pruned++;
+      }
+      else
+       ++i;
+    //    cerr<<"Pruned "<<pruned<<" records, left "<<SyncRes::s_negcache.size()<<"\n";
     last_prune=time(0);
   }
   if(now - last_stat>1800) { 

diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc
index 1654262af7a4a83e52c05d35ce264800ba46cec2..f176051d31924ad73a51162ab64e0cc5f7238c6c 100644 (file)
--- a/pdns/recursor_cache.cc
+++ b/pdns/recursor_cache.cc
@@ -58,21 +58,46 @@ int MemRecursorCache::get(time_t now, const string &qname, const QType& qt, set<
   return -1;
 }
   
+/* the code below is rather tricky - it basically replaces the stuff cached for qname by content, but it is special
+   cased for when inserting identical records with only differing ttls, in which case the entry is not
+   touched, but only given a new ttd */
 void MemRecursorCache::replace(const string &qname, const QType& qt,  const set<DNSResourceRecord>& content)
 {
   set<StoredRecord>& stored=d_cache[toLowerCanonic(qname)+"|"+qt.getName()];
 
-  for(set<StoredRecord>::iterator k=stored.begin();k!=stored.end();++k) 
-    k->d_string.prune();
+  set<StoredRecord>::iterator k;
+  typedef vector<set<StoredRecord>::iterator> touched_t;
+  touched_t touched;
 
-  stored.clear();
+  // walk through new content, encode it as new
+  StoredRecord dr;
 
   for(set<DNSResourceRecord>::const_iterator i=content.begin(); i != content.end(); ++i) {
-    StoredRecord dr;
     dr.d_ttd=i->ttl;
     dr.d_string=DNSRR2String(*i);
-    stored.insert(dr);
-    //cerr<<"Storing: "<< toLowerCanonic(qname)+"|"+qt.getName() << " <=> '"<<i->content<<"', ttd="<<i->ttl<<endl;
+    k=stored.find(dr);
+    if(k!=stored.end()) {           // was it there already?
+      //      cerr<<"updating record '"<<qname<<"' -> '"<<i->content<<"'\n";
+      k->d_ttd=i->ttl;              // update ttl
+      touched.push_back(k);         // note that this record is here to stay
+    }
+    else {
+      //      cerr<<"inserting record '"<<qname<<"' -> '"<<i->content<<"'\n";
+      touched.push_back(stored.insert(dr).first);  // same thing
+    }
+  }
+
+  for(k=stored.begin(); k!=stored.end(); ) {                     // walk over the stored set of records
+    touched_t::const_iterator j;                                
+    for(j=touched.begin(); j!=touched.end() && *j != k ; ++j);   // walk over touched iterators
+    if(j==touched.end()) {                                       // this record was not there
+      //      DNSResourceRecord rr=String2DNSRR(qname, qt,  k->d_string, 0); 
+      //      cerr<<"removing from record '"<<qname<<"' '"<<rr.content<<"'\n";
+      k->d_string.prune();                                      
+      stored.erase(k++);                                         // cleanup
+    }
+    else
+      ++k;
   }
 }
   

diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh
index f4b4a897f065b74756f125e17bac0e6647d2fa83..9ad8385ed50e692a07dcd7043327b820856fcfc4 100644 (file)
--- a/pdns/recursor_cache.hh
+++ b/pdns/recursor_cache.hh
@@ -71,15 +71,16 @@ public:
 private:
   struct StoredRecord
   {
-    uint32_t d_ttd;
+    mutable uint32_t d_ttd;
     optString<> d_string;
     bool operator<(const StoredRecord& rhs) const
     {
-      return make_pair(d_ttd, d_string) < make_pair(rhs.d_ttd, rhs.d_string);
+      return d_string < rhs.d_string;
+      //      return make_pair(d_ttd, d_string) < make_pair(rhs.d_ttd, rhs.d_string);
     }
   };
   typedef map<string, set<StoredRecord> > cache_t;
-
+private:
   cache_t d_cache;
 };
 

diff --git a/pdns/syncres.cc b/pdns/syncres.cc
index 7697afb17c81d0f498332e05e82ec2784268d5c1..6efa21dd053a27bf08d873485a17f449bee767bb 100644 (file)
--- a/pdns/syncres.cc
+++ b/pdns/syncres.cc
@@ -37,6 +37,7 @@
 extern MemRecursorCache RC;
 
 map<string,NegCacheEntry> SyncRes::s_negcache;    
+map<string,DecayingEwma> SyncRes::s_nsSpeeds;
 unsigned int SyncRes::s_queries;
 unsigned int SyncRes::s_outgoingtimeouts;
 unsigned int SyncRes::s_outqueries;
@@ -229,7 +230,6 @@ bool SyncRes::doCacheCheck(const string &qname, const QType &qtype, vector<DNSRe
       giveNegative=true;
       sqname=ni->second.name;
       sqt="SOA";
-
     }
     else {
       LOG<<prefix<<qname<<": Entire record '"<<toLower(qname)<<"' was negatively cached, but entry expired"<<endl;
@@ -304,7 +304,7 @@ bool SyncRes::moreSpecificThan(const string& a, const string &b)
   return counta>countb;
 }
 
-static map<string,DecayingEwma> nsSpeeds;
+
 
 struct speedOrder
 {
@@ -324,7 +324,7 @@ inline vector<string> SyncRes::shuffle(set<string> &nameservers, const string &p
 
   for(set<string>::const_iterator i=nameservers.begin();i!=nameservers.end();++i) {
     rnameservers.push_back(*i);
-    DecayingEwma& temp=nsSpeeds[toLower(*i)];
+    DecayingEwma& temp=s_nsSpeeds[toLower(*i)];
     speeds[*i]=temp.get(&d_now);
   }
   random_shuffle(rnameservers.begin(),rnameservers.end());
@@ -405,7 +405,7 @@ int SyncRes::doResolveAt(set<string> nameservers, string auth, const string &qna
          else
            LOG<<prefix<<qname<<": error resolving"<<endl;
 
-         nsSpeeds[toLower(*tns)].submit(1000000, &d_now); // 1 sec
+         s_nsSpeeds[toLower(*tns)].submit(1000000, &d_now); // 1 sec
          
          s_throttle.throttle(d_now.tv_sec, remoteIP+"|"+qname+"|"+qtype.getName(),20,5);
          continue;
@@ -431,7 +431,7 @@ int SyncRes::doResolveAt(set<string> nameservers, string auth, const string &qna
        continue;
       }
       LOG<<prefix<<qname<<": Got "<<result.size()<<" answers from "<<*tns<<" ("<<remoteIP<<"), rcode="<<d_lwr.d_rcode<<", in "<<d_lwr.d_usec/1000<<"ms"<<endl;
-      nsSpeeds[toLower(*tns)].submit(d_lwr.d_usec, &d_now);
+      s_nsSpeeds[toLower(*tns)].submit(d_lwr.d_usec, &d_now);
 
       map<string,set<DNSResourceRecord> > tcache;
       // reap all answers from this packet that are acceptable

diff --git a/pdns/syncres.hh b/pdns/syncres.hh
index 35ab33bb504e4660ad220d501996d9b2bded895a..edd87eb69bce36445ae296f0f55ad960711e2264 100644 (file)
--- a/pdns/syncres.hh
+++ b/pdns/syncres.hh
@@ -64,7 +64,11 @@ public:
     } 
     else if(i->second.ttd > e.ttd || (i->second.count) < e.count) 
       d_cont[t]=e;
-
+  }
+  
+  unsigned int size()
+  {
+    return d_cont.size();
   }
 private:
   int d_limit;
@@ -164,6 +168,7 @@ public:
   unsigned int d_throttledqueries;
   unsigned int d_timeouts;
   static map<string,NegCacheEntry> s_negcache;    
+  static map<string,DecayingEwma> s_nsSpeeds;
   static Throttle<string> s_throttle;
 private:
   struct GetBestNSAnswer;