From 65a60c2c984427e143397b0327b2c49cdfa73b5d Mon Sep 17 00:00:00 2001 From: bert hubert Date: Fri, 30 Oct 2015 20:36:45 +0100 Subject: [PATCH] make all caches that can be cleaned in the recursor canonical ordered so we can do subtree cleaning. Before this commit, packet cache cleaning actually did not work in 4.x! --- pdns/dns.cc | 47 ----------------------- pdns/dns.hh | 83 ++++++++++++++++++++++++++++++++++++++++- pdns/rec_channel_rec.cc | 53 +++++++++++++++++++------- pdns/recpacketcache.cc | 15 ++++++-- pdns/recpacketcache.hh | 2 +- pdns/recursor_cache.cc | 36 ++++++++++++------ pdns/recursor_cache.hh | 4 +- pdns/reczones.cc | 10 ++--- pdns/syncres.hh | 7 ++-- pdns/ws-recursor.cc | 4 +- 10 files changed, 172 insertions(+), 89 deletions(-) diff --git a/pdns/dns.cc b/pdns/dns.cc index 46da40b1c..b52bb8ae9 100644 --- a/pdns/dns.cc +++ b/pdns/dns.cc @@ -66,53 +66,6 @@ private: const unsigned int d_length; }; -//! compares two dns packets, skipping the header, but including the question and the qtype -bool dnspacketLessThan(const std::string& a, const std::string& b) -{ - if(a.length() <= 12 || b.length() <= 12) - return a.length() < b.length(); -// throw runtime_error("Error parsing question in dnspacket comparison: packet too short"); - - // we find: 3www4ds9a2nl0XXYY, where XX and YY are each 2 bytes describing class and type - - BoundsCheckingPointer aSafe(a), bSafe(b); - int aPos=12, bPos=12; - - unsigned char aLabelLen, bLabelLen; - - do { - aLabelLen = aSafe[aPos++]; bLabelLen = bSafe[bPos++]; - // cerr<<"aLabelLen: "<<(int)aLabelLen<<", bLabelLen: "<< (int)bLabelLen<=(const unsigned char*)a.c_str() + a.size()) + return true; + + uint16_t aQtype = *(p+1)*256 + *(p+2); + uint16_t aQclass =*(p+3)*256 + *(p+4); + + for(p = (const unsigned char*)b.c_str()+12; p < (const unsigned char*)b.c_str() + b.size() && *p && rhscount < sizeof(rhspos); p+=*p+1) + rhspos[rhscount++]=(p-(const unsigned char*)b.c_str()); + + if(p>=(const unsigned char*)b.c_str() + b.size()) + return false; + + uint16_t bQtype = *(p+1)*256 + *(p+2); + uint16_t bQclass =*(p+3)*256 + *(p+4); + + if(ourcount == sizeof(ourpos) || rhscount==sizeof(rhspos)) { + DNSName aname(a.c_str(), a.size(), 12, false, &aQtype, &aQclass); + DNSName bname(b.c_str(), b.size(), 12, false, &bQtype, &bQclass); + + if(aname.slowCanonCompare(bname)) + return true; + if(aname!=bname) + return false; + return boost::tie(aQtype, aQclass) < boost::tie(bQtype, bQclass); + } + for(;;) { + if(ourcount == 0 && rhscount != 0) + return true; + if(ourcount == 0 && rhscount == 0) + break; + if(ourcount !=0 && rhscount == 0) + return false; + ourcount--; + rhscount--; + + bool res=std::lexicographical_compare( + a.c_str() + ourpos[ourcount] + 1, + a.c_str() + ourpos[ourcount] + 1 + *(a.c_str() + ourpos[ourcount]), + b.c_str() + rhspos[rhscount] + 1, + b.c_str() + rhspos[rhscount] + 1 + *(b.c_str() + rhspos[rhscount]), + [](const char& a, const char& b) { + return dns2_tolower(a) < dns2_tolower(b); + }); + + // cout<<"Forward: "<doWipeCache(canon) + t_packetCache->doWipePacketCache(canon)); + return new uint64_t(t_RC->doWipeCache(canon, subtree)); } +uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree) +{ + return new uint64_t(t_packetCache->doWipePacketCache(canon,0xffff, subtree)); +} -uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon) + +uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree) { - uint64_t res = t_sstorage->negcache.count(tie(canon)); - pair range=t_sstorage->negcache.equal_range(tie(canon)); - t_sstorage->negcache.erase(range.first, range.second); - return new uint64_t(res); + if(!subtree) { + uint64_t res = t_sstorage->negcache.count(tie(canon)); + auto range=t_sstorage->negcache.equal_range(tie(canon)); + t_sstorage->negcache.erase(range.first, range.second); + return new uint64_t(res); + } + else { + unsigned int erased=0; + for(auto iter = t_sstorage->negcache.lower_bound(tie(canon)); iter != t_sstorage->negcache.end(); ) { + if(!iter->d_qname.isPartOf(canon)) + break; + t_sstorage->negcache.erase(iter++); + erased++; + } + return new uint64_t(erased); + } } template string doWipeCache(T begin, T end) { - int count=0, countNeg=0; + int count=0, pcount=0, countNeg=0; for(T i=begin; i != end; ++i) { - DNSName canon=DNSName(*i); - count+= broadcastAccFunction(boost::bind(pleaseWipeCache, canon)); - countNeg+=broadcastAccFunction(boost::bind(pleaseWipeAndCountNegCache, canon)); + DNSName canon; + bool subtree=false; + if(boost::ends_with(*i, "$")) { + canon=DNSName(i->substr(0, i->size()-1)); + subtree=true; + } + else + canon=DNSName(*i); + + count+= broadcastAccFunction(boost::bind(pleaseWipeCache, canon, subtree)); + pcount+= broadcastAccFunction(boost::bind(pleaseWipePacketCache, canon, subtree)); + countNeg+=broadcastAccFunction(boost::bind(pleaseWipeAndCountNegCache, canon, subtree)); } - return "wiped "+lexical_cast(count)+" records, "+lexical_cast(countNeg)+" negative records\n"; + return "wiped "+std::to_string(count)+" records, "+std::to_string(countNeg)+" negative records, "+std::to_string(pcount)+" packets\n"; } template @@ -281,7 +306,7 @@ string setMinimumTTL(T begin, T end) if(end-begin != 1) return "Need to supply new minimum TTL number\n"; SyncRes::s_minimumTTL = atoi(begin->c_str()); - return "New minimum TTL: " + lexical_cast(SyncRes::s_minimumTTL) + "\n"; + return "New minimum TTL: " + std::to_string(SyncRes::s_minimumTTL) + "\n"; } diff --git a/pdns/recpacketcache.cc b/pdns/recpacketcache.cc index 8274a5aee..a102ca85f 100644 --- a/pdns/recpacketcache.cc +++ b/pdns/recpacketcache.cc @@ -15,7 +15,7 @@ RecursorPacketCache::RecursorPacketCache() d_hits = d_misses = 0; } -int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype) +int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype, bool subtree) { vector packet; DNSPacketWriter pw(packet, name, 0); @@ -33,9 +33,17 @@ int RecursorPacketCache::doWipePacketCache(const DNSName& name, uint16_t qtype) uint16_t t; DNSName found(iter->d_packet.c_str(), iter->d_packet.size(), 12, false, &t); - if(found==name) { // this is case insensitive - break; + // cout<<"At record "< range; - if(qtype==0xffff) - range=d_cache.equal_range(tie(name)); - else - range=d_cache.equal_range(tie(name, qtype)); - for(cache_t::const_iterator i=range.first; i != range.second; ) { - count++; - d_cache.erase(i++); + if(!sub) { + if(qtype==0xffff) + range=d_cache.equal_range(tie(name)); + else + range=d_cache.equal_range(tie(name, qtype)); + for(cache_t::const_iterator i=range.first; i != range.second; ) { + count++; + d_cache.erase(i++); + } + } + else { + for(auto iter = d_cache.lower_bound(tie(name)); iter != d_cache.end(); ) { + if(!iter->d_qname.isPartOf(name)) + break; + if(iter->d_qtype == qtype || qtype == 0xffff) { + count++; + d_cache.erase(iter++); + } + else + iter++; + } } return count; } @@ -277,12 +291,12 @@ uint64_t MemRecursorCache::doDump(int fd) return 0; } fprintf(fp, "; main record cache dump from thread follows\n;\n"); - typedef cache_t::nth_index<1>::type sequence_t; - sequence_t& sidx=d_cache.get<1>(); + + auto& sidx=d_cache.get<0>(); uint64_t count=0; time_t now=time(0); - for(sequence_t::const_iterator i=sidx.begin(); i != sidx.end(); ++i) { + for(auto i=sidx.cbegin(); i != sidx.cend(); ++i) { for(auto j=i->d_records.cbegin(); j != i->d_records.cend(); ++j) { count++; try { diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index 1c643d084..b71b6fee0 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -40,7 +40,7 @@ public: uint64_t doDump(int fd); uint64_t doDumpNSSpeeds(int fd); - int doWipeCache(const DNSName& name, uint16_t qtype=0xffff); + int doWipeCache(const DNSName& name, bool sub, uint16_t qtype=0xffff); bool doAgeCache(time_t now, const DNSName& name, uint16_t qtype, int32_t newTTL); uint64_t cacheHits, cacheMisses; @@ -75,7 +75,7 @@ private: member, member >, - composite_key_compare, std::less > + composite_key_compare > >, sequenced<> > diff --git a/pdns/reczones.cc b/pdns/reczones.cc index 5c14a4abd..ed4fcbf85 100644 --- a/pdns/reczones.cc +++ b/pdns/reczones.cc @@ -54,6 +54,7 @@ void primeHints(void) }; DNSRecord arr, aaaarr, nsrr; + nsrr.d_name=DNSName("."); arr.d_type=QType::A; aaaarr.d_type=QType::AAAA; nsrr.d_type=QType::NS; @@ -254,7 +255,7 @@ string reloadAuthAndForwards() for(SyncRes::domainmap_t::const_iterator i = t_sstorage->domainmap->begin(); i != t_sstorage->domainmap->end(); ++i) { for(SyncRes::AuthDomain::records_t::const_iterator j = i->second.d_records.begin(); j != i->second.d_records.end(); ++j) - broadcastAccFunction(boost::bind(pleaseWipeCache, j->d_name)); + broadcastAccFunction(boost::bind(pleaseWipeCache, j->d_name, false)); } string configname=::arg()["config-dir"]+"/recursor.conf"; @@ -294,12 +295,11 @@ string reloadAuthAndForwards() // purge again - new zones need to blank out the cache for(SyncRes::domainmap_t::const_iterator i = newDomainMap->begin(); i != newDomainMap->end(); ++i) { - for(SyncRes::AuthDomain::records_t::const_iterator j = i->second.d_records.begin(); j != i->second.d_records.end(); ++j) - broadcastAccFunction(boost::bind(pleaseWipeCache, j->d_name)); + broadcastAccFunction(boost::bind(pleaseWipeCache, i->first, true)); + broadcastAccFunction(boost::bind(pleaseWipePacketCache, i->first, true)); + broadcastAccFunction(boost::bind(pleaseWipeAndCountNegCache, i->first, true)); } - // this is pretty blunt - broadcastFunction(pleaseWipeNegCache); broadcastFunction(boost::bind(pleaseUseNewSDomainsMap, newDomainMap)); delete original; return "ok\n"; diff --git a/pdns/syncres.hh b/pdns/syncres.hh index 99a10256c..618240ae7 100644 --- a/pdns/syncres.hh +++ b/pdns/syncres.hh @@ -324,7 +324,7 @@ public: member, member >, - composite_key_compare, std::less > + composite_key_compare > >, sequenced<> > @@ -651,7 +651,8 @@ uint64_t* pleaseGetConcurrentQueries(); uint64_t* pleaseGetThrottleSize(); uint64_t* pleaseGetPacketCacheHits(); uint64_t* pleaseGetPacketCacheSize(); -uint64_t* pleaseWipeCache(const DNSName& canon); -uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon); +uint64_t* pleaseWipeCache(const DNSName& canon, bool subtree=false); +uint64_t* pleaseWipePacketCache(const DNSName& canon, bool subtree); +uint64_t* pleaseWipeAndCountNegCache(const DNSName& canon, bool subtree=false); void doCarbonDump(void*); #endif diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc index 49da6d1cf..af39651c6 100644 --- a/pdns/ws-recursor.cc +++ b/pdns/ws-recursor.cc @@ -418,8 +418,8 @@ static void apiServerFlushCache(HttpRequest* req, HttpResponse* resp) { throw HttpMethodNotAllowedException(); DNSName canon(req->getvars["domain"]); - int count = broadcastAccFunction(boost::bind(pleaseWipeCache, canon)); - count += broadcastAccFunction(boost::bind(pleaseWipeAndCountNegCache, canon)); + int count = broadcastAccFunction(boost::bind(pleaseWipeCache, canon, false)); + count += broadcastAccFunction(boost::bind(pleaseWipeAndCountNegCache, canon, false)); map object; object["count"] = lexical_cast(count); object["result"] = "Flushed cache."; -- 2.40.0