From: Remi Gacogne Date: Sun, 1 Apr 2018 18:02:28 +0000 (+0200) Subject: rec: Add a hashed index to the query cache X-Git-Tag: dnsdist-1.3.1~102^2~4 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=4c8d258513787980874ddd386fd73b27621d2105;p=pdns rec: Add a hashed index to the query cache --- diff --git a/pdns/recursor_cache.cc b/pdns/recursor_cache.cc index acaad7e14..1b41143b1 100644 --- a/pdns/recursor_cache.cc +++ b/pdns/recursor_cache.cc @@ -29,16 +29,16 @@ unsigned int MemRecursorCache::bytes() const { unsigned int ret=0; - for(cache_t::const_iterator i=d_cache.begin(); i!=d_cache.end(); ++i) { + for(const auto& i : d_cache) { ret+=sizeof(struct CacheEntry); - ret+=(unsigned int)i->d_qname.toString().length(); - for(auto j=i->d_records.begin(); j!= i->d_records.end(); ++j) - ret+= sizeof(*j); // XXX WRONG we don't know the stored size! j->size(); + ret+=(unsigned int)i.d_qname.toString().length(); + for(const auto& record : i.d_records) + ret+= sizeof(record); // XXX WRONG we don't know the stored size! j->size(); } return ret; } -int32_t MemRecursorCache::handleHit(cache_t::iterator entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth) +int32_t MemRecursorCache::handleHit(MemRecursorCache::OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth) { int32_t ttd = entry->d_ttd; @@ -48,6 +48,8 @@ int32_t MemRecursorCache::handleHit(cache_t::iterator entry, const DNSName& qnam // cerr<<"Looking at "<d_records.size()<<" records for this name"<reserve(res->size() + entry->d_records.size()); + for(const auto& k : entry->d_records) { DNSRecord dr; dr.d_name = qname; @@ -56,7 +58,7 @@ int32_t MemRecursorCache::handleHit(cache_t::iterator entry, const DNSName& qnam dr.d_content = k; dr.d_ttl = static_cast(entry->d_ttd); dr.d_place = DNSResourceRecord::ANSWER; - res->push_back(dr); + res->push_back(std::move(dr)); } } @@ -143,21 +145,22 @@ MemRecursorCache::cache_t::const_iterator MemRecursorCache::getEntryUsingECSInde } // returns -1 for no hits -std::pair MemRecursorCache::getEntries(const DNSName &qname, const QType& qt) +std::pair MemRecursorCache::getEntries(const DNSName &qname, const QType& qt) { // cerr<<"looking up "<< qname<<"|"+qt.getName()<<"\n"; if(!d_cachecachevalid || d_cachedqname!= qname) { // cerr<<"had cache cache miss"<(); + d_cachecache = idx.equal_range(qname); + d_cachecachevalid = true; } // else cerr<<"had cache cache hit!"<d_auth) return false; @@ -210,17 +213,18 @@ int32_t MemRecursorCache::get(time_t now, const DNSName &qname, const QType& qt, auto entries = getEntries(qname, qt); if(entries.first!=entries.second) { - for(cache_t::const_iterator i=entries.first; i != entries.second; ++i) { + for(auto i=entries.first; i != entries.second; ++i) { + auto firstIndexIterator = d_cache.project(i); if (i->d_ttd <= now) { - moveCacheItemToFront(d_cache, i); + moveCacheItemToFront(d_cache, firstIndexIterator); continue; } - if (!entryMatches(i, qtype, requireAuth, who)) + if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) continue; - ttd = handleHit(i, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); + ttd = handleHit(firstIndexIterator, qname, who, res, signatures, authorityRecs, variable, state, wasAuth); if(qt.getCode()!=QType::ANY && qt.getCode()!=QType::ADDR) // normally if we have a hit, we are done break; @@ -320,38 +324,51 @@ int MemRecursorCache::doWipeCache(const DNSName& name, bool sub, uint16_t qtype) if(!sub) { pair ecsIndexRange; - if(qtype==0xffff) { - range = d_cache.equal_range(tie(name)); - ecsIndexRange = d_ecsIndex.equal_range(tie(name)); - } - else { - range=d_cache.equal_range(tie(name, qtype)); - ecsIndexRange = d_ecsIndex.equal_range(tie(name, qtype)); + auto& idx = d_cache.get(); + auto range = idx.equal_range(name); + for(auto& i=range.first; i != range.second; ) { + if (qtype == 0xffff || i->d_qtype == qtype) { + count++; + idx.erase(i++); + } + else { + ++i; + } } - for(cache_t::const_iterator i=range.first; i != range.second; ) { - count++; - d_cache.erase(i++); + if (qtype == 0xffff) { + auto& ecsIdx = d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(name); + for(auto i = ecsIndexRange.first; i != ecsIndexRange.second; ) { + ecsIdx.erase(i++); + } } - for(auto i = ecsIndexRange.first; i != ecsIndexRange.second; ) { - d_ecsIndex.erase(i++); + else { + auto& ecsIdx = d_ecsIndex.get(); + auto ecsIndexRange = ecsIdx.equal_range(tie(name, qtype)); + for(auto i = ecsIndexRange.first; i != ecsIndexRange.second; ) { + ecsIdx.erase(i++); + } } } else { - for(auto iter = d_cache.lower_bound(tie(name)); iter != d_cache.end(); ) { + auto& idx = d_cache.get(); + auto& ecsIdx = d_ecsIndex.get(); + + for(auto iter = idx.lower_bound(name); iter != idx.end(); ) { if(!iter->d_qname.isPartOf(name)) break; if(iter->d_qtype == qtype || qtype == 0xffff) { count++; - d_cache.erase(iter++); + idx.erase(iter++); } else iter++; } - for(auto iter = d_ecsIndex.lower_bound(tie(name)); iter != d_ecsIndex.end(); ) { + for(auto iter = ecsIdx.lower_bound(name); iter != ecsIdx.end(); ) { if(!iter->d_qname.isPartOf(name)) break; if(iter->d_qtype == qtype || qtype == 0xffff) { - d_ecsIndex.erase(iter++); + ecsIdx.erase(iter++); } else { iter++; @@ -406,7 +423,9 @@ bool MemRecursorCache::updateValidationStatus(time_t now, const DNSName &qname, auto entries = getEntries(qname, qt); for(auto i = entries.first; i != entries.second; ++i) { - if (!entryMatches(i, qtype, requireAuth, who)) + auto firstIndexIterator = d_cache.project(i); + + if (!entryMatches(firstIndexIterator, qtype, requireAuth, who)) continue; i->d_state = newState; @@ -426,7 +445,7 @@ uint64_t MemRecursorCache::doDump(int fd) return 0; } fprintf(fp, "; main record cache dump from thread follows\n;\n"); - const auto& sidx=d_cache.get<1>(); + const auto& sidx=d_cache.get(); uint64_t count=0; time_t now=time(0); diff --git a/pdns/recursor_cache.hh b/pdns/recursor_cache.hh index bf35c9fc1..163465be8 100644 --- a/pdns/recursor_cache.hh +++ b/pdns/recursor_cache.hh @@ -143,44 +143,64 @@ private: uint16_t d_qtype; }; + struct HashedTag {}; + struct SequencedTag {}; + struct NameOnlyHashedTag {}; + struct OrderedTag {}; + typedef multi_index_container< CacheEntry, indexed_by < - ordered_unique< - composite_key< - CacheEntry, - member, - member, - member - >, - composite_key_compare, std::less > + ordered_unique, + composite_key< + CacheEntry, + member, + member, + member + >, + composite_key_compare, std::less > >, - sequenced<> + sequenced >, + hashed_non_unique, + member + > > > cache_t; + + typedef MemRecursorCache::cache_t::index::type::iterator OrderedTagIterator_t; + typedef MemRecursorCache::cache_t::index::type::iterator NameOnlyHashedTagIterator_t; + typedef multi_index_container< ECSIndexEntry, indexed_by < - ordered_unique < + hashed_unique , composite_key< ECSIndexEntry, member, member > + >, + ordered_unique, + composite_key< + ECSIndexEntry, + member, + member + >, + composite_key_compare > > > > ecsIndex_t; cache_t d_cache; ecsIndex_t d_ecsIndex; - pair d_cachecache; + std::pair d_cachecache; DNSName d_cachedqname; bool d_cachecachevalid; - bool entryMatches(cache_t::const_iterator& entry, uint16_t qt, bool requireAuth, const ComboAddress& who); - std::pair getEntries(const DNSName &qname, const QType& qt); + bool entryMatches(OrderedTagIterator_t& entry, uint16_t qt, bool requireAuth, const ComboAddress& who); + std::pair getEntries(const DNSName &qname, const QType& qt); cache_t::const_iterator getEntryUsingECSIndex(time_t now, const DNSName &qname, uint16_t qtype, bool requireAuth, const ComboAddress& who); - int32_t handleHit(cache_t::iterator entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth); + int32_t handleHit(OrderedTagIterator_t& entry, const DNSName& qname, const ComboAddress& who, vector* res, vector>* signatures, std::vector>* authorityRecs, bool* variable, vState* state, bool* wasAuth); public: void preRemoval(const CacheEntry& entry)