From 3e7dcee6a478218ee5e95c98cbaff55afa97037e Mon Sep 17 00:00:00 2001 From: bert hubert Date: Sat, 4 Jun 2016 12:19:16 +0200 Subject: [PATCH] wip commit for auth ixfr slaving --- pdns/communicator.cc | 3 +- pdns/communicator.hh | 24 +++- pdns/ixfr.cc | 9 +- pdns/ixfr.hh | 5 +- pdns/slavecommunicator.cc | 254 ++++++++++++++++++++++++-------------- 5 files changed, 189 insertions(+), 106 deletions(-) diff --git a/pdns/communicator.cc b/pdns/communicator.cc index daf405a2e..f472a4027 100644 --- a/pdns/communicator.cc +++ b/pdns/communicator.cc @@ -37,8 +37,7 @@ #include "arguments.hh" #include "packetcache.hh" -// #include "namespaces.hh" - +// there can be MANY OF THESE void CommunicatorClass::retrievalLoopThread(void) { for(;;) { diff --git a/pdns/communicator.hh b/pdns/communicator.hh index 767410a27..a40f18742 100644 --- a/pdns/communicator.hh +++ b/pdns/communicator.hh @@ -1,6 +1,6 @@ /* PowerDNS Versatile Database Driven Nameserver - Copyright (C) 2002-2010 PowerDNS.COM BV + Copyright (C) 2002-2016 PowerDNS.COM BV This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License version 2 @@ -31,6 +31,7 @@ #include #include #include +#include using namespace boost::multi_index; #include @@ -140,6 +141,8 @@ private: }; +struct ZoneStatus; + /** this class contains a thread that communicates with other nameserver and does housekeeping. Initially, it is notified only of zones that need to be pulled in because they have been updated. */ @@ -191,13 +194,15 @@ private: pthread_mutex_t d_holelock; void launchRetrievalThreads(); void suck(const DNSName &domain, const string &remote); - void ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote); + void ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote, boost::scoped_ptr& pdl, + ZoneStatus& zs, vector* axfr); void slaveRefresh(PacketHandler *P); void masterUpdateCheck(PacketHandler *P); pthread_mutex_t d_lock; UniQueue d_suckdomains; + set d_inprogress; Semaphore d_suck_sem; Semaphore d_any_sem; @@ -210,6 +215,21 @@ private: bool d_havepriosuckrequest; bool d_masterschanged, d_slaveschanged; bool d_preventSelfNotification; + + struct RemoveSentinel + { + explicit RemoveSentinel(const DNSName& dn, CommunicatorClass* cc) : d_dn(dn), d_cc(cc) + {} + + ~RemoveSentinel() + { + Lock l(&d_cc->d_lock); + d_cc->d_inprogress.erase(d_dn); + } + DNSName d_dn; + CommunicatorClass* d_cc; +}; + }; // class that one day might be more than a function to help you get IP addresses for a nameserver diff --git a/pdns/ixfr.cc b/pdns/ixfr.cc index b30b84f06..06d9fe5a9 100644 --- a/pdns/ixfr.cc +++ b/pdns/ixfr.cc @@ -5,8 +5,9 @@ #include "dnssecinfra.hh" -// if you the remove,add pairs always remove a SOA and add a new one. If you get an empty remove, it means you got an AXFR! -vector, vector > > getIXFRDeltas(const ComboAddress& master, const DNSName& zone, const DNSRecord& oursr, const TSIGTriplet& tt) +// Returns pairs of "remove & add" vectors. If you get an empty remove, it means you got an AXFR! +vector, vector > > getIXFRDeltas(const ComboAddress& master, const DNSName& zone, const DNSRecord& oursr, + const TSIGTriplet& tt, const ComboAddress* laddr) { vector, vector > > ret; vector packet; @@ -39,8 +40,8 @@ vector, vector > > getIXFRDeltas(const ComboAd Socket s(master.sin4.sin_family, SOCK_STREAM); // cout<<"going to connect"<, vector > > getIXFRDeltas(const ComboAddress& master, const DNSName& zone, const DNSRecord& sr, - const TSIGTriplet& tt=TSIGTriplet()); +vector, vector > > getIXFRDeltas(const ComboAddress& master, const DNSName& zone, + const DNSRecord& sr, const TSIGTriplet& tt=TSIGTriplet(), + const ComboAddress* laddr=0); diff --git a/pdns/slavecommunicator.cc b/pdns/slavecommunicator.cc index 4280361d4..d19383b47 100644 --- a/pdns/slavecommunicator.cc +++ b/pdns/slavecommunicator.cc @@ -45,7 +45,7 @@ #include "lua-auth.hh" #include "namespaces.hh" #include "common_startup.hh" -#include + #include "ixfr.hh" using boost::scoped_ptr; @@ -59,10 +59,10 @@ void CommunicatorClass::addSuckRequest(const DNSName &domain, const string &mast pair res; res=d_suckdomains.push_back(sr); - if(res.second) { d_suck_sem.post(); } + } struct ZoneStatus @@ -77,10 +77,12 @@ struct ZoneStatus unsigned int soa_serial{0}; set nsset, qnames, secured; uint32_t domain_id; + int numDeltas{0}; }; -void CommunicatorClass::ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote) +void CommunicatorClass::ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, const ComboAddress& laddr, const ComboAddress& remote, scoped_ptr& pdl, + ZoneStatus& zs, vector* axfr) { UeberBackend B; // fresh UeberBackend @@ -101,63 +103,121 @@ void CommunicatorClass::ixfrSuck(const DNSName &domain, const TSIGTriplet& tt, c DNSRecord dr; dr.d_content = std::make_shared(DNSName("."), DNSName("."), st); - auto deltas = getIXFRDeltas(remote, domain, dr, tt); - cout<<"Got "<& rrset) - // first thing we need to do is delete every {qname/qt} present in before and not in after + // which thinks in terms of RRSETs + // however, IXFR does not, and removes and adds *records* (bummer) + // this means that we must group updates by {qname,qtype}, retrieve the RRSET, apply + // the add/remove updates, and replaceRRSet the whole thing. - typedef set> present_t; - present_t pbefore, pafter; - for(const auto& b: before) - pbefore.insert({b.d_name, b.d_type}); - for(const auto& a: after) - pafter.insert({a.d_name, a.d_type}); - - vector> diff; - - set_difference(pbefore.cbegin(), pbefore.cend(), pafter.cbegin(), pafter.cend(), back_inserter(diff)); + + map, pair, vector > > grouped; + + for(const auto& x: remove) + grouped[{x.d_name, x.d_type}].first.push_back(x); + for(const auto& x: add) + grouped[{x.d_name, x.d_type}].second.push_back(x); di.backend->startTransaction(domain, -1); - for(const auto& gone : diff) { - // cerr<<"Removing "<replaceRRSet(di.id, gone.first+domain, QType(gone.second), vector()); - } - cout<<"Removed "< rrset; + B.lookup(QType(g.first.second), g.first.first, 0, di.id); + while(B.get(rr)) { + rrset.push_back(DNSRecord{rr}); + } + // O(N^2)! + rrset.erase(remove_if(rrset.begin(), rrset.end(), + [&g](const DNSRecord& dr) { + return count(g.second.first.cbegin(), + g.second.first.cend(), dr); + }), rrset.end()); + // the DNSRecord== operator compares on name, type, class and lowercase content representation + + for(const auto& x : g.second.second) { + rrset.push_back(x); + } - map, vector> replacement; - - for(const auto& add : after) { - DNSResourceRecord dr(add); - dr.qname += domain; - dr.domain_id=di.id; - if(add.d_type == QType::SOA) - cout<<"New SOA: "<getZoneRepresentation()<content<replaceRRSet(di.id, rep.first.first+domain, QType(rep.first.second), rep.second); - } + vector replacement; + for(const auto& x : rrset) { + DNSResourceRecord dr(x); + dr.qname += domain; + dr.domain_id = di.id; + if(x.d_type == QType::SOA) { + // cout<<"New SOA: "<getZoneRepresentation()<(x); + zs.soa_serial=sr->d_st.serial; + } + + replacement.push_back(dr); + } - cout<<"Added "<replaceRRSet(di.id, g.first.first+domain, QType(g.first.second), replacement); + } di.backend->commitTransaction(); } } catch(std::exception& p) { - cerr<<"Got exception: "< doAxfr(const ComboAddress& raddr, const DNSName& domai vector out; if(!pdl || !pdl->axfrfilter(raddr, domain, *i, out)) { - out.push_back(*i); + out.push_back(*i); // if axfrfilter didn't do anything, we put our record in 'out' ourselves } for(DNSResourceRecord& rr : out) { - switch(rr.qtype.getCode()) { - case QType::NSEC3PARAM: { - zs.ns3pr = NSEC3PARAMRecordContent(rr.content); - zs.isDnssecZone = zs.isNSEC3 = true; - zs.isNarrow = false; - continue; - } - case QType::NSEC3: { - NSEC3RecordContent ns3rc(rr.content); - if (firstNSEC3) { - zs.isDnssecZone = zs.isPresigned = true; - firstNSEC3 = false; - } else if (zs.optOutFlag != (ns3rc.d_flags & 1)) - throw PDNSException("Zones with a mixture of Opt-Out NSEC3 RRs and non-Opt-Out NSEC3 RRs are not supported."); - zs.optOutFlag = ns3rc.d_flags & 1; - if (ns3rc.d_set.count(QType::NS) && !(rr.qname==domain)) { - DNSName hashPart = DNSName(toLower(rr.qname.makeRelative(domain).toString())); - zs.secured.insert(hashPart); - } - continue; - } - case QType::NSEC: { - zs.isDnssecZone = zs.isPresigned = true; - continue; - } - case QType::SOA: { + processRecordForZS(domain, firstNSEC3, rr, zs); + if(rr.qtype.getCode() == QType::SOA) { if(soa_received) continue; //skip the last SOA SOAData sd; fillSOAData(rr.content,sd); zs.soa_serial = sd.serial; soa_received = true; - break; - } - case QType::NS: { - if(rr.qname!=domain) - zs.nsset.insert(rr.qname); - break; - } - default: - break; } - zs.qnames.insert(rr.qname); - - rr.domain_id=zs.domain_id; rrs.push_back(rr); + } } } return rrs; } + void CommunicatorClass::suck(const DNSName &domain, const string &remote) { + { + Lock l(&d_lock); + if(d_inprogress.count(domain)) { + return; + } + d_inprogress.insert(domain); + } + RemoveSentinel rs(domain, this); // this removes us from d_inprogress when we go out of scope + L< rrs; if(dk.isSecuredZone(domain)) { hadDnssecZone=true; hadPresigned=dk.isPresigned(domain); @@ -334,11 +369,37 @@ void CommunicatorClass::suck(const DNSName &domain, const string &remote) else if(di.serial) { vector meta; B.getDomainMetadata(domain, "IXFR", meta); - if(!meta.empty() && meta[0]=="1") - return ixfrSuck(domain, tt, laddr, raddr); + if(!meta.empty() && meta[0]=="1") { + vector axfr; + ixfrSuck(domain, tt, laddr, raddr, pdl, zs, &axfr); + if(!axfr.empty()) { + L<(dr); + zs.soa_serial = sd->d_st.serial; + } + rrs.push_back(rr); + } + } + else { + L< rrs = doAxfr(raddr, domain, tt, laddr, pdl, zs); + if(rrs.empty()) { + L<startTransaction(domain, zs.domain_id); - L<getBackend(); vector rdomains; - vector sdomains; // the bool is for 'presigned' + vector sdomains; vector trysuperdomains; { @@ -670,10 +731,9 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) delete r; } } - - if(rdomains.empty()) // if we have priority domains, check them first + if(rdomains.empty()) { // if we have priority domains, check them first B->getUnfreshSlaveInfos(&rdomains); - + } DNSSECKeeper dk(B); // NOW HEAR THIS! This DK uses our B backend, so no interleaved access! { Lock l(&d_lock); @@ -687,9 +747,12 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) continue; // remove unfresh domains already queued for AXFR, no sense polling them again sr.master=*di.masters.begin(); - if(nameindex.count(sr)) { + if(nameindex.count(sr)) { // this does NOT however protect us against AXFRs already in progress! continue; } + if(d_inprogress.count(sr.domain)) // this does + continue; + DomainNotificationInfo dni; dni.di=di; dni.dnssecOk = dk.isPresigned(di.zone); @@ -718,12 +781,11 @@ void CommunicatorClass::slaveRefresh(PacketHandler *P) sdomains.push_back(dni); } } - if(sdomains.empty()) { if(d_slaveschanged) { Lock l(&d_lock); - L<