From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Tue, 12 Jun 2018 14:36:39 +0000 (+0200)
Subject: rec: Add support for multiple rpz masters as failover
X-Git-Tag: dnsdist-1.3.1~19^2
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=5f31188631d00d7c8d05abae064b72f29ccabb63;p=pdns

rec: Add support for multiple rpz masters as failover
---

diff --git a/pdns/rec-lua-conf.cc b/pdns/rec-lua-conf.cc
index daa254200..6f79ab01b 100644
--- a/pdns/rec-lua-conf.cc
+++ b/pdns/rec-lua-conf.cc
@@ -92,7 +92,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
   if(!ifs)
     throw PDNSException("Cannot open file '"+fname+"': "+strerror(errno));
 
-  std::vector<std::tuple<ComboAddress, boost::optional<DNSFilterEngine::Policy>, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, std::shared_ptr<SOARecordContent>, std::string> > rpzMasterThreads;
+  std::vector<std::tuple<std::vector<ComboAddress>, boost::optional<DNSFilterEngine::Policy>, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, std::shared_ptr<SOARecordContent>, std::string> > rpzMasterThreads;
 
   auto luaconfsLocal = g_luaconfs.getLocal();
   lci.generation = luaconfsLocal->generation + 1;
@@ -139,7 +139,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       }
     });
 
-  Lua.writeFunction("rpzMaster", [&lci, &rpzMasterThreads](const string& master_, const string& zoneName, const boost::optional<std::unordered_map<string,boost::variant<uint32_t, string>>>& options) {
+  Lua.writeFunction("rpzMaster", [&lci, &rpzMasterThreads](const boost::variant<string, std::vector<std::pair<int, string> > >& masters_, const string& zoneName, const boost::optional<std::unordered_map<string,boost::variant<uint32_t, string>>>& options) {
 
       boost::optional<DNSFilterEngine::Policy> defpol;
       std::shared_ptr<DNSFilterEngine::Zone> zone = std::make_shared<DNSFilterEngine::Zone>();
@@ -149,7 +149,16 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       uint16_t axfrTimeout = 20;
       uint32_t maxTTL = std::numeric_limits<uint32_t>::max();
       ComboAddress localAddress;
-      ComboAddress master(master_, 53);
+      std::vector<ComboAddress> masters;
+      if (masters_.type() == typeid(string)) {
+        masters.push_back(ComboAddress(boost::get<std::string>(masters_), 53));
+      }
+      else {
+        for (const auto& master : boost::get<std::vector<std::pair<int, std::string>>>(masters_)) {
+          masters.push_back(ComboAddress(master.second, 53));
+        }
+      }
+
       size_t zoneIdx;
       std::string dumpFile;
       std::shared_ptr<SOARecordContent> sr = nullptr;
@@ -198,9 +207,13 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
           }
         }
 
-        if (localAddress != ComboAddress() && localAddress.sin4.sin_family != master.sin4.sin_family) {
-          // We were passed a localAddress, check if its AF matches the master's
-          throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
+        if (localAddress != ComboAddress()) {
+          // We were passed a localAddress, check if its AF matches the masters'
+          for (const auto& master : masters) {
+            if (localAddress.sin4.sin_family != master.sin4.sin_family) {
+              throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
+            }
+          }
         }
 
         DNSName domain(zoneName);
@@ -236,7 +249,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
         exit(1);  // FIXME proper exit code?
       }
 
-      rpzMasterThreads.push_back(std::make_tuple(master, defpol, maxTTL, zoneIdx, tt, maxReceivedXFRMBytes, localAddress, axfrTimeout, sr, dumpFile));
+      rpzMasterThreads.push_back(std::make_tuple(masters, defpol, maxTTL, zoneIdx, tt, maxReceivedXFRMBytes, localAddress, axfrTimeout, sr, dumpFile));
     });
 
   typedef vector<pair<int,boost::variant<string, vector<pair<int, string> > > > > argvec_t;
diff --git a/pdns/recursordist/docs/lua-config/rpz.rst b/pdns/recursordist/docs/lua-config/rpz.rst
index 40abc8761..fe1785c15 100644
--- a/pdns/recursordist/docs/lua-config/rpz.rst
+++ b/pdns/recursordist/docs/lua-config/rpz.rst
@@ -33,9 +33,13 @@ In this example, 'policy.rpz' denotes the name of the zone to query for.
 
 .. function:: rpzMaster(address, name, settings)
 
+  .. versionchanged:: 4.2.0:
+
+    The first parameter can be a list of addresses.
+
   Load an RPZ from AXFR and keep retrieving with IXFR.
 
-  :param str address: The IP address to transfer the RPZ from
+  :param str address: The IP address to transfer the RPZ from. Also accepts a list of addresses since 4.2.0 in which case they will be tried one after another in the submitted order until a response is obtained
   :param str name: The name of this RPZ
   :param {} settings: A table to settings, see below
 
diff --git a/pdns/rpzloader.cc b/pdns/rpzloader.cc
index 9f26311f3..6ecc9c324 100644
--- a/pdns/rpzloader.cc
+++ b/pdns/rpzloader.cc
@@ -175,7 +175,7 @@ void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zon
   }
 }
 
-shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
+static shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
 {
   g_log<<Logger::Warning<<"Loading RPZ zone '"<<zoneName<<"' from "<<master.toStringWithPort()<<endl;
   if(!tt.name.empty())
@@ -337,7 +337,7 @@ static bool dumpZoneToDisk(const DNSName& zoneName, const std::shared_ptr<DNSFil
   return true;
 }
 
-void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, std::shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration)
+void RPZIXFRTracker(const std::vector<ComboAddress> masters, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, std::shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration)
 {
   bool isPreloaded = sr != nullptr;
   auto luaconfsLocal = g_luaconfs.getLocal();
@@ -356,29 +356,34 @@ void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine:
 
     /* full copy, as promised */
     std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);
-    try {
-      sr=loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
-      if(refresh == 0) {
-        refresh = sr->d_st.refresh;
+    for (const auto& master : masters) {
+      try {
+        sr = loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
+        if(refresh == 0) {
+          refresh = sr->d_st.refresh;
+        }
+        newZone->setSerial(sr->d_st.serial);
+        setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);
+
+        g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
+            lci.dfe.setZone(zoneIdx, newZone);
+          });
+
+        if (!dumpZoneFileName.empty()) {
+          dumpZoneToDisk(zoneName, newZone, dumpZoneFileName);
+        }
+
+        /* no need to try another master */
+        break;
       }
-      newZone->setSerial(sr->d_st.serial);
-      setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);
-
-      g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
-        lci.dfe.setZone(zoneIdx, newZone);
-      });
-
-      if (!dumpZoneFileName.empty()) {
-        dumpZoneToDisk(zoneName, newZone, dumpZoneFileName);
+      catch(const std::exception& e) {
+        g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
+        incRPZFailedTransfers(polName);
+      }
+      catch(const PDNSException& e) {
+        g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
+        incRPZFailedTransfers(polName);
       }
-    }
-    catch(const std::exception& e) {
-      g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
-      incRPZFailedTransfers(polName);
-    }
-    catch(const PDNSException& e) {
-      g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
-      incRPZFailedTransfers(polName);
     }
 
     if (!sr) {
@@ -411,22 +416,31 @@ void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine:
       return;
     }
 
-    g_log<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
     vector<pair<vector<DNSRecord>, vector<DNSRecord> > > deltas;
+    for (const auto& master : masters) {
+      g_log<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
+
+      ComboAddress local(localAddress);
+      if (local == ComboAddress()) {
+        local = getQueryLocalAddress(master.sin4.sin_family, 0);
+      }
 
-    ComboAddress local(localAddress);
-    if (local == ComboAddress())
-      local = getQueryLocalAddress(master.sin4.sin_family, 0);
+      try {
+        deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
 
-    try {
-      deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
-    } catch(std::runtime_error& e ){
-      g_log<<Logger::Warning<<e.what()<<endl;
-      incRPZFailedTransfers(polName);
-      continue;
+        /* no need to try another master */
+        break;
+      } catch(const std::runtime_error& e ){
+        g_log<<Logger::Warning<<e.what()<<endl;
+        incRPZFailedTransfers(polName);
+        continue;
+      }
     }
-    if(deltas.empty())
+
+    if(deltas.empty()) {
       continue;
+    }
+
     g_log<<Logger::Info<<"Processing "<<deltas.size()<<" delta"<<addS(deltas)<<" for RPZ "<<zoneName<<endl;
 
     oldZone = luaconfsLocal->dfe.getZone(zoneIdx);
diff --git a/pdns/rpzloader.hh b/pdns/rpzloader.hh
index 5faf77d8b..b29f14501 100644
--- a/pdns/rpzloader.hh
+++ b/pdns/rpzloader.hh
@@ -27,9 +27,8 @@
 extern bool g_logRPZChanges;
 
 std::shared_ptr<SOARecordContent> loadRPZFromFile(const std::string& fname, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL);
-std::shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout);
 void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zone> zone, bool addOrRemove, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL);
-void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration);
+void RPZIXFRTracker(const std::vector<ComboAddress> master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration);
 
 struct rpzStats
 {
diff --git a/regression-tests.recursor-dnssec/test_RPZ.py b/regression-tests.recursor-dnssec/test_RPZ.py
index 9ea04fe93..b2ca1bf96 100644
--- a/regression-tests.recursor-dnssec/test_RPZ.py
+++ b/regression-tests.recursor-dnssec/test_RPZ.py
@@ -150,7 +150,8 @@ class RPZRecursorTest(RecursorTest):
 
     global rpzServerPort
     _lua_config_file = """
-    rpzMaster('127.0.0.1:%d', 'zone.rpz.', { refresh=1 })
+    -- The first server is a bogus one, to test that we correctly fail over to the second one
+    rpzMaster({'127.0.0.1:9999', '127.0.0.1:%d'}, 'zone.rpz.', { refresh=1 })
     """ % (rpzServerPort)
     _confdir = 'RPZ'
     _config_template = """