rec: Add support for multiple rpz masters as failover

diff --git a/pdns/rec-lua-conf.cc b/pdns/rec-lua-conf.cc
index daa25420095c9fe7912195d528b306a1c5fbb0e7..6f79ab01bcec4e78b48ed08604dc439df81d5d9d 100644 (file)
--- a/pdns/rec-lua-conf.cc
+++ b/pdns/rec-lua-conf.cc
@@ -92,7 +92,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
   if(!ifs)
     throw PDNSException("Cannot open file '"+fname+"': "+strerror(errno));
 
-  std::vector<std::tuple<ComboAddress, boost::optional<DNSFilterEngine::Policy>, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, std::shared_ptr<SOARecordContent>, std::string> > rpzMasterThreads;
+  std::vector<std::tuple<std::vector<ComboAddress>, boost::optional<DNSFilterEngine::Policy>, uint32_t, size_t, TSIGTriplet, size_t, ComboAddress, uint16_t, std::shared_ptr<SOARecordContent>, std::string> > rpzMasterThreads;
 
   auto luaconfsLocal = g_luaconfs.getLocal();
   lci.generation = luaconfsLocal->generation + 1;
@@ -139,7 +139,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       }
     });
 
-  Lua.writeFunction("rpzMaster", [&lci, &rpzMasterThreads](const string& master_, const string& zoneName, const boost::optional<std::unordered_map<string,boost::variant<uint32_t, string>>>& options) {
+  Lua.writeFunction("rpzMaster", [&lci, &rpzMasterThreads](const boost::variant<string, std::vector<std::pair<int, string> > >& masters_, const string& zoneName, const boost::optional<std::unordered_map<string,boost::variant<uint32_t, string>>>& options) {
 
       boost::optional<DNSFilterEngine::Policy> defpol;
       std::shared_ptr<DNSFilterEngine::Zone> zone = std::make_shared<DNSFilterEngine::Zone>();
@@ -149,7 +149,16 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
       uint16_t axfrTimeout = 20;
       uint32_t maxTTL = std::numeric_limits<uint32_t>::max();
       ComboAddress localAddress;
-      ComboAddress master(master_, 53);
+      std::vector<ComboAddress> masters;
+      if (masters_.type() == typeid(string)) {
+        masters.push_back(ComboAddress(boost::get<std::string>(masters_), 53));
+      }
+      else {
+        for (const auto& master : boost::get<std::vector<std::pair<int, std::string>>>(masters_)) {
+          masters.push_back(ComboAddress(master.second, 53));
+        }
+      }
+
       size_t zoneIdx;
       std::string dumpFile;
       std::shared_ptr<SOARecordContent> sr = nullptr;
@@ -198,9 +207,13 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
           }
         }
 
-        if (localAddress != ComboAddress() && localAddress.sin4.sin_family != master.sin4.sin_family) {
-          // We were passed a localAddress, check if its AF matches the master's
-          throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
+        if (localAddress != ComboAddress()) {
+          // We were passed a localAddress, check if its AF matches the masters'
+          for (const auto& master : masters) {
+            if (localAddress.sin4.sin_family != master.sin4.sin_family) {
+              throw PDNSException("Master address("+master.toString()+") is not of the same Address Family as the local address ("+localAddress.toString()+").");
+            }
+          }
         }
 
         DNSName domain(zoneName);
@@ -236,7 +249,7 @@ void loadRecursorLuaConfig(const std::string& fname, bool checkOnly)
         exit(1);  // FIXME proper exit code?
       }
 
-      rpzMasterThreads.push_back(std::make_tuple(master, defpol, maxTTL, zoneIdx, tt, maxReceivedXFRMBytes, localAddress, axfrTimeout, sr, dumpFile));
+      rpzMasterThreads.push_back(std::make_tuple(masters, defpol, maxTTL, zoneIdx, tt, maxReceivedXFRMBytes, localAddress, axfrTimeout, sr, dumpFile));
     });
 
   typedef vector<pair<int,boost::variant<string, vector<pair<int, string> > > > > argvec_t;

diff --git a/pdns/recursordist/docs/lua-config/rpz.rst b/pdns/recursordist/docs/lua-config/rpz.rst
index 40abc87612f1940e69d8e29d145cd16c4993ed7d..fe1785c15172bf017fa404d4024bd22339843133 100644 (file)
--- a/pdns/recursordist/docs/lua-config/rpz.rst
+++ b/pdns/recursordist/docs/lua-config/rpz.rst
@@ -33,9 +33,13 @@ In this example, 'policy.rpz' denotes the name of the zone to query for.
 
 .. function:: rpzMaster(address, name, settings)
 
+  .. versionchanged:: 4.2.0:
+
+    The first parameter can be a list of addresses.
+
   Load an RPZ from AXFR and keep retrieving with IXFR.
 
-  :param str address: The IP address to transfer the RPZ from
+  :param str address: The IP address to transfer the RPZ from. Also accepts a list of addresses since 4.2.0 in which case they will be tried one after another in the submitted order until a response is obtained
   :param str name: The name of this RPZ
   :param {} settings: A table to settings, see below
 

diff --git a/pdns/rpzloader.cc b/pdns/rpzloader.cc
index 9f26311f3f9bf14cb4df12d1f143c039fa2b4c25..6ecc9c3244c144b1f3679d06034e888bac5aba57 100644 (file)
--- a/pdns/rpzloader.cc
+++ b/pdns/rpzloader.cc
@@ -175,7 +175,7 @@ void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zon
   }
 }
 
-shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
+static shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, uint16_t axfrTimeout)
 {
   g_log<<Logger::Warning<<"Loading RPZ zone '"<<zoneName<<"' from "<<master.toStringWithPort()<<endl;
   if(!tt.name.empty())
@@ -337,7 +337,7 @@ static bool dumpZoneToDisk(const DNSName& zoneName, const std::shared_ptr<DNSFil
   return true;
 }
 
-void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, std::shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration)
+void RPZIXFRTracker(const std::vector<ComboAddress> masters, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, std::shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration)
 {
   bool isPreloaded = sr != nullptr;
   auto luaconfsLocal = g_luaconfs.getLocal();
@@ -356,29 +356,34 @@ void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine:
 
     /* full copy, as promised */
     std::shared_ptr<DNSFilterEngine::Zone> newZone = std::make_shared<DNSFilterEngine::Zone>(*oldZone);
-    try {
-      sr=loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
-      if(refresh == 0) {
-        refresh = sr->d_st.refresh;
+    for (const auto& master : masters) {
+      try {
+        sr = loadRPZFromServer(master, zoneName, newZone, defpol, maxTTL, tt, maxReceivedBytes, localAddress, axfrTimeout);
+        if(refresh == 0) {
+          refresh = sr->d_st.refresh;
+        }
+        newZone->setSerial(sr->d_st.serial);
+        setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);
+
+        g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
+            lci.dfe.setZone(zoneIdx, newZone);
+          });
+
+        if (!dumpZoneFileName.empty()) {
+          dumpZoneToDisk(zoneName, newZone, dumpZoneFileName);
+        }
+
+        /* no need to try another master */
+        break;
       }
-      newZone->setSerial(sr->d_st.serial);
-      setRPZZoneNewState(polName, sr->d_st.serial, newZone->size(), true);
-
-      g_luaconfs.modify([zoneIdx, &newZone](LuaConfigItems& lci) {
-        lci.dfe.setZone(zoneIdx, newZone);
-      });
-
-      if (!dumpZoneFileName.empty()) {
-        dumpZoneToDisk(zoneName, newZone, dumpZoneFileName);
+      catch(const std::exception& e) {
+        g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
+        incRPZFailedTransfers(polName);
+      }
+      catch(const PDNSException& e) {
+        g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
+        incRPZFailedTransfers(polName);
       }
-    }
-    catch(const std::exception& e) {
-      g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.what()<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
-      incRPZFailedTransfers(polName);
-    }
-    catch(const PDNSException& e) {
-      g_log<<Logger::Warning<<"Unable to load RPZ zone '"<<zoneName<<"' from '"<<master<<"': '"<<e.reason<<"'. (Will try again in "<<(refresh > 0 ? refresh : 10)<<" seconds...)"<<endl;
-      incRPZFailedTransfers(polName);
     }
 
     if (!sr) {
@@ -411,22 +416,31 @@ void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine:
       return;
     }
 
-    g_log<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
     vector<pair<vector<DNSRecord>, vector<DNSRecord> > > deltas;
+    for (const auto& master : masters) {
+      g_log<<Logger::Info<<"Getting IXFR deltas for "<<zoneName<<" from "<<master.toStringWithPort()<<", our serial: "<<getRR<SOARecordContent>(dr)->d_st.serial<<endl;
+
+      ComboAddress local(localAddress);
+      if (local == ComboAddress()) {
+        local = getQueryLocalAddress(master.sin4.sin_family, 0);
+      }
 
-    ComboAddress local(localAddress);
-    if (local == ComboAddress())
-      local = getQueryLocalAddress(master.sin4.sin_family, 0);
+      try {
+        deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
 
-    try {
-      deltas = getIXFRDeltas(master, zoneName, dr, tt, &local, maxReceivedBytes);
-    } catch(std::runtime_error& e ){
-      g_log<<Logger::Warning<<e.what()<<endl;
-      incRPZFailedTransfers(polName);
-      continue;
+        /* no need to try another master */
+        break;
+      } catch(const std::runtime_error& e ){
+        g_log<<Logger::Warning<<e.what()<<endl;
+        incRPZFailedTransfers(polName);
+        continue;
+      }
     }
-    if(deltas.empty())
+
+    if(deltas.empty()) {
       continue;
+    }
+
     g_log<<Logger::Info<<"Processing "<<deltas.size()<<" delta"<<addS(deltas)<<" for RPZ "<<zoneName<<endl;
 
     oldZone = luaconfsLocal->dfe.getZone(zoneIdx);

diff --git a/pdns/rpzloader.hh b/pdns/rpzloader.hh
index 5faf77d8b1dbd95c9ad343b2fb430c9492cf79f4..b29f145011e127bb8d9160e14b2217d06c44a684 100644 (file)
--- a/pdns/rpzloader.hh
+++ b/pdns/rpzloader.hh
@@ -27,9 +27,8 @@
 extern bool g_logRPZChanges;
 
 std::shared_ptr<SOARecordContent> loadRPZFromFile(const std::string& fname, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL);
-std::shared_ptr<SOARecordContent> loadRPZFromServer(const ComboAddress& master, const DNSName& zoneName, std::shared_ptr<DNSFilterEngine::Zone> zone, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout);
 void RPZRecordToPolicy(const DNSRecord& dr, std::shared_ptr<DNSFilterEngine::Zone> zone, bool addOrRemove, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL);
-void RPZIXFRTracker(const ComboAddress& master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration);
+void RPZIXFRTracker(const std::vector<ComboAddress> master, boost::optional<DNSFilterEngine::Policy> defpol, uint32_t maxTTL, size_t zoneIdx, const TSIGTriplet& tt, size_t maxReceivedBytes, const ComboAddress& localAddress, const uint16_t axfrTimeout, shared_ptr<SOARecordContent> sr, std::string dumpZoneFileName, uint64_t configGeneration);
 
 struct rpzStats
 {

diff --git a/regression-tests.recursor-dnssec/test_RPZ.py b/regression-tests.recursor-dnssec/test_RPZ.py
index 9ea04fe932185815c8cbee6bac67995dad366c00..b2ca1bf96cd1e8979cd0cf940d1e2c6e538fa586 100644 (file)
--- a/regression-tests.recursor-dnssec/test_RPZ.py
+++ b/regression-tests.recursor-dnssec/test_RPZ.py
@@ -150,7 +150,8 @@ class RPZRecursorTest(RecursorTest):
 
     global rpzServerPort
     _lua_config_file = """
-    rpzMaster('127.0.0.1:%d', 'zone.rpz.', { refresh=1 })
+    -- The first server is a bogus one, to test that we correctly fail over to the second one
+    rpzMaster({'127.0.0.1:9999', '127.0.0.1:%d'}, 'zone.rpz.', { refresh=1 })
     """ % (rpzServerPort)
     _confdir = 'RPZ'
     _config_template = """