Implement first draft for cluster config staged sync

diff --git a/lib/remote/CMakeLists.txt b/lib/remote/CMakeLists.txt
index da0006aa19e14fe9e0528c145c7ba68924a590bd..2c5a0326a3178135eff9c925373853737f3ae9ad 100644 (file)
--- a/lib/remote/CMakeLists.txt
+++ b/lib/remote/CMakeLists.txt
@@ -59,6 +59,7 @@ set_target_properties (
 #install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api\")")
 install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/log\")")
 install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/zones\")")
+install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/api/zones-stage\")")
 install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/certs\")")
 install(CODE "file(MAKE_DIRECTORY \"\$ENV{DESTDIR}${ICINGA2_FULL_DATADIR}/certificate-requests\")")
 

diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp
index 411028630f9a856e12a5065987d7815ce5e328eb..03fc6847528c7666b1cf7073ff6fa61e8c7503b8 100644 (file)
--- a/lib/remote/apilistener-filesync.cpp
+++ b/lib/remote/apilistener-filesync.cpp
@@ -62,7 +62,8 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir)
        return config;
 }
 
-bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, const String& configDir, bool authoritative)
+bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo,
+       const String& configDir, const String& zoneName, std::vector<String>& relativePaths, bool authoritative)
 {
        bool configChange = false;
 
@@ -105,6 +106,9 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con
                                if (!Utility::Match("*/.timestamp", kv.first))
                                        configChange = true;
 
+                               /* Store the relative config file path for later. */
+                               relativePaths.push_back(zoneName + "/" + kv.first);
+
                                String path = configDir + "/" + kv.first;
                                Log(LogInformation, "ApiListener")
                                        << "Updating configuration file: " << path;
@@ -123,6 +127,7 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con
                }
        }
 
+       /* Update with staging information TODO - use `authoritative` as flag. */
        Log(LogInformation, "ApiListener")
                << "Applying configuration file update for path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '"
                << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' ("
@@ -197,7 +202,8 @@ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const
 
        ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
 
-       UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, true);
+       std::vector<String> relativePaths;
+       UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, zone->GetName(), relativePaths, true);
 }
 
 void ApiListener::SyncZoneDirs() const
@@ -289,26 +295,34 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
        Dictionary::Ptr updateV2 = params->Get("update_v2");
 
        bool configChange = false;
+       std::vector<String> relativePaths;
 
        ObjectLock olock(updateV1);
        for (const Dictionary::Pair& kv : updateV1) {
+
+               /* Check for the configured zones. */
                Zone::Ptr zone = Zone::GetByName(kv.first);
+               String zoneName = zone->GetName();
 
                if (!zone) {
                        Log(LogWarning, "ApiListener")
-                               << "Ignoring config update for unknown zone '" << kv.first << "'.";
+                               << "Ignoring config update for unknown zone '" << zoneName << "'.";
                        continue;
                }
 
+               /* Whether we already have configuration in zones.d. */
                if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) {
                        Log(LogWarning, "ApiListener")
-                               << "Ignoring config update for zone '" << kv.first << "' because we have an authoritative version of the zone's config.";
+                               << "Ignoring config update for zone '" << zoneName << "' because we have an authoritative version of the zone's config.";
                        continue;
                }
 
-               String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName();
+               /* Put the received configuration into our stage directory. */
+               String currentConfigDir = GetApiZonesDir() + zoneName;
+               String stageConfigDir = GetApiZonesStageDir() + zoneName;
 
-               Utility::MkDirP(oldDir, 0700);
+               Utility::MkDirP(currentConfigDir, 0700);
+               Utility::MkDirP(stageConfigDir, 0700);
 
                ConfigDirInformation newConfigInfo;
                newConfigInfo.UpdateV1 = kv.second;
@@ -317,16 +331,70 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D
                        newConfigInfo.UpdateV2 = updateV2->Get(kv.first);
 
                Dictionary::Ptr newConfig = kv.second;
-               ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir);
+               ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir);
 
-               if (UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, false))
+               /* Move the received configuration into our stage directory first. */
+               if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false))
                        configChange = true;
        }
 
        if (configChange) {
-               Log(LogInformation, "ApiListener", "Restarting after configuration change.");
-               Application::RequestRestart();
+               /* Spawn a validation process. On success, move the staged configuration
+                * into production and restart.
+                */
+               AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths, true);
        }
 
        return Empty;
 }
+
+void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr,
+       const String& stageConfigDir, const String& currentConfigDir,
+       const std::vector<String>& relativePaths, bool reload)
+{
+       String logFile = GetApiZonesStageDir() + "/startup.log";
+       std::ofstream fpLog(logFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc);
+       fpLog << pr.Output;
+       fpLog.close();
+
+       String statusFile = GetApiZonesStageDir() + "/status";
+       std::ofstream fpStatus(statusFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc);
+       fpStatus << pr.ExitStatus;
+       fpStatus.close();
+
+       /* validation went fine, copy stage and reload */
+       if (pr.ExitStatus == 0) {
+               for (const String& path : relativePaths) {
+                       /* TODO: Better error handling with existing files. */
+                       Log(LogCritical, "ApiListener")
+                               << "Copying file '" << path << "' from config sync staging to production directory.";
+
+                       Utility::CopyFile(GetApiZonesStageDir() + path, GetApiZonesDir() + path);
+               }
+
+               if (reload)
+                       Application::RequestRestart();
+       } else {
+               Log(LogCritical, "ApiListener")
+                       << "Config validation failed for staged cluster config sync. Stage not put in production, aborting.";
+       }
+}
+
+void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir,
+       const std::vector<String>& relativePaths, bool reload)
+{
+       VERIFY(Application::GetArgC() >= 1);
+
+       // prepare arguments
+       Array::Ptr args = new Array({
+               Application::GetExePath(Application::GetArgV()[0]),
+               "daemon",
+               "--validate",
+               "--define",
+               "ZonesDir=" + GetApiZonesStageDir()
+       });
+
+       Process::Ptr process = new Process(Process::PrepareCommand(args));
+       process->SetTimeout(300);
+       process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths, reload));
+}

diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp
index a7f0f66a7fd18b28454fe145fc73d0c34931a68a..b9f8b9780cf059e43eeab39f11d2883b04fda0b6 100644 (file)
--- a/lib/remote/apilistener.cpp
+++ b/lib/remote/apilistener.cpp
@@ -59,6 +59,16 @@ String ApiListener::GetApiDir()
        return Configuration::DataDir + "/api/";
 }
 
+String ApiListener::GetApiZonesDir()
+{
+       return GetApiDir() + "zones/";
+}
+
+String ApiListener::GetApiZonesStageDir()
+{
+       return GetApiDir() + "zones-stage/";
+}
+
 String ApiListener::GetCertsDir()
 {
        return Configuration::DataDir + "/certs/";

diff --git a/lib/remote/apilistener.hpp b/lib/remote/apilistener.hpp
index 022ad8281aad15505c32455fc10bee2d233f55a4..e8512cd6ba4bab7bd9be918dbe0843fc36d782a6 100644 (file)
--- a/lib/remote/apilistener.hpp
+++ b/lib/remote/apilistener.hpp
@@ -9,6 +9,7 @@
 #include "remote/endpoint.hpp"
 #include "remote/messageorigin.hpp"
 #include "base/configobject.hpp"
+#include "base/process.hpp"
 #include "base/timer.hpp"
 #include "base/workqueue.hpp"
 #include "base/tcpsocket.hpp"
@@ -47,6 +48,8 @@ public:
        ApiListener();
 
        static String GetApiDir();
+       static String GetApiZonesDir();
+       static String GetApiZonesStageDir();
        static String GetCertsDir();
        static String GetCaDir();
        static String GetCertificateRequestsDir();
@@ -169,7 +172,8 @@ private:
 
        static ConfigDirInformation LoadConfigDir(const String& dir);
        static Dictionary::Ptr MergeConfigUpdate(const ConfigDirInformation& config);
-       static bool UpdateConfigDir(const ConfigDirInformation& oldConfig, const ConfigDirInformation& newConfig, const String& configDir, bool authoritative);
+       static bool UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo,
+               const String& configDir, const String& zoneName, std::vector<String>& relativePaths, bool authoritative);
 
        void SyncZoneDirs() const;
        void SyncZoneDir(const Zone::Ptr& zone) const;
@@ -177,6 +181,12 @@ private:
        static void ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file);
        void SendConfigUpdate(const JsonRpcConnection::Ptr& aclient);
 
+       static void TryActivateZonesStageCallback(const ProcessResult& pr,
+               const String& stageConfigDir, const String& currentConfigDir,
+               const std::vector<String>& relativePaths, bool reload);
+       static void AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir,
+               const std::vector<String>& relativePaths, bool reload);
+
        /* configsync */
        void UpdateConfigObject(const ConfigObject::Ptr& object, const MessageOrigin::Ptr& origin,
                const JsonRpcConnection::Ptr& client = nullptr);