X-Git-Url: https://granicus.if.org/sourcecode?a=blobdiff_plain;ds=sidebyside;f=lib%2Fremote%2Fapilistener-filesync.cpp;h=4b31673f699d44527014fa292d12efda50dfd98e;hb=043824a6a97153602a05eb78b4f021fcecc9d93b;hp=9e73c9bb818d10fbfd82067f39ebe3ae55098a2f;hpb=e8e4fe75087e6aa8f01cf542358f8fcb668d220f;p=icinga2 diff --git a/lib/remote/apilistener-filesync.cpp b/lib/remote/apilistener-filesync.cpp index 9e73c9bb8..4b31673f6 100644 --- a/lib/remote/apilistener-filesync.cpp +++ b/lib/remote/apilistener-filesync.cpp @@ -7,6 +7,7 @@ #include "base/logger.hpp" #include "base/convert.hpp" #include "base/exception.hpp" +#include "base/utility.hpp" #include #include @@ -14,6 +15,16 @@ using namespace icinga; REGISTER_APIFUNCTION(Update, config, &ApiListener::ConfigUpdateHandler); +boost::mutex ApiListener::m_ConfigSyncStageLock; + +/** + * Read the given file and store it in the config information structure. + * Callback function for Glob(). + * + * @param config Reference to the config information object. + * @param path File path. + * @param file Full file name. + */ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& path, const String& file) { CONTEXT("Creating config update for file '" + file + "'"); @@ -29,6 +40,11 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& Dictionary::Ptr update; + /* + * 'update' messages contain conf files. 'update_v2' syncs everything else (.timestamp). + * + * **Keep this intact to stay compatible with older clients.** + */ if (Utility::Match("*.conf", file)) update = config.UpdateV1; else @@ -37,6 +53,12 @@ void ApiListener::ConfigGlobHandler(ConfigDirInformation& config, const String& update->Set(file.SubStr(path.GetLength()), content); } +/** + * Compatibility helper for merging config update v1 and v2 into a global result. + * + * @param config Config information structure. + * @returns Dictionary which holds the merged information. + */ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& config) { Dictionary::Ptr result = new Dictionary(); @@ -50,6 +72,12 @@ Dictionary::Ptr ApiListener::MergeConfigUpdate(const ConfigDirInformation& confi return result; } +/** + * Load the given config dir and read their file content into the config structure. + * + * @param dir Path to the config directory. + * @returns ConfigInformation structure. + */ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) { ConfigDirInformation config; @@ -59,7 +87,25 @@ ConfigDirInformation ApiListener::LoadConfigDir(const String& dir) return config; } -bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, const String& configDir, bool authoritative) +/** + * Diffs the old current configuration with the new configuration + * and copies the collected content. Detects whether a change + * happened, this is used for later restarts. + * + * This generic function is called in two situations: + * - Local zones.d to var/lib/api/zones copy on the master (authoritative: true) + * - Received config update on a cluster node (authoritative: false) + * + * @param oldConfigInfo Config information struct for the current old deployed config. + * @param newConfigInfo Config information struct for the received synced config. + * @param configDir Destination for copying new files (production, or stage dir). + * @param zoneName Currently processed zone, for storing the relative paths for later. + * @param relativePaths Reference which stores all updated config path destinations. + * @param Whether we're authoritative for this config. + * @returns Whether a config change happened. + */ +bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, const ConfigDirInformation& newConfigInfo, + const String& configDir, const String& zoneName, std::vector& relativePaths, bool authoritative) { bool configChange = false; @@ -102,6 +148,9 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con if (!Utility::Match("*/.timestamp", kv.first)) configChange = true; + /* Store the relative config file path for later. */ + relativePaths.push_back(zoneName + "/" + kv.first); + String path = configDir + "/" + kv.first; Log(LogInformation, "ApiListener") << "Updating configuration file: " << path; @@ -120,14 +169,17 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con } } + /* Log something whether we're authoritative or receing a staged config. */ Log(LogInformation, "ApiListener") - << "Applying configuration file update for path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" + << "Applying configuration file update for " << (authoritative ? "" : "stage ") + << "path '" << configDir << "' (" << numBytes << " Bytes). Received timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", newTimestamp) << "' (" << std::fixed << std::setprecision(6) << newTimestamp << "), Current timestamp '" << Utility::FormatDateTime("%Y-%m-%d %H:%M:%S %z", oldTimestamp) << "' (" << oldTimestamp << ")."; + /* If the update removes a path, delete it on disk. */ ObjectLock xlock(oldConfig); for (const Dictionary::Pair& kv : oldConfig) { if (!newConfig->Contains(kv.first)) { @@ -138,6 +190,8 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con } } + /* Consider that one of the paths leaves an empty directory here. Such is not copied from stage to prod and purged then automtically. */ + String tsPath = configDir + "/.timestamp"; if (!Utility::PathExists(tsPath)) { std::ofstream fp(tsPath.CStr(), std::ofstream::out | std::ostream::trunc); @@ -156,13 +210,29 @@ bool ApiListener::UpdateConfigDir(const ConfigDirInformation& oldConfigInfo, con return configChange; } +/** + * Sync a zone directory where we have an authoritative copy (zones.d, etc.) + * + * This function collects the registered zone config dirs from + * the config compiler and reads the file content into the config + * information structure. + * + * Returns early when there are no updates. + * + * @param zone Pointer to the zone object being synced. + */ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const { + if (!zone) + return; + ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = new Dictionary(); newConfigInfo.UpdateV2 = new Dictionary(); - for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zone->GetName())) { + String zoneName = zone->GetName(); + + for (const ZoneFragment& zf : ConfigCompiler::GetZoneDirs(zoneName)) { ConfigDirInformation newConfigPart = LoadConfigDir(zf.Path); { @@ -185,18 +255,25 @@ void ApiListener::SyncZoneDir(const Zone::Ptr& zone) const if (sumUpdates == 0) return; - String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName(); + String currentDir = Configuration::DataDir + "/api/zones/" + zoneName; Log(LogInformation, "ApiListener") - << "Copying " << sumUpdates << " zone configuration files for zone '" << zone->GetName() << "' to '" << oldDir << "'."; + << "Copying " << sumUpdates << " zone configuration files for zone '" << zoneName << "' to '" << currentDir << "'."; - Utility::MkDirP(oldDir, 0700); + ConfigDirInformation oldConfigInfo = LoadConfigDir(currentDir); - ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir); + /* Purge files to allow deletion via zones.d. */ + Utility::RemoveDirRecursive(currentDir); + Utility::MkDirP(currentDir, 0700); - UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, true); + std::vector relativePaths; + UpdateConfigDir(oldConfigInfo, newConfigInfo, currentDir, zoneName, relativePaths, true); } +/** + * Entrypoint for updating all authoritative configs into var/lib/icinga2/api/zones + * + */ void ApiListener::SyncZoneDirs() const { for (const Zone::Ptr& zone : ConfigType::GetObjectsByType()) { @@ -208,6 +285,14 @@ void ApiListener::SyncZoneDirs() const } } +/** + * Entrypoint for sending a file based config update to a cluster client. + * This includes security checks for zone relations. + * Loads the zone config files where this client belongs to + * and sends the 'config::Update' JSON-RPC message. + * + * @param aclient Connected JSON-RPC client. + */ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) { Endpoint::Ptr endpoint = aclient->GetEndpoint(); @@ -255,8 +340,21 @@ void ApiListener::SendConfigUpdate(const JsonRpcConnection::Ptr& aclient) aclient->SendMessage(message); } +/** + * Registered handler when a new config::Update message is received. + * + * Checks destination and permissions first, then analyses the update. + * The newly received configuration is not copied to production immediately, + * but into the staging directory first. + * Last, the async validation and restart is triggered. + * + * @param origin Where this message came from. + * @param params Message parameters including the config updates. + * @returns Empty, required by the interface. + */ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const Dictionary::Ptr& params) { + /* Verify permissions and trust relationship. */ if (!origin->FromClient->GetEndpoint() || (origin->FromZone && !Zone::GetLocalZone()->IsChildOf(origin->FromZone))) return Empty; @@ -273,6 +371,11 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D return Empty; } + /* Only one transaction is allowed, concurrent message handlers need to wait. + * This affects two parent endpoints sending the config in the same moment. + */ + boost::mutex::scoped_lock lock(m_ConfigSyncStageLock); + Log(LogInformation, "ApiListener") << "Applying config update from endpoint '" << origin->FromClient->GetEndpoint()->GetName() << "' of zone '" << GetFromZoneName(origin->FromZone) << "'."; @@ -281,44 +384,205 @@ Value ApiListener::ConfigUpdateHandler(const MessageOrigin::Ptr& origin, const D Dictionary::Ptr updateV2 = params->Get("update_v2"); bool configChange = false; + std::vector relativePaths; + /* + * We can and must safely purge the staging directory, as the difference is taken between + * runtime production config and newly received configuration. + */ + String apiZonesStageDir = GetApiZonesStageDir(); + + if (Utility::PathExists(apiZonesStageDir)) + Utility::RemoveDirRecursive(apiZonesStageDir); + + Utility::MkDirP(apiZonesStageDir, 0700); + + /* Analyse and process the update. */ ObjectLock olock(updateV1); for (const Dictionary::Pair& kv : updateV1) { - Zone::Ptr zone = Zone::GetByName(kv.first); + + /* Check for the configured zones. */ + String zoneName = kv.first; + Zone::Ptr zone = Zone::GetByName(zoneName); if (!zone) { Log(LogWarning, "ApiListener") - << "Ignoring config update for unknown zone '" << kv.first << "'."; + << "Ignoring config update for unknown zone '" << zoneName << "'."; continue; } - if (ConfigCompiler::HasZoneConfigAuthority(kv.first)) { + /* Whether we already have configuration in zones.d. */ + if (ConfigCompiler::HasZoneConfigAuthority(zoneName)) { Log(LogWarning, "ApiListener") - << "Ignoring config update for zone '" << kv.first << "' because we have an authoritative version of the zone's config."; + << "Ignoring config update for zone '" << zoneName << "' because we have an authoritative version of the zone's config."; continue; } - String oldDir = Configuration::DataDir + "/api/zones/" + zone->GetName(); + /* Put the received configuration into our stage directory. */ + String currentConfigDir = GetApiZonesDir() + zoneName; + String stageConfigDir = GetApiZonesStageDir() + zoneName; - Utility::MkDirP(oldDir, 0700); + Utility::MkDirP(currentConfigDir, 0700); + Utility::MkDirP(stageConfigDir, 0700); + /* Merge the config information. */ ConfigDirInformation newConfigInfo; newConfigInfo.UpdateV1 = kv.second; if (updateV2) newConfigInfo.UpdateV2 = updateV2->Get(kv.first); - Dictionary::Ptr newConfig = kv.second; - ConfigDirInformation oldConfigInfo = LoadConfigDir(oldDir); + /* Load the current production config details. */ + ConfigDirInformation currentConfigInfo = LoadConfigDir(currentConfigDir); - if (UpdateConfigDir(oldConfigInfo, newConfigInfo, oldDir, false)) + /* Diff the current production configuration with the received configuration. + * If there was a change, collect a signal for later stage validation. + */ + if (UpdateConfigDir(currentConfigInfo, newConfigInfo, stageConfigDir, zoneName, relativePaths, false)) configChange = true; } if (configChange) { - Log(LogInformation, "ApiListener", "Restarting after configuration change."); - Application::RequestRestart(); + /* Spawn a validation process. On success, move the staged configuration + * into production and restart. + */ + AsyncTryActivateZonesStage(GetApiZonesStageDir(), GetApiZonesDir(), relativePaths); } return Empty; } + +/** + * Callback for stage config validation. + * When validation was successful, the configuration is copied from + * stage to production and a restart is triggered. + * On failure, there's no restart and this is logged. + * + * @param pr Result of the validation process. + * @param stageConfigDir TODO + * @param currentConfigDir TODO + * @param relativePaths Collected paths which are copied from stage to current. + */ +void ApiListener::TryActivateZonesStageCallback(const ProcessResult& pr, + const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths) +{ + String logFile = GetApiZonesStageDir() + "/startup.log"; + std::ofstream fpLog(logFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fpLog << pr.Output; + fpLog.close(); + + String statusFile = GetApiZonesStageDir() + "/status"; + std::ofstream fpStatus(statusFile.CStr(), std::ofstream::out | std::ostream::binary | std::ostream::trunc); + fpStatus << pr.ExitStatus; + fpStatus.close(); + + /* validation went fine, copy stage and reload */ + if (pr.ExitStatus == 0) { + Log(LogInformation, "ApiListener") + << "Config validation for stage '" << GetApiZonesStageDir() << "' was OK, replacing into '" << GetApiZonesDir() << "' and triggering reload."; + + String apiZonesDir = GetApiZonesDir(); + + /* Purge production before copying stage. */ + if (Utility::PathExists(apiZonesDir)) + Utility::RemoveDirRecursive(apiZonesDir); + + Utility::MkDirP(apiZonesDir, 0700); + + /* Copy all synced configuration files from stage to production. */ + for (const String& path : relativePaths) { + Log(LogNotice, "ApiListener") + << "Copying file '" << path << "' from config sync staging to production zones directory."; + + String stagePath = GetApiZonesStageDir() + path; + String currentPath = GetApiZonesDir() + path; + + Utility::MkDirP(Utility::DirName(currentPath), 0700); + + Utility::CopyFile(stagePath, currentPath); + } + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) + listener->ClearLastFailedZonesStageValidation(); + + Application::RequestRestart(); + + return; + } + + /* Error case. */ + Log(LogCritical, "ApiListener") + << "Config validation failed for staged cluster config sync in '" << GetApiZonesStageDir() + << "'. Aborting. Logs: '" << logFile << "'"; + + ApiListener::Ptr listener = ApiListener::GetInstance(); + + if (listener) + listener->UpdateLastFailedZonesStageValidation(pr.Output); +} + +/** + * Spawns a new validation process and waits for its output. + * Sets 'System.ZonesStageVarDir' to override the config validation zone dirs with our current stage. + * + * @param stageConfigDir TODO + * @param currentConfigDir TODO + * @param relativePaths Required for later file operations in the callback. + */ +void ApiListener::AsyncTryActivateZonesStage(const String& stageConfigDir, const String& currentConfigDir, + const std::vector& relativePaths) +{ + VERIFY(Application::GetArgC() >= 1); + + /* Inherit parent process args. */ + Array::Ptr args = new Array({ + Application::GetExePath(Application::GetArgV()[0]), + }); + + for (int i = 1; i < Application::GetArgC(); i++) { + String argV = Application::GetArgV()[i]; + + if (argV == "-d" || argV == "--daemonize") + continue; + + args->Add(argV); + } + + args->Add("--validate"); + + /* Set the ZonesStageDir. This creates our own local chroot without any additional automated zone includes. */ + args->Add("--define"); + args->Add("System.ZonesStageVarDir=" + GetApiZonesStageDir()); + + Process::Ptr process = new Process(Process::PrepareCommand(args)); + process->SetTimeout(300); + process->Run(std::bind(&TryActivateZonesStageCallback, _1, stageConfigDir, currentConfigDir, relativePaths)); +} + +/** + * Update the structure from the last failed validation output. + * Uses the current timestamp. + * + * @param log The process output from the config validation. + */ +void ApiListener::UpdateLastFailedZonesStageValidation(const String& log) +{ + Dictionary::Ptr lastFailedZonesStageValidation = new Dictionary({ + { "log", log }, + { "ts", Utility::GetTime() } + }); + + SetLastFailedZonesStageValidation(lastFailedZonesStageValidation); +} + +/** + * Clear the structure for the last failed reload. + * + */ +void ApiListener::ClearLastFailedZonesStageValidation() +{ + SetLastFailedZonesStageValidation(Dictionary::Ptr()); +}