From ab7a799369b73ab1f46f35169e0110e2479cd567 Mon Sep 17 00:00:00 2001 From: Michael Friedrich Date: Fri, 22 Feb 2019 15:53:38 +0100 Subject: [PATCH] Implement ReloadTimeout constant and wait for enqueued checks on Stop() --- doc/17-language-reference.md | 1 + lib/base/application.cpp | 17 ++++++++++++++--- lib/base/configuration.cpp | 11 +++++++++++ lib/base/configuration.hpp | 4 ++++ lib/base/configuration.ti | 5 +++++ lib/checker/checkercomponent.cpp | 29 ++++++++++++++++++++++++++--- lib/remote/configpackageutility.cpp | 2 +- 7 files changed, 62 insertions(+), 7 deletions(-) diff --git a/doc/17-language-reference.md b/doc/17-language-reference.md index 039e3ad31..775684a1a 100644 --- a/doc/17-language-reference.md +++ b/doc/17-language-reference.md @@ -440,6 +440,7 @@ Constant | Description --------------------|------------------- Vars |**Read-write.** Contains a dictionary with global custom attributes. Not set by default. NodeName |**Read-write.** Contains the cluster node name. Set to the local hostname by default. +ReloadTimeout |**Read-write.** Defines the reload timeout for child processes. Defaults to `300s`. Environment |**Read-write.** The name of the Icinga environment. Included in the SNI host name for outbound connections. Not set by default. RunAsUser |**Read-write.** Defines the user the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig. RunAsGroup |**Read-write.** Defines the group the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig. diff --git a/lib/base/application.cpp b/lib/base/application.cpp index df93710b7..62ed10cc5 100644 --- a/lib/base/application.cpp +++ b/lib/base/application.cpp @@ -77,6 +77,14 @@ void Application::OnConfigLoaded() ASSERT(m_Instance == nullptr); m_Instance = this; + + String reloadTimeout; + + if (ScriptGlobal::Exists("ReloadTimeout")) + reloadTimeout = ScriptGlobal::Get("ReloadTimeout"); + + if (!reloadTimeout.IsEmpty()) + Configuration::ReloadTimeout = Convert::ToDouble(reloadTimeout); } /** @@ -401,8 +409,6 @@ static void ReloadProcessCallback(const ProcessResult& pr) pid_t Application::StartReloadProcess() { - Log(LogInformation, "Application", "Got reload command: Starting new instance."); - // prepare arguments ArrayData args; args.push_back(GetExePath(m_ArgV[0])); @@ -422,9 +428,14 @@ pid_t Application::StartReloadProcess() #endif /* _WIN32 */ Process::Ptr process = new Process(Process::PrepareCommand(new Array(std::move(args)))); - process->SetTimeout(300); + process->SetTimeout(Configuration::ReloadTimeout); process->Run(&ReloadProcessCallback); + Log(LogInformation, "Application") + << "Got reload command: Started new instance with PID '" + << (unsigned long)(process->GetPID()) << "' (timeout is " + << Configuration::ReloadTimeout << "s)."; + return process->GetPID(); } diff --git a/lib/base/configuration.cpp b/lib/base/configuration.cpp index 0103028de..c22b2f46b 100644 --- a/lib/base/configuration.cpp +++ b/lib/base/configuration.cpp @@ -42,6 +42,7 @@ String Configuration::PidPath; String Configuration::PkgDataDir; String Configuration::PrefixDir; String Configuration::ProgramData; +double Configuration::ReloadTimeout{300}; int Configuration::RLimitFiles; int Configuration::RLimitProcesses; int Configuration::RLimitStack; @@ -240,6 +241,16 @@ void Configuration::SetProgramData(const String& val, bool suppress_events, cons HandleUserWrite("ProgramData", &Configuration::ProgramData, val, m_ReadOnly); } +double Configuration::GetReloadTimeout() const +{ + return Configuration::ReloadTimeout; +} + +void Configuration::SetReloadTimeout(double val, bool suppress_events, const Value& cookie) +{ + HandleUserWrite("ReloadTimeout", &Configuration::ReloadTimeout, val, m_ReadOnly); +} + int Configuration::GetRLimitFiles() const { return Configuration::RLimitFiles; diff --git a/lib/base/configuration.hpp b/lib/base/configuration.hpp index 96ab2726f..694a02a8c 100644 --- a/lib/base/configuration.hpp +++ b/lib/base/configuration.hpp @@ -87,6 +87,9 @@ public: String GetProgramData() const override; void SetProgramData(const String& value, bool suppress_events = false, const Value& cookie = Empty) override; + double GetReloadTimeout() const override; + void SetReloadTimeout(double value, bool suppress_events = false, const Value& cookie = Empty) override; + int GetRLimitFiles() const override; void SetRLimitFiles(int value, bool suppress_events = false, const Value& cookie = Empty) override; @@ -147,6 +150,7 @@ public: static String PkgDataDir; static String PrefixDir; static String ProgramData; + static double ReloadTimeout; static int RLimitFiles; static int RLimitProcesses; static int RLimitStack; diff --git a/lib/base/configuration.ti b/lib/base/configuration.ti index 22f121411..356c7fa6b 100644 --- a/lib/base/configuration.ti +++ b/lib/base/configuration.ti @@ -111,6 +111,11 @@ abstract class Configuration set; }; + [config, no_storage, virtual] double ReloadTimeout { + get; + set; + }; + [config, no_storage, virtual] int RLimitFiles { get; set; diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp index cde975b72..31bf2b20f 100644 --- a/lib/checker/checkercomponent.cpp +++ b/lib/checker/checkercomponent.cpp @@ -84,18 +84,41 @@ void CheckerComponent::Start(bool runtimeCreated) void CheckerComponent::Stop(bool runtimeRemoved) { - Log(LogInformation, "CheckerComponent") - << "'" << GetName() << "' stopped."; - { boost::mutex::scoped_lock lock(m_Mutex); m_Stopped = true; m_CV.notify_all(); } + double wait = 0.0; + + while (GetPendingCheckables() > 0) { + Log(LogDebug, "CheckerComponent") + << "Waiting for running checks (" << GetPendingCheckables() + << ") to finish. Waited for " << wait << " seconds now."; + + Utility::Sleep(0.1); + wait += 0.1; + + /* Pick a timeout slightly shorther than the process reload timeout. */ + double waitMax = Configuration::ReloadTimeout - 30; + if (waitMax <= 0) + waitMax = 1; + + if (wait > waitMax) { + Log(LogWarning, "CheckerComponent") + << "Checks running too long for " << wait + << " seconds, hard shutdown before reload timeout: " << Configuration::ReloadTimeout << "."; + break; + } + } + m_ResultTimer->Stop(); m_Thread.join(); + Log(LogInformation, "CheckerComponent") + << "'" << GetName() << "' stopped."; + ObjectImpl::Stop(runtimeRemoved); } diff --git a/lib/remote/configpackageutility.cpp b/lib/remote/configpackageutility.cpp index 3b15724d3..d103b413a 100644 --- a/lib/remote/configpackageutility.cpp +++ b/lib/remote/configpackageutility.cpp @@ -235,7 +235,7 @@ void ConfigPackageUtility::AsyncTryActivateStage(const String& packageName, cons args->Add("ActiveStageOverride=" + packageName + ":" + stageName); Process::Ptr process = new Process(Process::PrepareCommand(args)); - process->SetTimeout(300); + process->SetTimeout(Configuration::ReloadTimeout); process->Run(std::bind(&TryActivateStageCallback, _1, packageName, stageName, reload)); } -- 2.40.0