Implement ReloadTimeout constant and wait for enqueued checks on Stop()

diff --git a/doc/17-language-reference.md b/doc/17-language-reference.md
index 039e3ad319bbc5eea32680eadfceb1d244ec4902..775684a1a83cfbe26776b9932bd841121dbe9985 100644 (file)
--- a/doc/17-language-reference.md
+++ b/doc/17-language-reference.md
@@ -440,6 +440,7 @@ Constant            | Description
 --------------------|-------------------
 Vars                |**Read-write.** Contains a dictionary with global custom attributes. Not set by default.
 NodeName            |**Read-write.** Contains the cluster node name. Set to the local hostname by default.
+ReloadTimeout       |**Read-write.** Defines the reload timeout for child processes. Defaults to `300s`.
 Environment         |**Read-write.** The name of the Icinga environment. Included in the SNI host name for outbound connections. Not set by default.
 RunAsUser           |**Read-write.** Defines the user the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig.
 RunAsGroup          |**Read-write.** Defines the group the Icinga 2 daemon is running as. Set in the Icinga 2 sysconfig.

diff --git a/lib/base/application.cpp b/lib/base/application.cpp
index df93710b71016312d91a152046439f3544e43b0b..62ed10cc5f22c9f702f599fae2b80a114df1c3a1 100644 (file)
--- a/lib/base/application.cpp
+++ b/lib/base/application.cpp
@@ -77,6 +77,14 @@ void Application::OnConfigLoaded()
 
        ASSERT(m_Instance == nullptr);
        m_Instance = this;
+
+       String reloadTimeout;
+
+       if (ScriptGlobal::Exists("ReloadTimeout"))
+               reloadTimeout = ScriptGlobal::Get("ReloadTimeout");
+
+       if (!reloadTimeout.IsEmpty())
+               Configuration::ReloadTimeout = Convert::ToDouble(reloadTimeout);
 }
 
 /**
@@ -401,8 +409,6 @@ static void ReloadProcessCallback(const ProcessResult& pr)
 
 pid_t Application::StartReloadProcess()
 {
-       Log(LogInformation, "Application", "Got reload command: Starting new instance.");
-
        // prepare arguments
        ArrayData args;
        args.push_back(GetExePath(m_ArgV[0]));
@@ -422,9 +428,14 @@ pid_t Application::StartReloadProcess()
 #endif /* _WIN32 */
 
        Process::Ptr process = new Process(Process::PrepareCommand(new Array(std::move(args))));
-       process->SetTimeout(300);
+       process->SetTimeout(Configuration::ReloadTimeout);
        process->Run(&ReloadProcessCallback);
 
+       Log(LogInformation, "Application")
+               << "Got reload command: Started new instance with PID '"
+               << (unsigned long)(process->GetPID()) << "' (timeout is "
+               << Configuration::ReloadTimeout << "s).";
+
        return process->GetPID();
 }
 

diff --git a/lib/base/configuration.cpp b/lib/base/configuration.cpp
index 0103028de739198086b30d2982fd153cd6d41656..c22b2f46b4028b39cdaa06a0ca161122f519c82b 100644 (file)
--- a/lib/base/configuration.cpp
+++ b/lib/base/configuration.cpp
@@ -42,6 +42,7 @@ String Configuration::PidPath;
 String Configuration::PkgDataDir;
 String Configuration::PrefixDir;
 String Configuration::ProgramData;
+double Configuration::ReloadTimeout{300};
 int Configuration::RLimitFiles;
 int Configuration::RLimitProcesses;
 int Configuration::RLimitStack;
@@ -240,6 +241,16 @@ void Configuration::SetProgramData(const String& val, bool suppress_events, cons
        HandleUserWrite("ProgramData", &Configuration::ProgramData, val, m_ReadOnly);
 }
 
+double Configuration::GetReloadTimeout() const
+{
+       return Configuration::ReloadTimeout;
+}
+
+void Configuration::SetReloadTimeout(double val, bool suppress_events, const Value& cookie)
+{
+       HandleUserWrite("ReloadTimeout", &Configuration::ReloadTimeout, val, m_ReadOnly);
+}
+
 int Configuration::GetRLimitFiles() const
 {
        return Configuration::RLimitFiles;

diff --git a/lib/base/configuration.hpp b/lib/base/configuration.hpp
index 96ab2726f53b1fc09c71bc3b235c21437ded6912..694a02a8c0872e73bbaa4a9f5139586470e8ad8a 100644 (file)
--- a/lib/base/configuration.hpp
+++ b/lib/base/configuration.hpp
@@ -87,6 +87,9 @@ public:
        String GetProgramData() const override;
        void SetProgramData(const String& value, bool suppress_events = false, const Value& cookie = Empty) override;
 
+       double GetReloadTimeout() const override;
+       void SetReloadTimeout(double value, bool suppress_events = false, const Value& cookie = Empty) override;
+
        int GetRLimitFiles() const override;
        void SetRLimitFiles(int value, bool suppress_events = false, const Value& cookie = Empty) override;
 
@@ -147,6 +150,7 @@ public:
        static String PkgDataDir;
        static String PrefixDir;
        static String ProgramData;
+       static double ReloadTimeout;
        static int RLimitFiles;
        static int RLimitProcesses;
        static int RLimitStack;

diff --git a/lib/base/configuration.ti b/lib/base/configuration.ti
index 22f121411a20883d37c3c69a736204b681020754..356c7fa6b20c449585f29da1cda5170f098dff5d 100644 (file)
--- a/lib/base/configuration.ti
+++ b/lib/base/configuration.ti
@@ -111,6 +111,11 @@ abstract class Configuration
                set;
        };
 
+       [config, no_storage, virtual] double ReloadTimeout {
+               get;
+               set;
+       };
+
        [config, no_storage, virtual] int RLimitFiles {
                get;
                set;

diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp
index cde975b7251bf1b029c11cacf5c17c8f9f4b09f5..31bf2b20f32b2155da84ba8cc3491bcec7d6c19f 100644 (file)
--- a/lib/checker/checkercomponent.cpp
+++ b/lib/checker/checkercomponent.cpp
@@ -84,18 +84,41 @@ void CheckerComponent::Start(bool runtimeCreated)
 
 void CheckerComponent::Stop(bool runtimeRemoved)
 {
-       Log(LogInformation, "CheckerComponent")
-               << "'" << GetName() << "' stopped.";
-
        {
                boost::mutex::scoped_lock lock(m_Mutex);
                m_Stopped = true;
                m_CV.notify_all();
        }
 
+       double wait = 0.0;
+
+       while (GetPendingCheckables() > 0) {
+               Log(LogDebug, "CheckerComponent")
+                       << "Waiting for running checks (" << GetPendingCheckables()
+                       << ") to finish. Waited for " << wait << " seconds now.";
+
+               Utility::Sleep(0.1);
+               wait += 0.1;
+
+               /* Pick a timeout slightly shorther than the process reload timeout. */
+               double waitMax = Configuration::ReloadTimeout - 30;
+               if (waitMax <= 0)
+                       waitMax = 1;
+
+               if (wait > waitMax) {
+                       Log(LogWarning, "CheckerComponent")
+                               << "Checks running too long for " << wait
+                               << " seconds, hard shutdown before reload timeout: " << Configuration::ReloadTimeout << ".";
+                       break;
+               }
+       }
+
        m_ResultTimer->Stop();
        m_Thread.join();
 
+       Log(LogInformation, "CheckerComponent")
+               << "'" << GetName() << "' stopped.";
+
        ObjectImpl<CheckerComponent>::Stop(runtimeRemoved);
 }
 

diff --git a/lib/remote/configpackageutility.cpp b/lib/remote/configpackageutility.cpp
index 3b15724d3cab4a2432e6cbfb7067cd34bf4656ed..d103b413a9b0c2988b6c145f4247cc405094344a 100644 (file)
--- a/lib/remote/configpackageutility.cpp
+++ b/lib/remote/configpackageutility.cpp
@@ -235,7 +235,7 @@ void ConfigPackageUtility::AsyncTryActivateStage(const String& packageName, cons
        args->Add("ActiveStageOverride=" + packageName + ":" + stageName);
 
        Process::Ptr process = new Process(Process::PrepareCommand(args));
-       process->SetTimeout(300);
+       process->SetTimeout(Configuration::ReloadTimeout);
        process->Run(std::bind(&TryActivateStageCallback, _1, packageName, stageName, reload));
 }