Fix incorrect re-scheduling behavior for command_endpoint checks

refs #8137

diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp
index 6b20cc471dac7c73992cb78064cc9f2e969b37a2..fb9d87188fbd3a114309857f77d00cdaaff22d16 100644 (file)
--- a/lib/checker/checkercomponent.cpp
+++ b/lib/checker/checkercomponent.cpp
@@ -69,7 +69,6 @@ void CheckerComponent::OnConfigLoaded(void)
        ConfigObject::OnActiveChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
        ConfigObject::OnPausedChanged.connect(bind(&CheckerComponent::ObjectHandler, this, _1));
 
-       Checkable::OnNewCheckResult.connect(bind(&CheckerComponent::CheckResultHandler, this, _1));
        Checkable::OnNextCheckChanged.connect(bind(&CheckerComponent::NextCheckChangedHandler, this, _1));
 }
 
@@ -122,7 +121,7 @@ void CheckerComponent::CheckThreadProc(void)
 
                double wait = checkable->GetNextCheck() - Utility::GetTime();
 
-               if (wait > 0 || m_PendingCheckables.size() >= GetConcurrentChecks()) {
+               if (wait > 0 || Checkable::GetPendingChecks() >= GetConcurrentChecks()) {
                        /* Wait for the next check. */
                        m_CV.timed_wait(lock, boost::posix_time::milliseconds(wait * 1000));
 
@@ -216,6 +215,27 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
 
                Log(LogCritical, "checker", output);
        }
+
+       {
+               boost::mutex::scoped_lock lock(m_Mutex);
+
+               /* remove the object from the list of pending objects; if it's not in the
+                * list this was a manual (i.e. forced) check and we must not re-add the
+                * object to the list because it's already there. */
+               CheckerComponent::CheckableSet::iterator it;
+               it = m_PendingCheckables.find(checkable);
+               if (it != m_PendingCheckables.end()) {
+                       m_PendingCheckables.erase(it);
+
+                       if (checkable->IsActive())
+                               m_IdleCheckables.insert(checkable);
+
+                       m_CV.notify_all();
+               }
+       }
+
+       Log(LogDebug, "CheckerComponent")
+           << "Check finished for object '" << checkable->GetName() << "'";
 }
 
 void CheckerComponent::ResultTimerHandler(void)
@@ -259,30 +279,6 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
        }
 }
 
-void CheckerComponent::CheckResultHandler(const Checkable::Ptr& checkable)
-{
-       {
-               boost::mutex::scoped_lock lock(m_Mutex);
-
-               /* remove the object from the list of pending objects; if it's not in the
-                * list this was a manual (i.e. forced) check and we must not re-add the
-                * object to the list because it's already there. */
-               CheckerComponent::CheckableSet::iterator it;
-               it = m_PendingCheckables.find(checkable);
-               if (it != m_PendingCheckables.end()) {
-                       m_PendingCheckables.erase(it);
-
-                       if (checkable->IsActive())
-                               m_IdleCheckables.insert(checkable);
-
-                       m_CV.notify_all();
-               }
-       }
-
-       Log(LogDebug, "CheckerComponent")
-           << "Check finished for object '" << checkable->GetName() << "'";
-}
-
 void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
        boost::mutex::scoped_lock lock(m_Mutex);

diff --git a/lib/checker/checkercomponent.hpp b/lib/checker/checkercomponent.hpp
index f132fe16cda34f6ab769fc5a93f3928bfcba8823..9896685233a8d679526e7286216ae8c621e622f2 100644 (file)
--- a/lib/checker/checkercomponent.hpp
+++ b/lib/checker/checkercomponent.hpp
@@ -97,7 +97,6 @@ private:
        void AdjustCheckTimer(void);
 
        void ObjectHandler(const ConfigObject::Ptr& object);
-       void CheckResultHandler(const Checkable::Ptr& checkable);
        void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 
        void RescheduleCheckTimer(void);

diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp
index f23cee7ed69734d0a0df345e404ba036939f5155..165fd56f7334a0e7649d76d22b388c81a7a4362f 100644 (file)
--- a/lib/icinga/checkable-check.cpp
+++ b/lib/icinga/checkable-check.cpp
@@ -41,6 +41,9 @@ boost::signals2::signal<void (const Checkable::Ptr&, const CheckResult::Ptr&, st
 boost::signals2::signal<void (const Checkable::Ptr&, NotificationType, const CheckResult::Ptr&, const String&, const String&)> Checkable::OnNotificationsRequested;
 boost::signals2::signal<void (const Checkable::Ptr&)> Checkable::OnNextCheckUpdated;
 
+boost::mutex Checkable::m_StatsMutex;
+int Checkable::m_PendingChecks = 0;
+
 CheckCommand::Ptr Checkable::GetCheckCommand(void) const
 {
        return dynamic_pointer_cast<CheckCommand>(NavigateCheckCommandRaw());
@@ -515,3 +518,21 @@ void Checkable::UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type)
                Log(LogWarning, "Checkable", "Unknown checkable type for statistic update.");
        }
 }
+
+void Checkable::IncreasePendingChecks(void)
+{
+       boost::mutex::scoped_lock lock(m_StatsMutex);
+       m_PendingChecks++;
+}
+
+void Checkable::DecreasePendingChecks(void)
+{
+       boost::mutex::scoped_lock lock(m_StatsMutex);
+       m_PendingChecks--;
+}
+
+int Checkable::GetPendingChecks(void)
+{
+       boost::mutex::scoped_lock lock(m_StatsMutex);
+       return m_PendingChecks;
+}

diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp
index abd966dfd1882d6f17ca456e4c44fd62bfc71dec..9abc19afef7eba1b3077e296947cb6cf778d49a9 100644 (file)
--- a/lib/icinga/checkable.hpp
+++ b/lib/icinga/checkable.hpp
@@ -178,6 +178,10 @@ public:
 
        virtual void ValidateCheckInterval(double value, const ValidationUtils& utils) override;
 
+       static void IncreasePendingChecks(void);
+       static void DecreasePendingChecks(void);
+       static int GetPendingChecks(void);
+
 protected:
        virtual void Start(bool runtimeCreated) override;
 
@@ -186,6 +190,9 @@ private:
        bool m_CheckRunning;
        long m_SchedulingOffset;
 
+       static boost::mutex m_StatsMutex;
+       static int m_PendingChecks;
+
        /* Downtimes */
        std::set<Downtime::Ptr> m_Downtimes;
        mutable boost::mutex m_DowntimeMutex;

diff --git a/lib/methods/pluginchecktask.cpp b/lib/methods/pluginchecktask.cpp
index ec8b882e60e2e70dcbce603a28ff8681f16a66d9..c1b1942d9b60bc7d44390cff8290a6961d7b101f 100644 (file)
--- a/lib/methods/pluginchecktask.cpp
+++ b/lib/methods/pluginchecktask.cpp
@@ -52,6 +52,8 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
        resolvers.push_back(std::make_pair("command", commandObj));
        resolvers.push_back(std::make_pair("icinga", IcingaApplication::GetInstance()));
 
+       Checkable::IncreasePendingChecks();
+
        PluginUtility::ExecuteCommand(commandObj, checkable, checkable->GetLastCheckResult(),
            resolvers, resolvedMacros, useResolvedMacros,
            boost::bind(&PluginCheckTask::ProcessFinishedHandler, checkable, cr, _1, _2));
@@ -59,6 +61,8 @@ void PluginCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckRes
 
 void PluginCheckTask::ProcessFinishedHandler(const Checkable::Ptr& checkable, const CheckResult::Ptr& cr, const Value& commandLine, const ProcessResult& pr)
 {
+       Checkable::DecreasePendingChecks();
+
        if (pr.ExitStatus > 3) {
                Process::Arguments parguments = Process::PrepareCommand(commandLine);
                Log(LogWarning, "PluginCheckTask")