CheckerComponent sometimes fails to schedule checks in time

fixes #12180

diff --git a/lib/checker/checkercomponent.cpp b/lib/checker/checkercomponent.cpp
index f335a91475a5aceb96042da3780d36e0911ff95d..3f5d2d1120985228b0d331898d26006502a1018e 100644 (file)
--- a/lib/checker/checkercomponent.cpp
+++ b/lib/checker/checkercomponent.cpp
@@ -117,9 +117,9 @@ void CheckerComponent::CheckThreadProc(void)
                        break;
 
                CheckTimeView::iterator it = idx.begin();
-               Checkable::Ptr checkable = *it;
+               CheckableScheduleInfo csi = *it;
 
-               double wait = checkable->GetNextCheck() - Utility::GetTime();
+               double wait = csi.NextCheck - Utility::GetTime();
 
                if (Checkable::GetPendingChecks() >= GetConcurrentChecks())
                        wait = 0.5;
@@ -131,6 +131,8 @@ void CheckerComponent::CheckThreadProc(void)
                        continue;
                }
 
+               Checkable::Ptr checkable = csi.Object;
+
                m_IdleCheckables.erase(checkable);
 
                bool forced = checkable->GetForceNextCheck();
@@ -170,7 +172,7 @@ void CheckerComponent::CheckThreadProc(void)
 
                /* reschedule the checkable if checks are disabled */
                if (!check) {
-                       m_IdleCheckables.insert(checkable);
+                       m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
                        lock.unlock();
 
                        checkable->UpdateNextCheck();
@@ -180,7 +182,7 @@ void CheckerComponent::CheckThreadProc(void)
                        continue;
                }
 
-               m_PendingCheckables.insert(checkable);
+               m_PendingCheckables.insert(GetCheckableScheduleInfo(checkable));
 
                lock.unlock();
 
@@ -232,7 +234,7 @@ void CheckerComponent::ExecuteCheckHelper(const Checkable::Ptr& checkable)
                        m_PendingCheckables.erase(it);
 
                        if (checkable->IsActive())
-                               m_IdleCheckables.insert(checkable);
+                               m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
 
                        m_CV.notify_all();
                }
@@ -250,7 +252,7 @@ void CheckerComponent::ResultTimerHandler(void)
                boost::mutex::scoped_lock lock(m_Mutex);
 
                msgbuf << "Pending checkables: " << m_PendingCheckables.size() << "; Idle checkables: " << m_IdleCheckables.size() << "; Checks/s: "
-                   << (CIB::GetActiveHostChecksStatistics(5) + CIB::GetActiveServiceChecksStatistics(5)) / 5.0;
+                   << (CIB::GetActiveHostChecksStatistics(60) + CIB::GetActiveServiceChecksStatistics(60)) / 60.0;
        }
 
        Log(LogNotice, "CheckerComponent", msgbuf.str());
@@ -273,7 +275,7 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
                        if (m_PendingCheckables.find(checkable) != m_PendingCheckables.end())
                                return;
 
-                       m_IdleCheckables.insert(checkable);
+                       m_IdleCheckables.insert(GetCheckableScheduleInfo(checkable));
                } else {
                        m_IdleCheckables.erase(checkable);
                        m_PendingCheckables.erase(checkable);
@@ -283,6 +285,14 @@ void CheckerComponent::ObjectHandler(const ConfigObject::Ptr& object)
        }
 }
 
+CheckableScheduleInfo CheckerComponent::GetCheckableScheduleInfo(const Checkable::Ptr& checkable)
+{
+       CheckableScheduleInfo csi;
+       csi.Object = checkable;
+       csi.NextCheck = checkable->GetNextCheck();
+       return csi;
+}
+
 void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
 {
        boost::mutex::scoped_lock lock(m_Mutex);
@@ -296,7 +306,10 @@ void CheckerComponent::NextCheckChangedHandler(const Checkable::Ptr& checkable)
                return;
 
        idx.erase(checkable);
-       idx.insert(checkable);
+
+       CheckableScheduleInfo csi = GetCheckableScheduleInfo(checkable);
+       idx.insert(csi);
+
        m_CV.notify_all();
 }
 

diff --git a/lib/checker/checkercomponent.hpp b/lib/checker/checkercomponent.hpp
index 9896685233a8d679526e7286216ae8c621e622f2..0e06d5ea5160343e62cdd48e4af071616e3f73d6 100644 (file)
--- a/lib/checker/checkercomponent.hpp
+++ b/lib/checker/checkercomponent.hpp
@@ -35,6 +35,15 @@
 namespace icinga
 {
 
+/**
+ * @ingroup checker
+ */
+struct CheckableScheduleInfo
+{
+       Checkable::Ptr Object;
+       double NextCheck;
+};
+
 /**
  * @ingroup checker
  */
@@ -45,9 +54,9 @@ struct CheckableNextCheckExtractor
        /**
         * @threadsafety Always.
         */
-       double operator()(const Checkable::Ptr& checkable)
+       double operator()(const CheckableScheduleInfo& csi)
        {
-               return checkable->GetNextCheck();
+               return csi.NextCheck;
        }
 };
 
@@ -61,9 +70,9 @@ public:
        DECLARE_OBJECTNAME(CheckerComponent);
 
        typedef boost::multi_index_container<
-               Checkable::Ptr,
+               CheckableScheduleInfo,
                boost::multi_index::indexed_by<
-                       boost::multi_index::ordered_unique<boost::multi_index::identity<Checkable::Ptr> >,
+                       boost::multi_index::ordered_unique<boost::multi_index::member<CheckableScheduleInfo, Checkable::Ptr, &CheckableScheduleInfo::Object> >,
                        boost::multi_index::ordered_non_unique<CheckableNextCheckExtractor>
                >
        > CheckableSet;
@@ -100,6 +109,8 @@ private:
        void NextCheckChangedHandler(const Checkable::Ptr& checkable);
 
        void RescheduleCheckTimer(void);
+
+       static CheckableScheduleInfo GetCheckableScheduleInfo(const Checkable::Ptr& checkable);
 };
 
 }