Implement flapping detection.

diff --git a/lib/icinga/Makefile.am b/lib/icinga/Makefile.am
index a513ad0385fcb7adc363e2745fb944e7f9e4b9a2..b2e46cb5ba19acfe2471fc5bc6a976ddd972c926 100644 (file)
--- a/lib/icinga/Makefile.am
+++ b/lib/icinga/Makefile.am
@@ -62,6 +62,7 @@ libicinga_la_SOURCES =  \
        service-comment.cpp \
        service-downtime.cpp \
        service-event.cpp \
+       service-flapping.cpp \
        service-notification.cpp \
        service.h \
        servicegroup.cpp \

diff --git a/lib/icinga/notification.cpp b/lib/icinga/notification.cpp
index 3034845cf04ee40ca924c64536c9926ce7f16be4..b0dcbb6dcb5e955662e83351d28de1c990eb5890 100644 (file)
--- a/lib/icinga/notification.cpp
+++ b/lib/icinga/notification.cpp
@@ -204,6 +204,10 @@ String Notification::NotificationTypeToString(NotificationType type)
                        return "PROBLEM";
                case NotificationRecovery:
                        return "RECOVERY";
+               case NotificationFlappingStart:
+                       return "FLAPPINGSTART";
+               case NotificationFlappingEnd:
+                       return "FLAPPINGEND";
                default:
                        return "UNKNOWN_NOTIFICATION";
        }

diff --git a/lib/icinga/notification.h b/lib/icinga/notification.h
index 62e9d805a451c8380275925086aa631aff4e818f..95024a0d614bb29b9fc7fb3da92ab071c927da47 100644 (file)
--- a/lib/icinga/notification.h
+++ b/lib/icinga/notification.h
@@ -42,7 +42,9 @@ enum NotificationType
        NotificationCustom,
        NotificationAcknowledgement,
        NotificationProblem,
-       NotificationRecovery
+       NotificationRecovery,
+       NotificationFlappingStart,
+       NotificationFlappingEnd,
 };
 
 class Service;

diff --git a/lib/icinga/service-check.cpp b/lib/icinga/service-check.cpp
index 0ccb506624663445a7282c5a4e6fdaf776f6952e..71361c9dbeea2cb9e01976346ead8745c7670fc4 100644 (file)
--- a/lib/icinga/service-check.cpp
+++ b/lib/icinga/service-check.cpp
@@ -367,8 +367,8 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
        SetState(static_cast<ServiceState>(state));
 
        bool call_eventhandler = false;
-
-       if (old_state != GetState()) {
+       bool stateChange = (old_state != GetState());
+       if (stateChange) {
                SetLastStateChange(now);
 
                /* remove acknowledgements */
@@ -449,6 +449,13 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
 
        olock.Lock();
        SetLastCheckResult(cr);
+
+       bool was_flapping, is_flapping;
+
+       was_flapping = IsFlapping();
+       UpdateFlappingStatus(stateChange);
+       is_flapping = IsFlapping();
+
        olock.Unlock();
 
        /* Flush the object so other instances see the service's
@@ -473,7 +480,11 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr)
        if (send_downtime_notification)
                RequestNotifications(in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr);
 
-       if (send_notification)
+       if (!was_flapping && is_flapping)
+               RequestNotifications(NotificationFlappingStart, cr);
+       else if (was_flapping && !is_flapping)
+               RequestNotifications(NotificationFlappingEnd, cr);
+       else if (send_notification)
                RequestNotifications(recovery ? NotificationRecovery : NotificationProblem, cr);
 }
 

diff --git a/lib/icinga/service-flapping.cpp b/lib/icinga/service-flapping.cpp
new file mode 100644 (file)
index 0000000..ca81259
--- /dev/null
+++ b/lib/icinga/service-flapping.cpp
@@ -0,0 +1,78 @@
+/******************************************************************************
+ * Icinga 2                                                                   *
+ * Copyright (C) 2012 Icinga Development Team (http://www.icinga.org/)        *
+ *                                                                            *
+ * This program is free software; you can redistribute it and/or              *
+ * modify it under the terms of the GNU General Public License                *
+ * as published by the Free Software Foundation; either version 2             *
+ * of the License, or (at your option) any later version.                     *
+ *                                                                            *
+ * This program is distributed in the hope that it will be useful,            *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of             *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
+ * GNU General Public License for more details.                               *
+ *                                                                            *
+ * You should have received a copy of the GNU General Public License          *
+ * along with this program; if not, write to the Free Software Foundation     *
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA.             *
+ ******************************************************************************/
+
+#include "icinga/service.h"
+#include "base/dynamictype.h"
+#include "base/objectlock.h"
+#include "base/logger_fwd.h"
+#include "base/timer.h"
+#include "base/utility.h"
+#include <boost/tuple/tuple.hpp>
+#include <boost/smart_ptr/make_shared.hpp>
+#include <boost/foreach.hpp>
+
+using namespace icinga;
+
+#define FLAPPING_INTERVAL (30 * 60)
+
+void Service::UpdateFlappingStatus(bool stateChange)
+{
+       double ts, now;
+       long counter;
+
+       now = Utility::GetTime();
+
+       if (m_FlappingLastChange.IsEmpty()) {
+               ts = now;
+               counter = 0;
+       } else {
+               ts = m_FlappingLastChange;
+               counter = m_FlappingCounter;
+       }
+
+       double diff = now - ts;
+
+       if (diff > 0)
+               counter -= 0.5 * m_FlappingCounter / (diff / FLAPPING_INTERVAL);
+
+       if (stateChange)
+               counter += diff;
+
+       m_FlappingCounter = counter;
+       Touch("flapping_counter");
+
+       m_FlappingLastChange = now;
+       Touch("flapping_lastchange");
+}
+
+bool Service::IsFlapping(void) const
+{
+       double threshold = 30;
+
+       if (!m_FlappingThreshold.IsEmpty())
+               threshold = m_FlappingThreshold;
+
+       if (m_FlappingCounter.IsEmpty())
+               return false;
+
+       long counter = m_FlappingCounter;
+
+       return (counter > threshold * FLAPPING_INTERVAL / 100);
+
+}

diff --git a/lib/icinga/service.cpp b/lib/icinga/service.cpp
index d297c313db0725bdfcf8152118cfce9d013701a7..63edfae1afbad684de3fc1f6392a97102ff13c14 100644 (file)
--- a/lib/icinga/service.cpp
+++ b/lib/icinga/service.cpp
@@ -83,6 +83,10 @@ Service::Service(const Dictionary::Ptr& serializedObject)
        RegisterAttribute("enable_notifications", Attribute_Replicated, &m_EnableNotifications);
        RegisterAttribute("force_next_notification", Attribute_Replicated, &m_ForceNextNotification);
 
+       RegisterAttribute("flapping_counter", Attribute_Replicated, &m_FlappingCounter);
+       RegisterAttribute("flapping_lastchange", Attribute_Replicated, &m_FlappingLastChange);
+       RegisterAttribute("flapping_threshold", Attribute_Config, &m_FlappingThreshold);
+
        SetSchedulingOffset(rand());
 }
 

diff --git a/lib/icinga/service.h b/lib/icinga/service.h
index 42d9489a006131eebe9a10714e27cec88170f637..664fcd22dde7d137eefd04eafce58abca72029a6 100644 (file)
--- a/lib/icinga/service.h
+++ b/lib/icinga/service.h
@@ -248,6 +248,10 @@ public:
        void ExecuteEventHandler(void);
        shared_ptr<EventCommand> GetEventCommand(void) const;
 
+       /* Flapping Detection */
+       bool IsFlapping(void) const;
+       void UpdateFlappingStatus(bool stateChange);
+
 protected:
        virtual void OnRegistrationCompleted(void);
        virtual void OnAttributeChanged(const String& name);
@@ -319,6 +323,11 @@ private:
 
        /* Event Handler */
        Attribute<String> m_EventCommand;
+
+       /* Flapping */
+       Attribute<long> m_FlappingCounter;
+       Attribute<double> m_FlappingLastChange;
+       Attribute<double> m_FlappingThreshold;
 };
 
 }