From 69fcd7fa40fbe1c968fb1f7d9897d0cd00c0a660 Mon Sep 17 00:00:00 2001 From: Gunnar Beutner Date: Fri, 21 Jun 2013 10:20:29 +0200 Subject: [PATCH] Implement flapping detection. --- lib/icinga/Makefile.am | 1 + lib/icinga/notification.cpp | 4 ++ lib/icinga/notification.h | 4 +- lib/icinga/service-check.cpp | 17 +++++-- lib/icinga/service-flapping.cpp | 78 +++++++++++++++++++++++++++++++++ lib/icinga/service.cpp | 4 ++ lib/icinga/service.h | 9 ++++ 7 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 lib/icinga/service-flapping.cpp diff --git a/lib/icinga/Makefile.am b/lib/icinga/Makefile.am index a513ad038..b2e46cb5b 100644 --- a/lib/icinga/Makefile.am +++ b/lib/icinga/Makefile.am @@ -62,6 +62,7 @@ libicinga_la_SOURCES = \ service-comment.cpp \ service-downtime.cpp \ service-event.cpp \ + service-flapping.cpp \ service-notification.cpp \ service.h \ servicegroup.cpp \ diff --git a/lib/icinga/notification.cpp b/lib/icinga/notification.cpp index 3034845cf..b0dcbb6dc 100644 --- a/lib/icinga/notification.cpp +++ b/lib/icinga/notification.cpp @@ -204,6 +204,10 @@ String Notification::NotificationTypeToString(NotificationType type) return "PROBLEM"; case NotificationRecovery: return "RECOVERY"; + case NotificationFlappingStart: + return "FLAPPINGSTART"; + case NotificationFlappingEnd: + return "FLAPPINGEND"; default: return "UNKNOWN_NOTIFICATION"; } diff --git a/lib/icinga/notification.h b/lib/icinga/notification.h index 62e9d805a..95024a0d6 100644 --- a/lib/icinga/notification.h +++ b/lib/icinga/notification.h @@ -42,7 +42,9 @@ enum NotificationType NotificationCustom, NotificationAcknowledgement, NotificationProblem, - NotificationRecovery + NotificationRecovery, + NotificationFlappingStart, + NotificationFlappingEnd, }; class Service; diff --git a/lib/icinga/service-check.cpp b/lib/icinga/service-check.cpp index 0ccb50662..71361c9db 100644 --- a/lib/icinga/service-check.cpp +++ b/lib/icinga/service-check.cpp @@ -367,8 +367,8 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr) SetState(static_cast(state)); bool call_eventhandler = false; - - if (old_state != GetState()) { + bool stateChange = (old_state != GetState()); + if (stateChange) { SetLastStateChange(now); /* remove acknowledgements */ @@ -449,6 +449,13 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr) olock.Lock(); SetLastCheckResult(cr); + + bool was_flapping, is_flapping; + + was_flapping = IsFlapping(); + UpdateFlappingStatus(stateChange); + is_flapping = IsFlapping(); + olock.Unlock(); /* Flush the object so other instances see the service's @@ -473,7 +480,11 @@ void Service::ProcessCheckResult(const Dictionary::Ptr& cr) if (send_downtime_notification) RequestNotifications(in_downtime ? NotificationDowntimeStart : NotificationDowntimeEnd, cr); - if (send_notification) + if (!was_flapping && is_flapping) + RequestNotifications(NotificationFlappingStart, cr); + else if (was_flapping && !is_flapping) + RequestNotifications(NotificationFlappingEnd, cr); + else if (send_notification) RequestNotifications(recovery ? NotificationRecovery : NotificationProblem, cr); } diff --git a/lib/icinga/service-flapping.cpp b/lib/icinga/service-flapping.cpp new file mode 100644 index 000000000..ca8125914 --- /dev/null +++ b/lib/icinga/service-flapping.cpp @@ -0,0 +1,78 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012 Icinga Development Team (http://www.icinga.org/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "icinga/service.h" +#include "base/dynamictype.h" +#include "base/objectlock.h" +#include "base/logger_fwd.h" +#include "base/timer.h" +#include "base/utility.h" +#include +#include +#include + +using namespace icinga; + +#define FLAPPING_INTERVAL (30 * 60) + +void Service::UpdateFlappingStatus(bool stateChange) +{ + double ts, now; + long counter; + + now = Utility::GetTime(); + + if (m_FlappingLastChange.IsEmpty()) { + ts = now; + counter = 0; + } else { + ts = m_FlappingLastChange; + counter = m_FlappingCounter; + } + + double diff = now - ts; + + if (diff > 0) + counter -= 0.5 * m_FlappingCounter / (diff / FLAPPING_INTERVAL); + + if (stateChange) + counter += diff; + + m_FlappingCounter = counter; + Touch("flapping_counter"); + + m_FlappingLastChange = now; + Touch("flapping_lastchange"); +} + +bool Service::IsFlapping(void) const +{ + double threshold = 30; + + if (!m_FlappingThreshold.IsEmpty()) + threshold = m_FlappingThreshold; + + if (m_FlappingCounter.IsEmpty()) + return false; + + long counter = m_FlappingCounter; + + return (counter > threshold * FLAPPING_INTERVAL / 100); + +} diff --git a/lib/icinga/service.cpp b/lib/icinga/service.cpp index d297c313d..63edfae1a 100644 --- a/lib/icinga/service.cpp +++ b/lib/icinga/service.cpp @@ -83,6 +83,10 @@ Service::Service(const Dictionary::Ptr& serializedObject) RegisterAttribute("enable_notifications", Attribute_Replicated, &m_EnableNotifications); RegisterAttribute("force_next_notification", Attribute_Replicated, &m_ForceNextNotification); + RegisterAttribute("flapping_counter", Attribute_Replicated, &m_FlappingCounter); + RegisterAttribute("flapping_lastchange", Attribute_Replicated, &m_FlappingLastChange); + RegisterAttribute("flapping_threshold", Attribute_Config, &m_FlappingThreshold); + SetSchedulingOffset(rand()); } diff --git a/lib/icinga/service.h b/lib/icinga/service.h index 42d9489a0..664fcd22d 100644 --- a/lib/icinga/service.h +++ b/lib/icinga/service.h @@ -248,6 +248,10 @@ public: void ExecuteEventHandler(void); shared_ptr GetEventCommand(void) const; + /* Flapping Detection */ + bool IsFlapping(void) const; + void UpdateFlappingStatus(bool stateChange); + protected: virtual void OnRegistrationCompleted(void); virtual void OnAttributeChanged(const String& name); @@ -319,6 +323,11 @@ private: /* Event Handler */ Attribute m_EventCommand; + + /* Flapping */ + Attribute m_FlappingCounter; + Attribute m_FlappingLastChange; + Attribute m_FlappingThreshold; }; } -- 2.40.0