Stash notifications until object authority has been updated once

refs #7086

diff --git a/lib/icinga/checkable-notification.cpp b/lib/icinga/checkable-notification.cpp
index 78c488dfe53336fb4ce292f9d91da566c03f2861..a65108c932cbfc29cf3ae94f65a1bb8072e40580 100644 (file)
--- a/lib/icinga/checkable-notification.cpp
+++ b/lib/icinga/checkable-notification.cpp
@@ -4,11 +4,13 @@
 #include "icinga/host.hpp"
 #include "icinga/icingaapplication.hpp"
 #include "icinga/service.hpp"
+#include "base/dictionary.hpp"
 #include "base/objectlock.hpp"
 #include "base/logger.hpp"
 #include "base/exception.hpp"
 #include "base/context.hpp"
 #include "base/convert.hpp"
+#include "remote/apilistener.hpp"
 
 using namespace icinga;
 
@@ -54,17 +56,47 @@ void Checkable::SendNotifications(NotificationType type, const CheckResult::Ptr&
                return;
 
        for (const Notification::Ptr& notification : notifications) {
-               try {
-                       if (!notification->IsPaused()) {
-                               notification->BeginExecuteNotification(type, cr, force, false, author, text);
-                       } else {
-                               Log(LogNotice, "Notification")
-                                       << "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+               if (ApiListener::UpdatedObjectAuthority()) {
+                       try {
+                               if (!notification->IsPaused()) {
+                                       auto stashedNotifications (notification->GetStashedNotifications());
+
+                                       if (stashedNotifications->GetLength()) {
+                                               Log(LogNotice, "Notification")
+                                                       << "Notification '" << notification->GetName() << "': there are some stashed notifications. Stashing notification to preserve order.";
+
+                                               stashedNotifications->Add(new Dictionary({
+                                                       {"type", type},
+                                                       {"cr", cr},
+                                                       {"force", force},
+                                                       {"reminder", false},
+                                                       {"author", author},
+                                                       {"text", text}
+                                               }));
+                                       } else {
+                                               notification->BeginExecuteNotification(type, cr, force, false, author, text);
+                                       }
+                               } else {
+                                       Log(LogNotice, "Notification")
+                                               << "Notification '" << notification->GetName() << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+                               }
+                       } catch (const std::exception& ex) {
+                               Log(LogWarning, "Checkable")
+                                       << "Exception occurred during notification '" << notification->GetName() << "' for checkable '"
+                                       << GetName() << "': " << DiagnosticInformation(ex, false);
                        }
-               } catch (const std::exception& ex) {
-                       Log(LogWarning, "Checkable")
-                               << "Exception occurred during notification '" << notification->GetName() << "' for checkable '"
-                               << GetName() << "': " << DiagnosticInformation(ex, false);
+               } else {
+                       Log(LogNotice, "Notification")
+                               << "Notification '" << notification->GetName() << "': object authority hasn't been updated, yet. Stashing notification.";
+
+                       notification->GetStashedNotifications()->Add(new Dictionary({
+                               {"type", type},
+                               {"cr", cr},
+                               {"force", force},
+                               {"reminder", false},
+                               {"author", author},
+                               {"text", text}
+                       }));
                }
        }
 }

diff --git a/lib/icinga/notification.ti b/lib/icinga/notification.ti
index f2aa77ed12dba1c6ff7a5e30d9003e350451ce75..0212a144a58228d01c0f49f1f069e75149486afa 100644 (file)
--- a/lib/icinga/notification.ti
+++ b/lib/icinga/notification.ti
@@ -80,6 +80,10 @@ class Notification : CustomVarObject < NotificationNameComposer
                default {{{ return false; }}}
        };
 
+       [state, no_user_view, no_user_modify] Array::Ptr stashed_notifications {
+               default {{{ return new Array(); }}}
+       };
+
        [state] Timestamp last_notification;
        [state] Timestamp next_notification;
        [state] int notification_number;

diff --git a/lib/notification/notificationcomponent.cpp b/lib/notification/notificationcomponent.cpp
index 36fe8ecaa68996c2d0f469698cdc12f6a08a605c..aa960120166f731d0d3cd72a5b1c9e38873f5875 100644 (file)
--- a/lib/notification/notificationcomponent.cpp
+++ b/lib/notification/notificationcomponent.cpp
@@ -10,6 +10,7 @@
 #include "base/utility.hpp"
 #include "base/exception.hpp"
 #include "base/statsfunction.hpp"
+#include "remote/apilistener.hpp"
 
 using namespace icinga;
 
@@ -72,12 +73,27 @@ void NotificationComponent::NotificationTimerHandler()
                        continue;
 
                String notificationName = notification->GetName();
+               bool updatedObjectAuthority = ApiListener::UpdatedObjectAuthority();
 
                /* Skip notification if paused, in a cluster setup & HA feature is enabled. */
-               if (notification->IsPaused() && myEndpoint && GetEnableHA()) {
-                       Log(LogNotice, "NotificationComponent")
-                               << "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
-                       continue;
+               if (notification->IsPaused()) {
+                       if (updatedObjectAuthority) {
+                               auto stashedNotifications (notification->GetStashedNotifications());
+                               ObjectLock olock(stashedNotifications);
+
+                               if (stashedNotifications->GetLength()) {
+                                       Log(LogNotice, "NotificationComponent")
+                                               << "Notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority. Dropping all stashed notifications.";
+
+                                       stashedNotifications->Clear();
+                               }
+                       }
+
+                       if (myEndpoint && GetEnableHA()) {
+                               Log(LogNotice, "NotificationComponent")
+                                       << "Reminder notification '" << notificationName << "': HA cluster active, this endpoint does not have the authority (paused=true). Skipping.";
+                               continue;
+                       }
                }
 
                Checkable::Ptr checkable = notification->GetCheckable();
@@ -85,6 +101,42 @@ void NotificationComponent::NotificationTimerHandler()
                if (!IcingaApplication::GetInstance()->GetEnableNotifications() || !checkable->GetEnableNotifications())
                        continue;
 
+               bool reachable = checkable->IsReachable(DependencyNotification);
+
+               if (reachable) {
+                       Array::Ptr unstashedNotifications = new Array();
+
+                       {
+                               auto stashedNotifications (notification->GetStashedNotifications());
+                               ObjectLock olock(stashedNotifications);
+
+                               stashedNotifications->CopyTo(unstashedNotifications);
+                               stashedNotifications->Clear();
+                       }
+
+                       ObjectLock olock(unstashedNotifications);
+
+                       for (Dictionary::Ptr unstashedNotification : unstashedNotifications) {
+                               try {
+                                       Log(LogNotice, "NotificationComponent")
+                                               << "Attempting to send stashed notification '" << notificationName << "'.";
+
+                                       notification->BeginExecuteNotification(
+                                               (NotificationType)(int)unstashedNotification->Get("type"),
+                                               (CheckResult::Ptr)unstashedNotification->Get("cr"),
+                                               (bool)unstashedNotification->Get("force"),
+                                               (bool)unstashedNotification->Get("reminder"),
+                                               (String)unstashedNotification->Get("author"),
+                                               (String)unstashedNotification->Get("text")
+                                       );
+                               } catch (const std::exception& ex) {
+                                       Log(LogWarning, "NotificationComponent")
+                                               << "Exception occurred during notification for object '"
+                                               << notificationName << "': " << DiagnosticInformation(ex, false);
+                               }
+                       }
+               }
+
                if (notification->GetInterval() <= 0 && notification->GetNoMoreNotifications()) {
                        Log(LogNotice, "NotificationComponent")
                                << "Reminder notification '" << notificationName << "': Notification was sent out once and interval=0 disables reminder notifications.";
@@ -94,8 +146,6 @@ void NotificationComponent::NotificationTimerHandler()
                if (notification->GetNextNotification() > now)
                        continue;
 
-               bool reachable = checkable->IsReachable(DependencyNotification);
-
                {
                        ObjectLock olock(notification);
                        notification->SetNextNotification(Utility::GetTime() + notification->GetInterval());