Make checks using 'command_endpoint' work inside HA zones

Previously there was no local processing of the executed
check result, which is mandatory inside a HA cluster.

Additionally this patch splits the command execution and
check result processing into more logical parts, executing
local checks, checks on the same command endpoint, and
remote checks.

More details in the referenced issue.

fixes #8249

Signed-off-by: Michael Friedrich <michael.friedrich@netways.de>

diff --git a/AUTHORS b/AUTHORS
index 0018eb74e0e85b7eb4218da10386b2a207c28139..a91663a9d667a73b99acc162a91b528b44aef9d8 100644 (file)
--- a/AUTHORS
+++ b/AUTHORS
@@ -12,6 +12,7 @@ Gaël Beaudoin <gaboo@gaboo.org>
 Gerd von Egidy <gerd@egidy.de>
 Gunnar Beutner <gunnar.beutner@netways.de>
 Ildar Hizbulin <hizel@vyborg.ru>
+James Pharaoh <james@pharaoh.uk>
 Jan Andres <jan.andres@berenberg.de>
 Jan Wagner <waja@cyconet.org>
 Jason Young <jyoung15@gmail.com>

diff --git a/lib/icinga/apievents.cpp b/lib/icinga/apievents.cpp
index 6e21c7f8637e4e7a269503dad409e02a5e1cc4d8..e3aed4c58cb757a519571e0ba459a986d75a89f2 100644 (file)
--- a/lib/icinga/apievents.cpp
+++ b/lib/icinga/apievents.cpp
@@ -1511,9 +1511,9 @@ Value ApiEvents::AcknowledgementClearedAPIHandler(const MessageOrigin& origin, c
 
 Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dictionary::Ptr& params)
 {
-       Endpoint::Ptr endpoint = origin.FromClient->GetEndpoint();
+       Endpoint::Ptr sourceEndpoint = origin.FromClient->GetEndpoint();
 
-       if (!endpoint || (origin.FromZone && !Zone::GetLocalZone()->IsChildOf(origin.FromZone)))
+       if (!sourceEndpoint || (origin.FromZone && !Zone::GetLocalZone()->IsChildOf(origin.FromZone)))
                return Empty;
 
        ApiListener::Ptr listener = ApiListener::GetInstance();
@@ -1539,11 +1539,12 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
                cr->SetState(ServiceUnknown);
                cr->SetOutput("'" + listener->GetName() + "' does not accept commands.");
                Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-               listener->SyncSendMessage(endpoint, message);
+               listener->SyncSendMessage(sourceEndpoint, message);
 
                return Empty;
        }
 
+       /* use a virtual host object for executing the command */
        Host::Ptr host = new Host();
        Dictionary::Ptr attrs = new Dictionary();
 
@@ -1559,7 +1560,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
                        cr->SetState(ServiceUnknown);
                        cr->SetOutput("Check command '" + command + "' does not exist.");
                        Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-                       listener->SyncSendMessage(endpoint, message);
+                       listener->SyncSendMessage(sourceEndpoint, message);
                        return Empty;
                }
        } else if (command_type == "event_command") {
@@ -1569,7 +1570,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
                return Empty;
 
        attrs->Set(command_type, params->Get("command"));
-       attrs->Set("command_endpoint", endpoint->GetName());
+       attrs->Set("command_endpoint", sourceEndpoint->GetName());
 
        Deserialize(host, attrs, false, FAConfig);
 
@@ -1586,7 +1587,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
 
        if (command_type == "check_command") {
                try {
-                       host->ExecuteCheck(macros, true);
+                       host->ExecuteRemoteCheck(macros);
                } catch (const std::exception& ex) {
                        CheckResult::Ptr cr = new CheckResult();
                        cr->SetState(ServiceUnknown);
@@ -1601,7 +1602,7 @@ Value ApiEvents::ExecuteCommandAPIHandler(const MessageOrigin& origin, const Dic
                        cr->SetExecutionEnd(now);
 
                        Dictionary::Ptr message = MakeCheckResultMessage(host, cr);
-                       listener->SyncSendMessage(endpoint, message);
+                       listener->SyncSendMessage(sourceEndpoint, message);
 
                        Log(LogCritical, "checker", output);
                }

diff --git a/lib/icinga/checkable-check.cpp b/lib/icinga/checkable-check.cpp
index 8ed1dd2dd804f18f52fbde9c81bf4e7e3c7c4630..a4da0c31dadfd7f96a21987634bb94f079f75b42 100644 (file)
--- a/lib/icinga/checkable-check.cpp
+++ b/lib/icinga/checkable-check.cpp
@@ -260,15 +260,37 @@ void Checkable::ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrig
 
        Endpoint::Ptr command_endpoint = GetCommandEndpoint();
 
-       if (command_endpoint && (Endpoint::GetLocalEndpoint() != command_endpoint) && GetExtension("agent_check")) {
+       if (command_endpoint && GetExtension("agent_check")) {
+               /* agent checks go through the api */
                ApiListener::Ptr listener = ApiListener::GetInstance();
 
                if (listener) {
+                       /* send message back to its origin */
                        Dictionary::Ptr message = ApiEvents::MakeCheckResultMessage(this, cr);
                        listener->SyncSendMessage(command_endpoint, message);
+
+                       /* HA cluster zone nodes must also process the check result locally
+                        * by fetching the real host/service object if existing
+                        */
+                       Host::Ptr tempHost;
+                       Service::Ptr tempService;
+                       tie(tempHost, tempService) = GetHostService(this);
+                       Host::Ptr realHost = Host::GetByName(tempHost->GetName());
+                       if (realHost) {
+                               Value agent_service_name = GetExtension("agent_service_name");
+                               if (!agent_service_name.IsEmpty()) {
+                                       Checkable::Ptr realCheckable;
+                                       realCheckable = realHost->GetServiceByShortName(agent_service_name);
+                                       if (realCheckable) {
+                                               realCheckable->ProcessCheckResult(cr, origin);
+                                       }
+                               }
+                       }
+
                }
 
                return;
+
        }
 
        bool reachable = IsReachable();
@@ -498,7 +520,21 @@ bool Checkable::IsCheckPending(void) const
        return m_CheckRunning;
 }
 
-void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useResolvedMacros)
+void Checkable::ExecuteRemoteCheck(const Dictionary::Ptr& resolvedMacros)
+{
+       CONTEXT("Executing remote check for object '" + GetName() + "'");
+
+       double scheduled_start = GetNextCheck();
+       double before_check = Utility::GetTime();
+
+       CheckResult::Ptr cr = new CheckResult();
+       cr->SetScheduleStart(scheduled_start);
+       cr->SetExecutionStart(before_check);
+
+       GetCheckCommand()->Execute(this, cr, resolvedMacros, true);
+}
+
+void Checkable::ExecuteCheck()
 {
        CONTEXT("Executing check for object '" + GetName() + "'");
 
@@ -526,23 +562,22 @@ void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useReso
        double scheduled_start = GetNextCheck();
        double before_check = Utility::GetTime();
 
-       CheckResult::Ptr result = new CheckResult();
+       CheckResult::Ptr cr = new CheckResult();
 
-       result->SetScheduleStart(scheduled_start);
-       result->SetExecutionStart(before_check);
+       cr->SetScheduleStart(scheduled_start);
+       cr->SetExecutionStart(before_check);
 
-       Dictionary::Ptr macros;
        Endpoint::Ptr endpoint = GetCommandEndpoint();
+       bool local = !endpoint || endpoint == Endpoint::GetLocalEndpoint();
 
-       if (endpoint && !useResolvedMacros)
-               macros = new Dictionary();
-       else
-               macros = resolvedMacros;
-
-       GetCheckCommand()->Execute(this, result, macros, useResolvedMacros);
+       if (local) {
+               GetCheckCommand()->Execute(this, cr, NULL, false);
+       } else {
+               Dictionary::Ptr macros = new Dictionary();
+               GetCheckCommand()->Execute(this, cr, macros, false);
 
-       if (endpoint && !useResolvedMacros) {
                if (endpoint->IsConnected()) {
+                       /* perform check on remote endpoint */
                        Dictionary::Ptr message = new Dictionary();
                        message->Set("jsonrpc", "2.0");
                        message->Set("method", "event::ExecuteCommand");
@@ -566,10 +601,15 @@ void Checkable::ExecuteCheck(const Dictionary::Ptr& resolvedMacros, bool useReso
 
                        if (listener)
                                listener->SyncSendMessage(endpoint, message);
+
                } else if (Application::GetInstance()->GetStartTime() < Utility::GetTime() - 30) {
-                       result->SetState(ServiceUnknown);
-                       result->SetOutput("Remote Icinga instance '" + endpoint->GetName() + "' is not connected.");
-                       ProcessCheckResult(result);
+                       /* fail to perform check on unconnected endpoint */
+                       cr->SetState(ServiceUnknown);
+
+                       cr->SetOutput("Remote Icinga instance '" + endpoint->GetName() +
+                           "' " + "is not connected to '" + Endpoint::GetLocalEndpoint()->GetName() + "'");
+
+                       ProcessCheckResult(cr);
                }
 
                {

diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp
index 1968c5d889d860ce468c25f7d12e99024ca6985b..91b462a99a6a910af189bbac43697014489751c7 100644 (file)
--- a/lib/icinga/checkable.hpp
+++ b/lib/icinga/checkable.hpp
@@ -135,8 +135,8 @@ public:
 
        static void UpdateStatistics(const CheckResult::Ptr& cr, CheckableType type);
 
-       void ExecuteCheck(const Dictionary::Ptr& resolvedMacros = Dictionary::Ptr(),
-           bool useResolvedMacros = false);
+       void ExecuteRemoteCheck(const Dictionary::Ptr& resolvedMacros = Dictionary::Ptr());
+       void ExecuteCheck();
        void ProcessCheckResult(const CheckResult::Ptr& cr, const MessageOrigin& origin = MessageOrigin());
 
        int GetModifiedAttributes(void) const;