Add query thresholds for the 'ido' check: Rate and pending queries

fixes #3924

diff --git a/doc/10-icinga-template-library.md b/doc/10-icinga-template-library.md
index 3fe35584c5f9a87c73ed884ce464cf8dfc70df5c..d149a13dd4d9571d5560ca11fd6839e14f4fec65 100644 (file)
--- a/doc/10-icinga-template-library.md
+++ b/doc/10-icinga-template-library.md
@@ -107,10 +107,15 @@ Check command for the built-in `ido` check.
 
 Custom attributes passed as [command parameters](03-monitoring-basics.md#command-passing-parameters):
 
-Name         | Description
--------------|---------------
-ido\_type    | **Required.** The type of the IDO connection object. Can be either "IdoMysqlConnection" or "IdoPgsqlConnection".
-ido\_name    | **Required.** The name of the IDO connection object.
+Name                            | Description
+--------------------------------|-----------------------------
+ido\_type                       | **Required.** The type of the IDO connection object. Can be either "IdoMysqlConnection" or "IdoPgsqlConnection".
+ido\_name                       | **Required.** The name of the IDO connection object.
+ido\_queries\_warning           | **Optional.** Warning threshold for queries/s. Applies if the rate is lower than the threshold.
+ido\_queries\_critical          | **Optional.** Critical threshold for queries/s. Applies if the rate is lower than the threshold.
+ido\_pending\_queries\_warning  | **Optional.** Warning threshold for pending queries. Applies if pending queries are higher than the threshold. Supersedes the `ido_queries` thresholds above.
+ido\_pending\_queries\_critical | **Optional.** Critical threshold for pending queries. Applies if pending queries are higher than the threshold. Supersedes the `ido_queries` thresholds above.
+
 
 ### dummy <a id="itl-dummy"></a>
 

diff --git a/lib/db_ido/idochecktask.cpp b/lib/db_ido/idochecktask.cpp
index f53741a600340aabe6481e55c806cb818d13bf12..7c667474d196dfb51a092a277e2d81eca27299ca 100644 (file)
--- a/lib/db_ido/idochecktask.cpp
+++ b/lib/db_ido/idochecktask.cpp
@@ -57,18 +57,35 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
        String idoName = MacroProcessor::ResolveMacros("$ido_name$", resolvers, checkable->GetLastCheckResult(),
                nullptr, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
 
+       String missingQueriesWarning;
+       String missingQueriesCritical;
+       String missingPendingQueriesWarning;
+       String missingPendingQueriesCritical;
+
+       double queriesWarning = MacroProcessor::ResolveMacros("$ido_queries_warning$", resolvers, checkable->GetLastCheckResult(),
+           &missingQueriesWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
+
+       double queriesCritical = MacroProcessor::ResolveMacros("$ido_queries_critical$", resolvers, checkable->GetLastCheckResult(),
+           &missingQueriesCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
+
+       double pendingQueriesWarning = MacroProcessor::ResolveMacros("$ido_pending_queries_warning$", resolvers, checkable->GetLastCheckResult(),
+           &missingPendingQueriesWarning, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
+
+       double pendingQueriesCritical = MacroProcessor::ResolveMacros("$ido_pending_queries_critical$", resolvers, checkable->GetLastCheckResult(),
+           &missingPendingQueriesCritical, MacroProcessor::EscapeCallback(), resolvedMacros, useResolvedMacros);
+
        if (resolvedMacros && !useResolvedMacros)
                return;
 
        if (idoType.IsEmpty()) {
-               cr->SetOutput("Macro 'ido_type' must be set.");
+               cr->SetOutput("Attribute 'ido_type' must be set.");
                cr->SetState(ServiceUnknown);
                checkable->ProcessCheckResult(cr);
                return;
        }
 
        if (idoName.IsEmpty()) {
-               cr->SetOutput("Macro 'ido_name' must be set.");
+               cr->SetOutput("Attribute 'ido_name' must be set.");
                cr->SetState(ServiceUnknown);
                checkable->ProcessCheckResult(cr);
                return;
@@ -77,7 +94,7 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
        Type::Ptr type = Type::GetByName(idoType);
 
        if (!type || !DbConnection::TypeInstance->IsAssignableFrom(type)) {
-               cr->SetOutput("IDO type '" + idoType + "' is invalid.");
+               cr->SetOutput("DB IDO type '" + idoType + "' is invalid.");
                cr->SetState(ServiceUnknown);
                checkable->ProcessCheckResult(cr);
                return;
@@ -89,7 +106,7 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
        DbConnection::Ptr conn = static_pointer_cast<DbConnection>(dtype->GetObject(idoName));
 
        if (!conn) {
-               cr->SetOutput("IDO connection '" + idoName + "' does not exist.");
+               cr->SetOutput("DB IDO connection '" + idoName + "' does not exist.");
                cr->SetState(ServiceUnknown);
                checkable->ProcessCheckResult(cr);
                return;
@@ -98,12 +115,14 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
        double qps = conn->GetQueryCount(60) / 60.0;
 
        if (conn->IsPaused()) {
-               cr->SetOutput("IDO connection is temporarily disabled on this cluster instance.");
+               cr->SetOutput("DB IDO connection is temporarily disabled on this cluster instance.");
                cr->SetState(ServiceOK);
                checkable->ProcessCheckResult(cr);
                return;
        }
 
+       double pendingQueries = conn->GetPendingQueryCount();
+
        if (!conn->GetConnected()) {
                if (conn->GetShouldConnect()) {
                        cr->SetOutput("Could not connect to the database server.");
@@ -112,29 +131,63 @@ void IdoCheckTask::ScriptFunc(const Checkable::Ptr& checkable, const CheckResult
                        cr->SetOutput("Not currently enabled: Another cluster instance is responsible for the IDO database.");
                        cr->SetState(ServiceOK);
                }
+
+               checkable->ProcessCheckResult(cr);
+
+               return;
+       }
+
+       /* Schema versions. */
+       String schema_version = conn->GetSchemaVersion();
+       std::ostringstream msgbuf;
+
+       if (Utility::CompareVersion(IDO_CURRENT_SCHEMA_VERSION, schema_version) < 0) {
+               msgbuf << "Outdated schema version: '" << schema_version << "'. Latest version: '"
+                   << IDO_CURRENT_SCHEMA_VERSION << "'."
+                   << " Queries per second: " << std::fixed << std::setprecision(3) << qps
+                   << " Pending queries: " << std::fixed << std::setprecision(3) << pendingQueries << ".";
+
+               cr->SetState(ServiceWarning);
        } else {
-               String schema_version = conn->GetSchemaVersion();
-               std::ostringstream msgbuf;
+               msgbuf << "Connected to the database server (Schema version: '" << schema_version << "')."
+                   << " Queries per second: " << std::fixed << std::setprecision(3) << qps
+                   << " Pending queries: " << std::fixed << std::setprecision(3) << pendingQueries << ".";
 
-               if (Utility::CompareVersion(IDO_CURRENT_SCHEMA_VERSION, schema_version) < 0) {
-                       msgbuf << "Outdated schema version: '" << schema_version << "'. Latest version: '" << IDO_CURRENT_SCHEMA_VERSION << "'.";
-                       cr->SetState(ServiceWarning);
-               } else {
-                       msgbuf << "Connected to the database server (Schema version: '" << schema_version << "').";
-                       cr->SetState(ServiceOK);
-               }
+               cr->SetState(ServiceOK);
+       }
+
+       /* Check whether the thresholds have been defined and match. */
+       if (missingQueriesCritical.IsEmpty() && qps < queriesCritical) {
+               msgbuf << " " << qps << " queries/s lower than critical threshold (" << queriesCritical << " queries/s).";
 
-               msgbuf << " Queries per second: " << std::fixed << std::setprecision(3) << qps;
+               cr->SetState(ServiceCritical);
+       } else if (missingQueriesWarning.IsEmpty() && qps < queriesWarning) {
+               msgbuf << " " << qps << " queries/s lower than warning threshold (" << queriesWarning << " queries/s).";
 
-               cr->SetOutput(msgbuf.str());
+               cr->SetState(ServiceWarning);
        }
 
+       if (missingPendingQueriesCritical.IsEmpty() && pendingQueries > pendingQueriesCritical) {
+               msgbuf << " " << pendingQueries << " pending queries greater than critical threshold ("
+                   << pendingQueriesCritical << " queries).";
+
+               cr->SetState(ServiceCritical);
+       } else if (missingPendingQueriesWarning.IsEmpty() && pendingQueries > pendingQueriesWarning) {
+               msgbuf << " " << pendingQueries << " pending queries greater than warning threshold ("
+                   << pendingQueriesWarning << " queries).";
+
+               cr->SetState(ServiceWarning);
+       }
+
+       cr->SetOutput(msgbuf.str());
+
        Array::Ptr perfdata = new Array();
-       perfdata->Add(new PerfdataValue("queries", qps));
+       perfdata->Add(new PerfdataValue("queries", qps, false, "", queriesWarning, queriesCritical));
        perfdata->Add(new PerfdataValue("queries_1min", conn->GetQueryCount(60)));
        perfdata->Add(new PerfdataValue("queries_5mins", conn->GetQueryCount(5 * 60)));
        perfdata->Add(new PerfdataValue("queries_15mins", conn->GetQueryCount(15 * 60)));
-       perfdata->Add(new PerfdataValue("pending_queries", conn->GetPendingQueryCount()));
+       perfdata->Add(new PerfdataValue("pending_queries", pendingQueries, false, "", pendingQueriesWarning, pendingQueriesCritical));
+
        cr->SetPerformanceData(perfdata);
 
        checkable->ProcessCheckResult(cr);