Add API & Cluster stats to /v1/status & icinga check performance metrics

refs #5133

diff --git a/lib/db_ido_mysql/idomysqlconnection.cpp b/lib/db_ido_mysql/idomysqlconnection.cpp
index b761d2b9023275f91004c9a38f03f80facec3342..881c1d5107fdb8e261c31f602c7236974037abad 100644 (file)
--- a/lib/db_ido_mysql/idomysqlconnection.cpp
+++ b/lib/db_ido_mysql/idomysqlconnection.cpp
@@ -53,13 +53,15 @@ void IdoMysqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
        Dictionary::Ptr nodes = new Dictionary();
 
        for (const IdoMysqlConnection::Ptr& idomysqlconnection : ConfigType::GetObjectsByType<IdoMysqlConnection>()) {
-               size_t items = idomysqlconnection->m_QueryQueue.GetLength();
+               size_t queryQueueItems = idomysqlconnection->m_QueryQueue.GetLength();
+               double queryQueueItemRate = idomysqlconnection->m_QueryQueue.GetTaskCount(60) / 60.0;
 
                Dictionary::Ptr stats = new Dictionary();
                stats->Set("version", idomysqlconnection->GetSchemaVersion());
                stats->Set("instance_name", idomysqlconnection->GetInstanceName());
                stats->Set("connected", idomysqlconnection->GetConnected());
-               stats->Set("query_queue_items", items);
+               stats->Set("query_queue_items", queryQueueItems);
+               stats->Set("query_queue_item_rate", queryQueueItemRate);
 
                nodes->Set(idomysqlconnection->GetName(), stats);
 
@@ -67,7 +69,8 @@ void IdoMysqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
                perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_1min", idomysqlconnection->GetQueryCount(60)));
                perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_5mins", idomysqlconnection->GetQueryCount(5 * 60)));
                perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_queries_15mins", idomysqlconnection->GetQueryCount(15 * 60)));
-               perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_items", items));
+               perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_items", queryQueueItems));
+               perfdata->Add(new PerfdataValue("idomysqlconnection_" + idomysqlconnection->GetName() + "_query_queue_item_rate", queryQueueItemRate));
        }
 
        status->Set("idomysqlconnection", nodes);

diff --git a/lib/db_ido_pgsql/idopgsqlconnection.cpp b/lib/db_ido_pgsql/idopgsqlconnection.cpp
index 723767baeb9c4eb0b8357741880a32b7e854fb56..3d1a8662c2c8d5dce391f8a77740bef030b7ade8 100644 (file)
--- a/lib/db_ido_pgsql/idopgsqlconnection.cpp
+++ b/lib/db_ido_pgsql/idopgsqlconnection.cpp
@@ -57,13 +57,15 @@ void IdoPgsqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
        Dictionary::Ptr nodes = new Dictionary();
 
        for (const IdoPgsqlConnection::Ptr& idopgsqlconnection : ConfigType::GetObjectsByType<IdoPgsqlConnection>()) {
-               size_t items = idopgsqlconnection->m_QueryQueue.GetLength();
+               size_t queryQueueItems = idopgsqlconnection->m_QueryQueue.GetLength();
+               double queryQueueItemRate = idopgsqlconnection->m_QueryQueue.GetTaskCount(60) / 60.0;
 
                Dictionary::Ptr stats = new Dictionary();
                stats->Set("version", idopgsqlconnection->GetSchemaVersion());
-               stats->Set("connected", idopgsqlconnection->GetConnected());
                stats->Set("instance_name", idopgsqlconnection->GetInstanceName());
-               stats->Set("query_queue_items", items);
+               stats->Set("connected", idopgsqlconnection->GetConnected());
+               stats->Set("query_queue_items", queryQueueItems);
+               stats->Set("query_queue_item_rate", queryQueueItemRate);
 
                nodes->Set(idopgsqlconnection->GetName(), stats);
 
@@ -71,7 +73,8 @@ void IdoPgsqlConnection::StatsFunc(const Dictionary::Ptr& status, const Array::P
                perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_1min", idopgsqlconnection->GetQueryCount(60)));
                perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_5mins", idopgsqlconnection->GetQueryCount(5 * 60)));
                perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_queries_15mins", idopgsqlconnection->GetQueryCount(15 * 60)));
-               perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_items", items));
+               perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_items", queryQueueItems));
+               perfdata->Add(new PerfdataValue("idopgsqlconnection_" + idopgsqlconnection->GetName() + "_query_queue_item_rate", queryQueueItemRate));
        }
 
        status->Set("idopgsqlconnection", nodes);

diff --git a/lib/perfdata/influxdbwriter.cpp b/lib/perfdata/influxdbwriter.cpp
index bf17bbdb5fcea654bf96b0f0cab996a0f9a56523..3596872fa567517267f57ff0b48d12f5a156b8b6 100644 (file)
--- a/lib/perfdata/influxdbwriter.cpp
+++ b/lib/perfdata/influxdbwriter.cpp
@@ -70,11 +70,13 @@ void InfluxdbWriter::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr&)
 
        for (const InfluxdbWriter::Ptr& influxdbwriter : ConfigType::GetObjectsByType<InfluxdbWriter>()) {
                size_t workQueueItems = influxdbwriter->m_WorkQueue.GetLength();
+               double workQueueItemRate = influxdbwriter->m_WorkQueue.GetTaskCount(60) / 60.0;
                size_t dataBufferItems = influxdbwriter->m_DataBuffer.size();
 
                //TODO: Collect more stats
                Dictionary::Ptr stats = new Dictionary();
                stats->Set("work_queue_items", workQueueItems);
+               stats->Set("work_queue_item_rate", workQueueItemRate);
                stats->Set("data_buffer_items", dataBufferItems);
 
                nodes->Set(influxdbwriter->GetName(), stats);

diff --git a/lib/remote/apilistener.cpp b/lib/remote/apilistener.cpp
index 21ccae3619be9f1ce7189cc64afb593d43b95118..dbc6ea3f983428604e4dfabd85e7670002b91ebd 100644 (file)
--- a/lib/remote/apilistener.cpp
+++ b/lib/remote/apilistener.cpp
@@ -30,6 +30,7 @@
 #include "base/logger.hpp"
 #include "base/objectlock.hpp"
 #include "base/stdiostream.hpp"
+#include "base/perfdatavalue.hpp"
 #include "base/application.hpp"
 #include "base/context.hpp"
 #include "base/statsfunction.hpp"
@@ -292,7 +293,7 @@ void ApiListener::AddConnection(const Endpoint::Ptr& endpoint)
        String host = endpoint->GetHost();
        String port = endpoint->GetPort();
 
-       Log(LogInformation, "JsonRpcConnection")
+       Log(LogInformation, "ApiListener")
            << "Reconnecting to API endpoint '" << endpoint->GetName() << "' via host '" << host << "' and port '" << port << "'";
 
        TcpSocket::Ptr client = new TcpSocket();
@@ -1068,7 +1069,7 @@ void ApiListener::StatsFunc(const Dictionary::Ptr& status, const Array::Ptr& per
 
        ObjectLock olock(stats.second);
        for (const Dictionary::Pair& kv : stats.second)
-               perfdata->Add("'api_" + kv.first + "'=" + Convert::ToString(kv.second));
+               perfdata->Add(new PerfdataValue("api_" + kv.first, kv.second));
 
        status->Set("api", stats.first);
 }
@@ -1152,10 +1153,50 @@ std::pair<Dictionary::Ptr, Dictionary::Ptr> ApiListener::GetStatus(void)
 
        status->Set("zones", connectedZones);
 
+       /* connection stats */
+       size_t jsonRpcClients = GetAnonymousClients().size();
+       size_t httpClients = GetHttpClients().size();
+       size_t workQueueItems = JsonRpcConnection::GetWorkQueueLength();
+       size_t workQueueCount = JsonRpcConnection::GetWorkQueueCount();
+       size_t syncQueueItems = m_SyncQueue.GetLength();
+       size_t relayQueueItems = m_RelayQueue.GetLength();
+       double workQueueItemRate = JsonRpcConnection::GetWorkQueueRate();
+       double syncQueueItemRate = m_SyncQueue.GetTaskCount(60) / 60.0;
+       double relayQueueItemRate = m_RelayQueue.GetTaskCount(60) / 60.0;
+
+       Dictionary::Ptr jsonRpc = new Dictionary();
+       jsonRpc->Set("clients", jsonRpcClients);
+       jsonRpc->Set("work_queue_items", workQueueItems);
+       jsonRpc->Set("work_queue_count", workQueueCount);
+       jsonRpc->Set("sync_queue_items", syncQueueItems);
+       jsonRpc->Set("relay_queue_items", relayQueueItems);
+
+       jsonRpc->Set("work_queue_item_rate", workQueueItemRate);
+       jsonRpc->Set("sync_queue_item_rate", syncQueueItemRate);
+       jsonRpc->Set("relay_queue_item_rate", relayQueueItemRate);
+
+       Dictionary::Ptr http = new Dictionary();
+       http->Set("clients", httpClients);
+
+       status->Set("json_rpc", jsonRpc);
+       status->Set("http", http);
+
+       /* performance data */
        perfdata->Set("num_endpoints", allEndpoints);
        perfdata->Set("num_conn_endpoints", Convert::ToDouble(allConnectedEndpoints->GetLength()));
        perfdata->Set("num_not_conn_endpoints", Convert::ToDouble(allNotConnectedEndpoints->GetLength()));
 
+       perfdata->Set("num_json_rpc_clients", jsonRpcClients);
+       perfdata->Set("num_http_clients", httpClients);
+       perfdata->Set("num_json_rpc_work_queue_items", workQueueItems);
+       perfdata->Set("num_json_rpc_work_queue_count", workQueueCount);
+       perfdata->Set("num_json_rpc_sync_queue_items", syncQueueItems);
+       perfdata->Set("num_json_rpc_relay_queue_items", relayQueueItems);
+
+       perfdata->Set("num_json_rpc_work_queue_item_rate", workQueueItemRate);
+       perfdata->Set("num_json_rpc_sync_queue_item_rate", syncQueueItemRate);
+       perfdata->Set("num_json_rpc_relay_queue_item_rate", relayQueueItemRate);
+
        return std::make_pair(status, perfdata);
 }
 

diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp
index 231ae52925fe327dff70705d7a40789f6132c5ff..dae89eb4cda9c84a1c8c301b4011739aeaebabab 100644 (file)
--- a/lib/remote/jsonrpcconnection.cpp
+++ b/lib/remote/jsonrpcconnection.cpp
@@ -340,3 +340,30 @@ void JsonRpcConnection::TimeoutTimerHandler(void)
        }
 }
 
+int JsonRpcConnection::GetWorkQueueCount(void)
+{
+       return l_JsonRpcConnectionWorkQueueCount;
+}
+
+int JsonRpcConnection::GetWorkQueueLength(void)
+{
+       size_t itemCount = 0;
+
+       for (size_t i = 0; i < l_JsonRpcConnectionWorkQueueCount; i++) {
+               itemCount += l_JsonRpcConnectionWorkQueues[i].GetLength();
+       }
+
+       return itemCount;
+}
+
+double JsonRpcConnection::GetWorkQueueRate(void)
+{
+       double rate = 0.0;
+
+       for (size_t i = 0; i < l_JsonRpcConnectionWorkQueueCount; i++) {
+               rate += l_JsonRpcConnectionWorkQueues[i].GetTaskCount(60) / 60.0;
+       }
+
+       return rate / l_JsonRpcConnectionWorkQueueCount;
+}
+

diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp
index 9cb1306b9cbe6d32d78153785db59361d1154d7d..62c134c277ae4f2c6c72d12954354e035a24a1df 100644 (file)
--- a/lib/remote/jsonrpcconnection.hpp
+++ b/lib/remote/jsonrpcconnection.hpp
@@ -71,6 +71,10 @@ public:
        static void HeartbeatTimerHandler(void);
        static Value HeartbeatAPIHandler(const intrusive_ptr<MessageOrigin>& origin, const Dictionary::Ptr& params);
 
+       static int GetWorkQueueCount(void);
+       static int GetWorkQueueLength(void);
+       static double GetWorkQueueRate(void);
+
 private:
        int m_ID;
        String m_Identity;