Improve reconnect behavior for the RedisWriter class

refs #4991

diff --git a/lib/redis/rediswriter.cpp b/lib/redis/rediswriter.cpp
index a723fb226e006f24a1bef3d116576ca76e808080..599120c5280008f92b9c8bb611586b17a9de4f0b 100644 (file)
--- a/lib/redis/rediswriter.cpp
+++ b/lib/redis/rediswriter.cpp
@@ -33,21 +33,60 @@ void RedisWriter::Start(bool runtimeCreated)
 {
        ObjectImpl<RedisWriter>::Start(runtimeCreated);
 
-       boost::thread thread(boost::bind(&RedisWriter::HandleEvents, this));
+       boost::thread thread(boost::bind(&RedisWriter::ConnectionThreadProc, this));
        thread.detach();
+}
 
+void RedisWriter::ConnectionThreadProc(void)
+{
        String path = GetPath();
        String host = GetHost();
 
+       Log(LogInformation, "RedisWriter", "Trying to connecto redis server");
+
        if (path.IsEmpty())
                m_Context = redisConnect(host.CStr(), GetPort());
        else
                m_Context = redisConnectUnix(path.CStr());
 
-       String password = GetPassword();
+       if (!m_Context || m_Context->err) {
+               if (!m_Context) {
+                       Log(LogWarning, "RedisWriter", "Cannot allocate redis context.");
+               } else {
+                       Log(LogWarning, "RedisWriter", "Connection error: ")
+                           << m_Context->errstr;
+               }
+       }
+
+       for (;;) {
+               String password = GetPassword();
+
+               if (!password.IsEmpty()) {
+                       redisReply *reply = reinterpret_cast<redisReply *>(redisCommand(m_Context, "AUTH %s", password.CStr()));
+
+                       if (reply->type == REDIS_REPLY_STATUS || reply->type == REDIS_REPLY_ERROR) {
+                               Log(LogInformation, "RedisWriter")
+                                   << "AUTH: " << reply->str;
+                       }
+
+                       freeReplyObject(reply);
+               }
+
+               HandleEvents();
+
+               for (;;) {
+                       Log(LogInformation, "RedisWriter", "Trying to reconnect to redis server");
+
+                       if (redisReconnect(m_Context) == REDIS_OK) {
+                               Log(LogInformation, "RedisWriter", "Connection to redis server was reestablished");
+                               break;
+                       }
 
-       void *reply = redisCommand(m_Context, "AUTH %s", password.CStr());
-       freeReplyObject(reply);
+                       Log(LogInformation, "RedisWriter", "Unable to reconnect to redis server: Waiting for next attempt");
+
+                       Utility::Sleep(15);
+               }
+       }
 }
 
 void RedisWriter::HandleEvents(void)
@@ -81,14 +120,24 @@ void RedisWriter::HandleEvents(void)
 
                String body = JsonEncode(result);
 
-               //TODO: Reconnect handling
-               try {
-                       void *reply = redisCommand(m_Context, "LPUSH icinga:events %s", body.CStr());
+               redisReply *reply = reinterpret_cast<redisReply *>(redisCommand(m_Context, "LPUSH icinga:events %s", body.CStr()));
+
+               if (!reply)
+                       break;
+
+               if (reply->type == REDIS_REPLY_STATUS || reply->type == REDIS_REPLY_ERROR) {
+                       Log(LogInformation, "RedisWriter")
+                           << "LPUSH icinga:events: " << reply->str;
+               }
+
+               if (reply->type == REDIS_REPLY_ERROR) {
                        freeReplyObject(reply);
-               } catch (const std::exception&) {
-                       queue->RemoveClient(this);
-                       EventQueue::UnregisterIfUnused(queueName, queue);
-                       throw;
+                       break;
                }
+
+               freeReplyObject(reply);
        }
+
+       queue->RemoveClient(this);
+       EventQueue::UnregisterIfUnused(queueName, queue);
 }

diff --git a/lib/redis/rediswriter.hpp b/lib/redis/rediswriter.hpp
index 9ee9d2b6c04861969375281b297dd54b79b5b966..34d8839e8b786616bed90b8e90ff72eb56ad4bef 100644 (file)
--- a/lib/redis/rediswriter.hpp
+++ b/lib/redis/rediswriter.hpp
@@ -40,6 +40,7 @@ public:
        virtual void Start(bool runtimeCreated) override;
 
 private:
+       void ConnectionThreadProc(void);
        void HandleEvents(void);
 
        redisContext *m_Context;