From: Gunnar Beutner Date: Fri, 31 Jan 2014 07:28:00 +0000 (+0100) Subject: Re-use IDs where possible. X-Git-Tag: v0.0.7~16^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=856f01198daa77b12fa26544f5d4bd7c9894eac8;p=icinga2 Re-use IDs where possible. Refs #5565 --- diff --git a/components/db_ido_mysql/idomysqlconnection.cpp b/components/db_ido_mysql/idomysqlconnection.cpp index 919eb7fa2..e333d0696 100644 --- a/components/db_ido_mysql/idomysqlconnection.cpp +++ b/components/db_ido_mysql/idomysqlconnection.cpp @@ -159,7 +159,7 @@ void IdoMysqlConnection::Reconnect(void) if (!mysql_init(&m_Connection)) BOOST_THROW_EXCEPTION(std::bad_alloc()); - if (!mysql_real_connect(&m_Connection, host, user, passwd, db, port, NULL, 0)) + if (!mysql_real_connect(&m_Connection, host, user, passwd, db, port, NULL, CLIENT_FOUND_ROWS)) BOOST_THROW_EXCEPTION(std::runtime_error(mysql_error(&m_Connection))); m_Connected = true; @@ -210,7 +210,7 @@ void IdoMysqlConnection::Reconnect(void) + "', '" + (reconnect ? "RECONNECT" : "INITIAL") + "', NOW())"); /* clear config tables for the initial config dump */ - ClearConfigTables(); + PrepareDatabase(); std::ostringstream q1buf; q1buf << "SELECT object_id, objecttype_id, name1, name2, is_active FROM " + GetTablePrefix() + "objects WHERE instance_id = " << static_cast(m_InstanceID); @@ -251,6 +251,8 @@ IdoMysqlResult IdoMysqlConnection::Query(const String& query) << errinfo_database_query(query) ); + m_AffectedRows = mysql_affected_rows(&m_Connection); + MYSQL_RES *result = mysql_use_result(&m_Connection); if (!result) { @@ -274,6 +276,13 @@ DbReference IdoMysqlConnection::GetLastInsertID(void) return DbReference(mysql_insert_id(&m_Connection)); } +int IdoMysqlConnection::GetAffectedRows(void) +{ + AssertOnWorkQueue(); + + return m_AffectedRows; +} + String IdoMysqlConnection::Escape(const String& s) { AssertOnWorkQueue(); @@ -431,10 +440,10 @@ void IdoMysqlConnection::ExecuteQuery(const DbQuery& query) { ASSERT(query.Category != DbCatInvalid); - m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::InternalExecuteQuery, this, query), true); + m_QueryQueue.Enqueue(boost::bind(&IdoMysqlConnection::InternalExecuteQuery, this, query, (DbQueryType *)NULL), true); } -void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query) +void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride) { boost::mutex::scoped_lock lock(m_ConnectionMutex); @@ -468,7 +477,11 @@ void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query) } } - if ((query.Type & DbQueryInsert) && (query.Type & DbQueryUpdate)) { + type = typeOverride ? *typeOverride : query.Type; + + bool upsert = false; + + if ((type & DbQueryInsert) && (type & DbQueryUpdate)) { bool hasid = false; ASSERT(query.Object); @@ -480,12 +493,11 @@ void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query) else ASSERT(!"Invalid query flags."); - if (hasid) - type = DbQueryUpdate; - else - type = DbQueryInsert; - } else - type = query.Type; + if (!hasid) + upsert = true; + + type = DbQueryUpdate; + } switch (type) { case DbQueryInsert: @@ -541,6 +553,15 @@ void IdoMysqlConnection::InternalExecuteQuery(const DbQuery& query) Query(qbuf.str()); + if (upsert && GetAffectedRows() == 0) { + lock.unlock(); + + DbQueryType to = DbQueryInsert; + InternalExecuteQuery(query, &to); + + return; + } + if (query.Object) { if (query.ConfigUpdate) SetConfigUpdate(query.Object, true); @@ -573,3 +594,15 @@ void IdoMysqlConnection::InternalCleanUpExecuteQuery(const String& table, const Convert::ToString(static_cast(m_InstanceID)) + " AND " + time_column + " < FROM_UNIXTIME(" + Convert::ToString(static_cast(max_age)) + ")"); } + +void IdoMysqlConnection::FillIDCache(const DbType::Ptr& type) +{ + String query = "SELECT " + type->GetIDColumn() + " AS object_id, " + type->GetTable() + "_id FROM " + GetTablePrefix() + type->GetTable() + "s"; + IdoMysqlResult result = Query(query); + + Dictionary::Ptr row; + + while ((row = FetchRow(result))) { + SetInsertID(type, DbReference(row->Get("object_id")), DbReference(row->Get(type->GetTable() + "_id"))); + } +} diff --git a/components/db_ido_mysql/idomysqlconnection.h b/components/db_ido_mysql/idomysqlconnection.h index 52aeb4a84..17d6751e6 100644 --- a/components/db_ido_mysql/idomysqlconnection.h +++ b/components/db_ido_mysql/idomysqlconnection.h @@ -49,6 +49,7 @@ protected: virtual void DeactivateObject(const DbObject::Ptr& dbobj); virtual void ExecuteQuery(const DbQuery& query); virtual void CleanUpExecuteQuery(const String& table, const String& time_key, double time_value); + virtual void FillIDCache(const DbType::Ptr& type); private: DbReference m_InstanceID; @@ -58,12 +59,14 @@ private: boost::mutex m_ConnectionMutex; bool m_Connected; MYSQL m_Connection; + int m_AffectedRows; Timer::Ptr m_ReconnectTimer; Timer::Ptr m_TxTimer; IdoMysqlResult Query(const String& query); DbReference GetLastInsertID(void); + int GetAffectedRows(void); String Escape(const String& s); Dictionary::Ptr FetchRow(const IdoMysqlResult& result); void DiscardRows(const IdoMysqlResult& result); @@ -80,7 +83,7 @@ private: void TxTimerHandler(void); void ReconnectTimerHandler(void); - void InternalExecuteQuery(const DbQuery& query); + void InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride = NULL); void InternalCleanUpExecuteQuery(const String& table, const String& time_key, double time_value); virtual void ClearConfigTable(const String& table); diff --git a/components/db_ido_pgsql/idopgsqlconnection.cpp b/components/db_ido_pgsql/idopgsqlconnection.cpp index 2013bccd6..3abeb2bed 100644 --- a/components/db_ido_pgsql/idopgsqlconnection.cpp +++ b/components/db_ido_pgsql/idopgsqlconnection.cpp @@ -210,7 +210,7 @@ void IdoPgsqlConnection::Reconnect(void) + "', '" + (reconnect ? "RECONNECT" : "INITIAL") + "', NOW())"); /* clear config tables for the initial config dump */ - ClearConfigTables(); + PrepareDatabase(); std::ostringstream q1buf; q1buf << "SELECT object_id, objecttype_id, name1, name2, is_active FROM " + GetTablePrefix() + "objects WHERE instance_id = " << static_cast(m_InstanceID); @@ -269,6 +269,9 @@ IdoPgsqlResult IdoPgsqlConnection::Query(const String& query) ); } + char *rowCount = PQcmdTuples(result); + m_AffectedRows = atoi(rowCount); + return IdoPgsqlResult(result, std::ptr_fun(PQclear)); } @@ -289,6 +292,13 @@ DbReference IdoPgsqlConnection::GetSequenceValue(const String& table, const Stri return DbReference(Convert::ToLong(row->Get("id"))); } +int IdoPgsqlConnection::GetAffectedRows(void) +{ + AssertOnWorkQueue(); + + return m_AffectedRows; +} + String IdoPgsqlConnection::Escape(const String& s) { AssertOnWorkQueue(); @@ -434,10 +444,10 @@ void IdoPgsqlConnection::ExecuteQuery(const DbQuery& query) { ASSERT(query.Category != DbCatInvalid); - m_QueryQueue.Enqueue(boost::bind(&IdoPgsqlConnection::InternalExecuteQuery, this, query), true); + m_QueryQueue.Enqueue(boost::bind(&IdoPgsqlConnection::InternalExecuteQuery, this, query, (DbQueryType *)NULL), true); } -void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query) +void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride) { boost::mutex::scoped_lock lock(m_ConnectionMutex); @@ -471,6 +481,10 @@ void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query) } } + type = typeOverride ? *typeOverride : query.Type; + + bool upsert = false; + if ((query.Type & DbQueryInsert) && (query.Type & DbQueryUpdate)) { bool hasid = false; @@ -483,12 +497,11 @@ void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query) else ASSERT(!"Invalid query flags."); - if (hasid) - type = DbQueryUpdate; - else - type = DbQueryInsert; - } else - type = query.Type; + if (!hasid) + upsert = true; + + type = DbQueryUpdate; + } switch (type) { case DbQueryInsert: @@ -543,6 +556,15 @@ void IdoPgsqlConnection::InternalExecuteQuery(const DbQuery& query) Query(qbuf.str()); + if (upsert && GetAffectedRows() == 0) { + lock.unlock(); + + DbQueryType to = DbQueryInsert; + InternalExecuteQuery(query, &to); + + return; + } + if (query.Object) { if (query.ConfigUpdate) SetConfigUpdate(query.Object, true); @@ -582,3 +604,17 @@ void IdoPgsqlConnection::InternalCleanUpExecuteQuery(const String& table, const Convert::ToString(static_cast(m_InstanceID)) + " AND " + time_column + " < TO_TIMESTAMP(" + Convert::ToString(static_cast(max_age)) + ")"); } + +void IdoPgsqlConnection::FillIDCache(const DbType::Ptr& type) +{ + String query = "SELECT " + type->GetIDColumn() + " AS object_id, " + type->GetTable() + "_id FROM " + GetTablePrefix() + type->GetTable() + "s"; + IdoPgsqlResult result = Query(query); + + Dictionary::Ptr row; + + int index = 0; + while ((row = FetchRow(result, index))) { + index++; + SetInsertID(type, DbReference(row->Get("object_id")), DbReference(row->Get(type->GetTable() + "_id"))); + } +} diff --git a/components/db_ido_pgsql/idopgsqlconnection.h b/components/db_ido_pgsql/idopgsqlconnection.h index 7a0046738..20e4b07d3 100644 --- a/components/db_ido_pgsql/idopgsqlconnection.h +++ b/components/db_ido_pgsql/idopgsqlconnection.h @@ -49,6 +49,7 @@ protected: virtual void DeactivateObject(const DbObject::Ptr& dbobj); virtual void ExecuteQuery(const DbQuery& query); virtual void CleanUpExecuteQuery(const String& table, const String& time_key, double time_value); + virtual void FillIDCache(const DbType::Ptr& type); private: DbReference m_InstanceID; @@ -57,12 +58,14 @@ private: boost::mutex m_ConnectionMutex; PGconn *m_Connection; + int m_AffectedRows; Timer::Ptr m_ReconnectTimer; Timer::Ptr m_TxTimer; IdoPgsqlResult Query(const String& query); DbReference GetSequenceValue(const String& table, const String& column); + int GetAffectedRows(void); String Escape(const String& s); Dictionary::Ptr FetchRow(const IdoPgsqlResult& result, int row); @@ -78,7 +81,7 @@ private: void TxTimerHandler(void); void ReconnectTimerHandler(void); - void InternalExecuteQuery(const DbQuery& query); + void InternalExecuteQuery(const DbQuery& query, DbQueryType *typeOverride = NULL); void InternalCleanUpExecuteQuery(const String& table, const String& time_key, double time_value); virtual void ClearConfigTable(const String& table); diff --git a/lib/db_ido/dbconnection.cpp b/lib/db_ido/dbconnection.cpp index 4b5b5a995..038958efa 100644 --- a/lib/db_ido/dbconnection.cpp +++ b/lib/db_ido/dbconnection.cpp @@ -183,17 +183,33 @@ DbReference DbConnection::GetObjectID(const DbObject::Ptr& dbobj) const void DbConnection::SetInsertID(const DbObject::Ptr& dbobj, const DbReference& dbref) { + SetInsertID(dbobj->GetType(), GetObjectID(dbobj), dbref); +} + +void DbConnection::SetInsertID(const DbType::Ptr& type, const DbReference& objid, const DbReference& dbref) +{ + if (!objid.IsValid()) + return; + if (dbref.IsValid()) - m_InsertIDs[dbobj] = dbref; + m_InsertIDs[std::make_pair(type, objid)] = dbref; else - m_InsertIDs.erase(dbobj); + m_InsertIDs.erase(std::make_pair(type, objid)); } DbReference DbConnection::GetInsertID(const DbObject::Ptr& dbobj) const { - std::map::const_iterator it; + return GetInsertID(dbobj->GetType(), GetObjectID(dbobj)); +} - it = m_InsertIDs.find(dbobj); +DbReference DbConnection::GetInsertID(const DbType::Ptr& type, const DbReference& objid) const +{ + if (!objid.IsValid()) + return DbReference(); + + std::map, DbReference>::const_iterator it; + + it = m_InsertIDs.find(std::make_pair(type, objid)); if (it == m_InsertIDs.end()) return DbReference(); @@ -293,17 +309,17 @@ void DbConnection::UpdateAllObjects(void) } } -void DbConnection::ClearConfigTables(void) +void DbConnection::PrepareDatabase(void) { /* TODO make hardcoded table names modular */ - ClearConfigTable("commands"); + //ClearConfigTable("commands"); ClearConfigTable("comments"); ClearConfigTable("contact_addresses"); ClearConfigTable("contact_notificationcommands"); ClearConfigTable("contactgroup_members"); - ClearConfigTable("contactgroups"); - ClearConfigTable("contacts"); - ClearConfigTable("contactstatus"); + //ClearConfigTable("contactgroups"); + //ClearConfigTable("contacts"); + //ClearConfigTable("contactstatus"); ClearConfigTable("customvariables"); ClearConfigTable("customvariablestatus"); ClearConfigTable("host_contactgroups"); @@ -311,18 +327,22 @@ void DbConnection::ClearConfigTables(void) ClearConfigTable("host_parenthosts"); ClearConfigTable("hostdependencies"); ClearConfigTable("hostgroup_members"); - ClearConfigTable("hostgroups"); - ClearConfigTable("hosts"); - ClearConfigTable("hoststatus"); + //ClearConfigTable("hostgroups"); + //ClearConfigTable("hosts"); + //ClearConfigTable("hoststatus"); ClearConfigTable("programstatus"); ClearConfigTable("scheduleddowntime"); - ClearConfigTable("service_contactgroups"); - ClearConfigTable("service_contacts"); - ClearConfigTable("servicedependencies"); - ClearConfigTable("servicegroup_members"); - ClearConfigTable("servicegroups"); - ClearConfigTable("services"); - ClearConfigTable("servicestatus"); - ClearConfigTable("timeperiod_timeranges"); - ClearConfigTable("timeperiods"); + //ClearConfigTable("service_contactgroups"); + //ClearConfigTable("service_contacts"); + //ClearConfigTable("servicedependencies"); + //ClearConfigTable("servicegroup_members"); + //ClearConfigTable("servicegroups"); + //ClearConfigTable("services"); + //ClearConfigTable("servicestatus"); + //ClearConfigTable("timeperiod_timeranges"); + //ClearConfigTable("timeperiods"); + + BOOST_FOREACH(const DbType::Ptr& type, DbType::GetAllTypes()) { + FillIDCache(type); + } } diff --git a/lib/db_ido/dbconnection.h b/lib/db_ido/dbconnection.h index 0aa88eae7..0664d9337 100644 --- a/lib/db_ido/dbconnection.h +++ b/lib/db_ido/dbconnection.h @@ -45,7 +45,9 @@ public: DbReference GetObjectID(const DbObject::Ptr& dbobj) const; void SetInsertID(const DbObject::Ptr& dbobj, const DbReference& dbref); + void SetInsertID(const DbType::Ptr& type, const DbReference& objid, const DbReference& dbref); DbReference GetInsertID(const DbObject::Ptr& dbobj) const; + DbReference GetInsertID(const DbType::Ptr& type, const DbReference& objid) const; void SetNotificationInsertID(const DynamicObject::Ptr& obj, const DbReference& dbref); DbReference GetNotificationInsertID(const DynamicObject::Ptr& obj) const; @@ -69,14 +71,15 @@ protected: virtual void DeactivateObject(const DbObject::Ptr& dbobj) = 0; virtual void CleanUpExecuteQuery(const String& table, const String& time_column, double max_age); + virtual void FillIDCache(const DbType::Ptr& type) = 0; void UpdateAllObjects(void); - void ClearConfigTables(void); + void PrepareDatabase(void); private: std::map m_ObjectIDs; - std::map m_InsertIDs; + std::map, DbReference> m_InsertIDs; std::map m_NotificationInsertIDs; std::set m_ActiveObjects; std::set m_ConfigUpdates; diff --git a/lib/db_ido/dbtype.cpp b/lib/db_ido/dbtype.cpp index 66cee3045..79fecb4e3 100644 --- a/lib/db_ido/dbtype.cpp +++ b/lib/db_ido/dbtype.cpp @@ -112,3 +112,16 @@ DbType::TypeMap& DbType::GetTypes(void) static DbType::TypeMap tm; return tm; } + +std::set DbType::GetAllTypes(void) +{ + std::set result; + + boost::mutex::scoped_lock lock(GetStaticMutex()); + std::pair kv; + BOOST_FOREACH(kv, GetTypes()) { + result.insert(kv.second); + } + + return result; +} diff --git a/lib/db_ido/dbtype.h b/lib/db_ido/dbtype.h index f77a47cd0..3d4c206f6 100644 --- a/lib/db_ido/dbtype.h +++ b/lib/db_ido/dbtype.h @@ -58,6 +58,8 @@ public: shared_ptr GetOrCreateObjectByName(const String& name1, const String& name2); + static std::set GetAllTypes(void); + private: std::vector m_Names; String m_Table;