Make sure that all strings in the IDO database are UTF8-encoded

fixes #10554

diff --git a/lib/base/utility.cpp b/lib/base/utility.cpp
index 834e5dec5dedb2bd9eb2233ef26f612b19a7de4b..22b7ed380cf906709eafc7e94890ed0ad02e8cf1 100644 (file)
--- a/lib/base/utility.cpp
+++ b/lib/base/utility.cpp
@@ -1654,3 +1654,39 @@ String Utility::GetPlatformArchitecture(void)
        return UnameHelper('m');
 #endif /* _WIN32 */
 }
+
+String Utility::ValidateUTF8(const String& input)
+{
+       String output;
+       size_t length = input.GetLength();
+
+       for (size_t i = 0; i < length; i++) {
+               if ((input[i] & 0x80) == 0) {
+                       output += input[i];
+                       continue;
+               }
+
+               if ((input[i] & 0xE0) == 0xC0 && length > i + 1 &&
+                   (input[i + 1] & 0xC0) == 0x80) {
+                       output += input[i];
+                       output += input[i + 1];
+                       i++;
+                       continue;
+               }
+
+               if ((input[i] & 0xF0) == 0xE0 && length > i + 2 &&
+                   (input[i + 1] & 0xC0) == 0x80 && (input[i + 2] & 0xC0) == 0x80) {
+                       output += input[i];
+                       output += input[i + 1];
+                       output += input[i + 2];
+                       i += 2;
+                       continue;
+               }
+
+               output += '\xEF';
+               output += '\xBF';
+               output += '\xBD';
+       }
+
+       return output;
+}

diff --git a/lib/base/utility.hpp b/lib/base/utility.hpp
index 3a44c176b02a4437b65ec3d16ec60b58180ff308..e4f4120e937d9a5fe4d91e3f14a8f9969bc9de1b 100644 (file)
--- a/lib/base/utility.hpp
+++ b/lib/base/utility.hpp
@@ -139,6 +139,8 @@ public:
        static String GetPlatformVersion(void);
        static String GetPlatformArchitecture(void);
 
+       static String ValidateUTF8(const String& input);
+
 private:
        Utility(void);
        static void CollectPaths(const String& path, std::vector<String>& paths);

diff --git a/lib/db_ido_mysql/idomysqlconnection.cpp b/lib/db_ido_mysql/idomysqlconnection.cpp
index 97b45e336090ef545cf8e59ce63ad19dc4c5f4aa..e9ddea685bd7adc07ca8e774f45c1751fc6b04f4 100644 (file)
--- a/lib/db_ido_mysql/idomysqlconnection.cpp
+++ b/lib/db_ido_mysql/idomysqlconnection.cpp
@@ -574,10 +574,12 @@ String IdoMysqlConnection::Escape(const String& s)
 {
        AssertOnWorkQueue();
 
-       size_t length = s.GetLength();
-       char *to = new char[s.GetLength() * 2 + 1];
+       String utf8s = Utility::ValidateUTF8(s);
 
-       mysql_real_escape_string(&m_Connection, to, s.CStr(), length);
+       size_t length = utf8s.GetLength();
+       char *to = new char[utf8s.GetLength() * 2 + 1];
+
+       mysql_real_escape_string(&m_Connection, to, utf8s.CStr(), length);
 
        String result = String(to);
 

diff --git a/lib/db_ido_pgsql/idopgsqlconnection.cpp b/lib/db_ido_pgsql/idopgsqlconnection.cpp
index d93145c1dd06b55c10a90f378a50d3f482160818..1286717efb6f371753f61b0687420e82d6c39c43 100644 (file)
--- a/lib/db_ido_pgsql/idopgsqlconnection.cpp
+++ b/lib/db_ido_pgsql/idopgsqlconnection.cpp
@@ -464,10 +464,12 @@ String IdoPgsqlConnection::Escape(const String& s)
 {
        AssertOnWorkQueue();
 
-       size_t length = s.GetLength();
-       char *to = new char[s.GetLength() * 2 + 1];
+       String utf8s = Utility::ValidateUTF8(s);
 
-       PQescapeStringConn(m_Connection, to, s.CStr(), length, NULL);
+       size_t length = utf8s.GetLength();
+       char *to = new char[utf8s.GetLength() * 2 + 1];
+
+       PQescapeStringConn(m_Connection, to, utf8s.CStr(), length, NULL);
 
        String result = String(to);