From 94e3311b97448324d67ba9a527854271373329d9 Mon Sep 17 00:00:00 2001 From: Andrew Dunstan Date: Sat, 8 Jun 2013 09:12:48 -0400 Subject: [PATCH] Handle Unicode surrogate pairs correctly when processing JSON. In 9.2, Unicode escape sequences are not analysed at all other than to make sure that they are in the form \uXXXX. But in 9.3 many of the new operators and functions try to turn JSON text values into text in the server encoding, and this includes de-escaping Unicode escape sequences. This processing had not taken into account the possibility that this might contain a surrogate pair to designate a character outside the BMP. That is now handled correctly. This also enforces correct use of surrogate pairs, something that is not done by the type's input routines. This fact is noted in the docs. --- doc/src/sgml/func.sgml | 9 ++++++ src/backend/utils/adt/json.c | 52 ++++++++++++++++++++++++++++++ src/test/regress/expected/json.out | 23 +++++++++++++ src/test/regress/sql/json.sql | 8 +++++ 4 files changed, 92 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 2c02fd1e1b..3adb36579e 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -10150,6 +10150,15 @@ table2-mapping + + + The json functions and operators can impose stricter validity requirements + than the type's input functions. In particular, they check much more closely that any use + of Unicode surrogate pairs to designate characters outside the Unicode Basic Multilingual + Plane is correct. + + + The extension has a cast from hstore to diff --git a/src/backend/utils/adt/json.c b/src/backend/utils/adt/json.c index aaf99bddf2..d8046c5b54 100644 --- a/src/backend/utils/adt/json.c +++ b/src/backend/utils/adt/json.c @@ -646,6 +646,7 @@ json_lex_string(JsonLexContext *lex) { char *s; int len; + int hi_surrogate = -1; if (lex->strval != NULL) resetStringInfo(lex->strval); @@ -718,6 +719,36 @@ json_lex_string(JsonLexContext *lex) int utf8len; char *converted; + if (ch >= 0xd800 && ch <= 0xdbff) + { + if (hi_surrogate != -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("high order surrogate must not follow a high order surrogate."), + report_json_context(lex))); + hi_surrogate = (ch & 0x3ff) << 10; + continue; + } + else if (ch >= 0xdc00 && ch <= 0xdfff) + { + if (hi_surrogate == -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("low order surrogate must follow a high order surrogate."), + report_json_context(lex))); + ch = 0x10000 + hi_surrogate + (ch & 0x3ff); + hi_surrogate = -1; + } + + if (hi_surrogate != -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("low order surrogate must follow a high order surrogate."), + report_json_context(lex))); + unicode_to_utf8(ch, (unsigned char *) utf8str); utf8len = pg_utf_mblen((unsigned char *) utf8str); utf8str[utf8len] = '\0'; @@ -730,6 +761,13 @@ json_lex_string(JsonLexContext *lex) } else if (lex->strval != NULL) { + if (hi_surrogate != -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("low order surrogate must follow a high order surrogate."), + report_json_context(lex))); + switch (*s) { case '"': @@ -784,11 +822,25 @@ json_lex_string(JsonLexContext *lex) } else if (lex->strval != NULL) { + if (hi_surrogate != -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("low order surrogate must follow a high order surrogate."), + report_json_context(lex))); + appendStringInfoChar(lex->strval, *s); } } + if (hi_surrogate != -1) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), + errmsg("invalid input syntax for type json"), + errdetail("low order surrogate must follow a high order surrogate."), + report_json_context(lex))); + /* Hooray, we found the end of the string! */ lex->prev_token_terminator = lex->token_terminator; lex->token_terminator = s + 1; diff --git a/src/test/regress/expected/json.out b/src/test/regress/expected/json.out index 1d7cf5ff2f..293c742962 100644 --- a/src/test/regress/expected/json.out +++ b/src/test/regress/expected/json.out @@ -920,3 +920,26 @@ select * from json_populate_recordset(row('def',99,null)::jpop,'[{"a":[100,200,3 ERROR: cannot call json_populate_recordset on a nested object select * from json_populate_recordset(row('def',99,null)::jpop,'[{"c":[100,200,300],"x":43.2},{"a":{"z":true},"b":3,"c":"2012-01-20 10:42:53"}]') q; ERROR: cannot call json_populate_recordset on a nested object +-- handling of unicode surrogate pairs +select json '{ "a": "\ud83d\ude04\ud83d\udc36" }' -> 'a' as correct; + correct +---------------------------- + "\ud83d\ude04\ud83d\udc36" +(1 row) + +select json '{ "a": "\ud83d\ud83d" }' -> 'a'; -- 2 high surrogates in a row +ERROR: invalid input syntax for type json +DETAIL: high order surrogate must not follow a high order surrogate. +CONTEXT: JSON data, line 1: { "a":... +select json '{ "a": "\ude04\ud83d" }' -> 'a'; -- surrogates in wrong order +ERROR: invalid input syntax for type json +DETAIL: low order surrogate must follow a high order surrogate. +CONTEXT: JSON data, line 1: { "a":... +select json '{ "a": "\ud83dX" }' -> 'a'; -- orphan high surrogate +ERROR: invalid input syntax for type json +DETAIL: low order surrogate must follow a high order surrogate. +CONTEXT: JSON data, line 1: { "a":... +select json '{ "a": "\ude04X" }' -> 'a'; -- orphan low surrogate +ERROR: invalid input syntax for type json +DETAIL: low order surrogate must follow a high order surrogate. +CONTEXT: JSON data, line 1: { "a":... diff --git a/src/test/regress/sql/json.sql b/src/test/regress/sql/json.sql index 8a136d7a27..5b6bc36517 100644 --- a/src/test/regress/sql/json.sql +++ b/src/test/regress/sql/json.sql @@ -296,3 +296,11 @@ select * from json_populate_recordset(null::jpop,'[{"a":"blurfl","x":43.2},{"b": select * from json_populate_recordset(row('def',99,null)::jpop,'[{"a":"blurfl","x":43.2},{"b":3,"c":"2012-01-20 10:42:53"}]') q; select * from json_populate_recordset(row('def',99,null)::jpop,'[{"a":[100,200,300],"x":43.2},{"a":{"z":true},"b":3,"c":"2012-01-20 10:42:53"}]') q; select * from json_populate_recordset(row('def',99,null)::jpop,'[{"c":[100,200,300],"x":43.2},{"a":{"z":true},"b":3,"c":"2012-01-20 10:42:53"}]') q; + +-- handling of unicode surrogate pairs + +select json '{ "a": "\ud83d\ude04\ud83d\udc36" }' -> 'a' as correct; +select json '{ "a": "\ud83d\ud83d" }' -> 'a'; -- 2 high surrogates in a row +select json '{ "a": "\ude04\ud83d" }' -> 'a'; -- surrogates in wrong order +select json '{ "a": "\ud83dX" }' -> 'a'; -- orphan high surrogate +select json '{ "a": "\ude04X" }' -> 'a'; -- orphan low surrogate -- 2.40.0