From: Heikki Linnakangas <heikki.linnakangas@iki.fi>
Date: Mon, 6 Aug 2012 10:02:15 +0000 (+0300)
Subject: Perform conversion from Python unicode to string/bytes object via UTF-8.
X-Git-Tag: REL9_0_9~9
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=7fbe5aaaa8bd50e75474f8f671e66ec1b24a42ab;p=postgresql
Perform conversion from Python unicode to string/bytes object via UTF-8.
We used to convert the unicode object directly to a string in the server
encoding by calling Python's PyUnicode_AsEncodedString function. In other
words, we used Python's routines to do the encoding. However, that has a
few problems. First of all, it required keeping a mapping table of Python
encoding names and PostgreSQL encodings. But the real killer was that Python
doesn't support EUC_TW and MULE_INTERNAL encodings at all.
Instead, convert the Python unicode object to UTF-8, and use PostgreSQL's
encoding conversion functions to convert from UTF-8 to server encoding. We
were already doing the same in the other direction in PLyUnicode_FromString,
so this is more consistent, too.
Note: This makes SQL_ASCII to behave more leniently. We used to map
SQL_ASCII to Python's 'ascii', which on Python means strict 7-bit ASCII
only, so you got an error if the python string contained anything but pure
ASCII. You no longer get an error; you get the UTF-8 representation of the
string instead.
Backpatch to 9.0, where these conversions were introduced.
Jan Urbański
---
diff --git a/src/pl/plpython/expected/plpython_unicode_2.out b/src/pl/plpython/expected/plpython_unicode_2.out
deleted file mode 100644
index d6bd823db8..0000000000
--- a/src/pl/plpython/expected/plpython_unicode_2.out
+++ /dev/null
@@ -1,52 +0,0 @@
---
--- Unicode handling
---
-CREATE TABLE unicode_test (
- testvalue text NOT NULL
-);
-CREATE FUNCTION unicode_return() RETURNS text AS E'
-return u"\\x80"
-' LANGUAGE plpythonu;
-CREATE FUNCTION unicode_trigger() RETURNS trigger AS E'
-TD["new"]["testvalue"] = u"\\x80"
-return "MODIFY"
-' LANGUAGE plpythonu;
-CREATE TRIGGER unicode_test_bi BEFORE INSERT ON unicode_test
- FOR EACH ROW EXECUTE PROCEDURE unicode_trigger();
-CREATE FUNCTION unicode_plan1() RETURNS text AS E'
-plan = plpy.prepare("SELECT $1 AS testvalue", ["text"])
-rv = plpy.execute(plan, [u"\\x80"], 1)
-return rv[0]["testvalue"]
-' LANGUAGE plpythonu;
-CREATE FUNCTION unicode_plan2() RETURNS text AS E'
-plan = plpy.prepare("SELECT $1 || $2 AS testvalue", ["text", u"text"])
-rv = plpy.execute(plan, ["foo", "bar"], 1)
-return rv[0]["testvalue"]
-' LANGUAGE plpythonu;
-SELECT unicode_return();
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeError: ASCII encoding error: ordinal not in range(128)
-CONTEXT: while creating return value
-PL/Python function "unicode_return"
-INSERT INTO unicode_test (testvalue) VALUES ('test');
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeError: ASCII encoding error: ordinal not in range(128)
-CONTEXT: while modifying trigger row
-PL/Python function "unicode_trigger"
-SELECT * FROM unicode_test;
- testvalue
------------
-(0 rows)
-
-SELECT unicode_plan1();
-WARNING: PL/Python: plpy.Error: unrecognized error in PLy_spi_execute_plan
-CONTEXT: PL/Python function "unicode_plan1"
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeError: ASCII encoding error: ordinal not in range(128)
-CONTEXT: PL/Python function "unicode_plan1"
-SELECT unicode_plan2();
- unicode_plan2
----------------
- foobar
-(1 row)
-
diff --git a/src/pl/plpython/expected/plpython_unicode_3.out b/src/pl/plpython/expected/plpython_unicode_3.out
deleted file mode 100644
index 676845de4d..0000000000
--- a/src/pl/plpython/expected/plpython_unicode_3.out
+++ /dev/null
@@ -1,52 +0,0 @@
---
--- Unicode handling
---
-CREATE TABLE unicode_test (
- testvalue text NOT NULL
-);
-CREATE FUNCTION unicode_return() RETURNS text AS E'
-return u"\\x80"
-' LANGUAGE plpythonu;
-CREATE FUNCTION unicode_trigger() RETURNS trigger AS E'
-TD["new"]["testvalue"] = u"\\x80"
-return "MODIFY"
-' LANGUAGE plpythonu;
-CREATE TRIGGER unicode_test_bi BEFORE INSERT ON unicode_test
- FOR EACH ROW EXECUTE PROCEDURE unicode_trigger();
-CREATE FUNCTION unicode_plan1() RETURNS text AS E'
-plan = plpy.prepare("SELECT $1 AS testvalue", ["text"])
-rv = plpy.execute(plan, [u"\\x80"], 1)
-return rv[0]["testvalue"]
-' LANGUAGE plpythonu;
-CREATE FUNCTION unicode_plan2() RETURNS text AS E'
-plan = plpy.prepare("SELECT $1 || $2 AS testvalue", ["text", u"text"])
-rv = plpy.execute(plan, ["foo", "bar"], 1)
-return rv[0]["testvalue"]
-' LANGUAGE plpythonu;
-SELECT unicode_return();
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
-CONTEXT: while creating return value
-PL/Python function "unicode_return"
-INSERT INTO unicode_test (testvalue) VALUES ('test');
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
-CONTEXT: while modifying trigger row
-PL/Python function "unicode_trigger"
-SELECT * FROM unicode_test;
- testvalue
------------
-(0 rows)
-
-SELECT unicode_plan1();
-WARNING: PL/Python: plpy.Error: unrecognized error in PLy_spi_execute_plan
-CONTEXT: PL/Python function "unicode_plan1"
-ERROR: PL/Python: could not convert Python Unicode object to PostgreSQL server encoding
-DETAIL: UnicodeEncodeError: 'ascii' codec can't encode character u'\x80' in position 0: ordinal not in range(128)
-CONTEXT: PL/Python function "unicode_plan1"
-SELECT unicode_plan2();
- unicode_plan2
----------------
- foobar
-(1 row)
-
diff --git a/src/pl/plpython/plpython.c b/src/pl/plpython/plpython.c
index 85829a0d3b..b748af1d35 100644
--- a/src/pl/plpython/plpython.c
+++ b/src/pl/plpython/plpython.c
@@ -3686,66 +3686,56 @@ PLy_free(void *ptr)
static PyObject *
PLyUnicode_Bytes(PyObject *unicode)
{
- PyObject *rv;
- const char *serverenc;
+ PyObject *bytes, *rv;
+ char *utf8string, *encoded;
+
+ /* First encode the Python unicode object with UTF-8. */
+ bytes = PyUnicode_AsUTF8String(unicode);
+ if (bytes == NULL)
+ PLy_elog(ERROR, "could not convert Python Unicode object to bytes");
+
+ utf8string = PyBytes_AsString(bytes);
+ if (utf8string == NULL) {
+ Py_DECREF(bytes);
+ PLy_elog(ERROR, "could not extract bytes from encoded string");
+ }
/*
- * Map PostgreSQL encoding to a Python encoding name.
+ * Then convert to server encoding if necessary.
+ *
+ * PyUnicode_AsEncodedString could be used to encode the object directly
+ * in the server encoding, but Python doesn't support all the encodings
+ * that PostgreSQL does (EUC_TW and MULE_INTERNAL). UTF-8 is used as an
+ * intermediary in PLyUnicode_FromString as well.
*/
- switch (GetDatabaseEncoding())
+ if (GetDatabaseEncoding() != PG_UTF8)
{
- case PG_SQL_ASCII:
- /*
- * Mapping SQL_ASCII to Python's 'ascii' is a bit bogus. Python's
- * 'ascii' means true 7-bit only ASCII, while PostgreSQL's
- * SQL_ASCII means that anything is allowed, and the system doesn't
- * try to interpret the bytes in any way. But not sure what else
- * to do, and we haven't heard any complaints...
- */
- serverenc = "ascii";
- break;
- case PG_WIN1250:
- serverenc = "cp1250";
- break;
- case PG_WIN1251:
- serverenc = "cp1251";
- break;
- case PG_WIN1252:
- serverenc = "cp1252";
- break;
- case PG_WIN1253:
- serverenc = "cp1253";
- break;
- case PG_WIN1254:
- serverenc = "cp1254";
- break;
- case PG_WIN1255:
- serverenc = "cp1255";
- break;
- case PG_WIN1256:
- serverenc = "cp1256";
- break;
- case PG_WIN1257:
- serverenc = "cp1257";
- break;
- case PG_WIN1258:
- serverenc = "cp1258";
- break;
- case PG_WIN866:
- serverenc = "cp866";
- break;
- case PG_WIN874:
- serverenc = "cp874";
- break;
- default:
- /* Other encodings have the same name in Python. */
- serverenc = GetDatabaseEncodingName();
- break;
+ PG_TRY();
+ {
+ encoded = (char *) pg_do_encoding_conversion(
+ (unsigned char *) utf8string,
+ strlen(utf8string),
+ PG_UTF8,
+ GetDatabaseEncoding());
+ }
+ PG_CATCH();
+ {
+ Py_DECREF(bytes);
+ PG_RE_THROW();
+ }
+ PG_END_TRY();
}
+ else
+ encoded = utf8string;
+
+ /* finally, build a bytes object in the server encoding */
+ rv = PyBytes_FromStringAndSize(encoded, strlen(encoded));
+
+ /* if pg_do_encoding_conversion allocated memory, free it now */
+ if (utf8string != encoded)
+ pfree(encoded);
- rv = PyUnicode_AsEncodedString(unicode, serverenc, "strict");
- if (rv == NULL)
- PLy_elog(ERROR, "could not convert Python Unicode object to PostgreSQL server encoding");
+ Py_DECREF(bytes);
return rv;
}