From e3ae321222c32d64232c46e4062082effec79bcf Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 17 Nov 2012 21:14:58 +0100 Subject: [PATCH] Issue #15379: Fix passing of non-BMP characters as integers for the charmap decoder (already working as unicode strings). Patch by Serhiy Storchaka. --- Lib/test/test_codeccallbacks.py | 2 +- Lib/test/test_codecs.py | 105 ++++++++++++++++++++++++++++++++ Misc/NEWS | 3 + Objects/unicodeobject.c | 28 ++++++++- 4 files changed, 135 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 4825f43ff0..150a0d2a80 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -717,7 +717,7 @@ class CodecCallbackTest(unittest.TestCase): raise ValueError self.assertRaises(UnicodeError, codecs.charmap_decode, "\xff", "strict", {0xff: None}) self.assertRaises(ValueError, codecs.charmap_decode, "\xff", "strict", D()) - self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: sys.maxunicode+1}) + self.assertRaises(TypeError, codecs.charmap_decode, "\xff", "strict", {0xff: 0x110000}) def test_encodehelper(self): # enhance coverage of: diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 57669f82a8..f620b4cde3 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1534,6 +1534,10 @@ class CharmapTest(unittest.TestCase): (u"ab", 3) ) + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, b"\x00\x01\x02", "strict", u"ab" + ) + self.assertEqual( codecs.charmap_decode("\x00\x01\x02", "ignore", u"ab\ufffe"), (u"ab", 3) @@ -1545,6 +1549,107 @@ class CharmapTest(unittest.TestCase): (u"", len(allbytes)) ) + def test_decode_with_int2str_map(self): + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: u'a', 1: u'b', 2: u'c'}), + (u"abc", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: u'Aa', 1: u'Bb', 2: u'Cc'}), + (u"AaBbCc", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: u'\U0010FFFF', 1: u'b', 2: u'c'}), + (u"\U0010FFFFbc", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: u'a', 1: u'b', 2: u''}), + (u"ab", 3) + ) + + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, "\x00\x01\x02", "strict", + {0: u'a', 1: u'b'} + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "replace", + {0: u'a', 1: u'b'}), + (u"ab\ufffd", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "replace", + {0: u'a', 1: u'b', 2: None}), + (u"ab\ufffd", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "ignore", + {0: u'a', 1: u'b'}), + (u"ab", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "ignore", + {0: u'a', 1: u'b', 2: None}), + (u"ab", 3) + ) + + allbytes = bytes(range(256)) + self.assertEqual( + codecs.charmap_decode(allbytes, "ignore", {}), + (u"", len(allbytes)) + ) + + def test_decode_with_int2int_map(self): + a = ord(u'a') + b = ord(u'b') + c = ord(u'c') + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: a, 1: b, 2: c}), + (u"abc", 3) + ) + + # Issue #15379 + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "strict", + {0: 0x10FFFF, 1: b, 2: c}), + (u"\U0010FFFFbc", 3) + ) + + self.assertRaises(TypeError, + codecs.charmap_decode, "\x00\x01\x02", "strict", + {0: 0x110000, 1: b, 2: c} + ) + + self.assertRaises(UnicodeDecodeError, + codecs.charmap_decode, "\x00\x01\x02", "strict", + {0: a, 1: b}, + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "replace", + {0: a, 1: b}), + (u"ab\ufffd", 3) + ) + + self.assertEqual( + codecs.charmap_decode("\x00\x01\x02", "ignore", + {0: a, 1: b}), + (u"ab", 3) + ) + + class WithStmtTest(unittest.TestCase): def test_encodedfile(self): f = StringIO.StringIO("\xc3\xbc") diff --git a/Misc/NEWS b/Misc/NEWS index 0b1de30116..7128a29b7f 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,9 @@ What's New in Python 2.7.4 Core and Builtins ----------------- +- Issue #15379: Fix passing of non-BMP characters as integers for the charmap + decoder (already working as unicode strings). Patch by Serhiy Storchaka. + - Issue #16453: Fix equality testing of dead weakref objects. - Issue #9535: Fix pending signals that have been received but not yet diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index a4c04f4186..b4c37fb464 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4127,12 +4127,36 @@ PyObject *PyUnicode_DecodeCharmap(const char *s, /* Apply mapping */ if (PyInt_Check(x)) { long value = PyInt_AS_LONG(x); - if (value < 0 || value > 65535) { + if (value < 0 || value > 0x10FFFF) { PyErr_SetString(PyExc_TypeError, - "character mapping must be in range(65536)"); + "character mapping must be in range(0x110000)"); Py_DECREF(x); goto onError; } + +#ifndef Py_UNICODE_WIDE + if (value > 0xFFFF) { + /* see the code for 1-n mapping below */ + if (extrachars < 2) { + /* resize first */ + Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v); + Py_ssize_t needed = 10 - extrachars; + extrachars += needed; + /* XXX overflow detection missing */ + if (_PyUnicode_Resize(&v, + PyUnicode_GET_SIZE(v) + needed) < 0) { + Py_DECREF(x); + goto onError; + } + p = PyUnicode_AS_UNICODE(v) + oldpos; + } + value -= 0x10000; + *p++ = 0xD800 | (value >> 10); + *p++ = 0xDC00 | (value & 0x3FF); + extrachars -= 2; + } + else +#endif *p++ = (Py_UNICODE)value; } else if (x == Py_None) { -- 2.40.0