#1477: ur'\U0010FFFF' used to raise in narrow unicode builds.
authorAmaury Forgeot d'Arc <amauryfa@gmail.com>
Mon, 24 Mar 2008 21:16:28 +0000 (21:16 +0000)
committerAmaury Forgeot d'Arc <amauryfa@gmail.com>
Mon, 24 Mar 2008 21:16:28 +0000 (21:16 +0000)
Corrected the raw-unicode-escape codec to use UTF-16 surrogates in
this case, like the unicode-escape codec does.

Backport of r61793 and r61853

Lib/test/test_unicode.py
Misc/NEWS
Objects/unicodeobject.c

index 4f757714582b9cf71e4f5c6870ef5ea42b175fb1..55fb8e17f081a9a4d6358fa72586a8d8c3275de0 100644 (file)
@@ -736,12 +736,25 @@ class UnicodeTest(
         print >>out, u'def\n'
 
     def test_ucs4(self):
-        if sys.maxunicode == 0xFFFF:
-            return
         x = u'\U00100000'
         y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
         self.assertEqual(x, y)
 
+        y = r'\U00100000'
+        x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
+        self.assertEqual(x, y)
+        y = r'\U00010000'
+        x = y.decode("raw-unicode-escape").encode("raw-unicode-escape")
+        self.assertEqual(x, y)
+
+        try:
+            '\U11111111'.decode("raw-unicode-escape")
+        except UnicodeDecodeError, e:
+            self.assertEqual(e.start, 0)
+            self.assertEqual(e.end, 10)
+        else:
+            self.fail("Should have raised UnicodeDecodeError")
+
     def test_conversion(self):
         # Make sure __unicode__() works properly
         class Foo0:
index a97fa52b2601516f089e8b46350fa18b8236e7cb..3e95a44da9962d6f18b0a0c4c804816529bc7009 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -11,6 +11,13 @@ What's New in Python 2.5.3?
 
 Core and builtins
 -----------------
+
+- Issue #1477: With narrow Unicode builds, the unicode escape sequence
+  \Uxxxxxxxx did not accept values outside the Basic Multilingual Plane.  This
+  affected raw unicode literals and the 'raw-unicode-escape' codec.  Now
+  UTF-16 surrogates are generated in this case, like normal unicode literals
+  and the 'unicode-escape' codec.
+
 - Issue #2321: use pymalloc for unicode object string data to reduce
   memory usage in some circumstances.
 
index c749ac51a7e5052acd19ddbe8a3b6b59f5124480..e2f1ed323d48e69199efb100d82512fe6c8a885e 100644 (file)
@@ -2273,8 +2273,22 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
            else
                x += 10 + c - 'A';
        }
-#ifndef Py_UNICODE_WIDE
-        if (x > 0x10000) {
+        if (x <= 0xffff)
+                /* UCS-2 character */
+                *p++ = (Py_UNICODE) x;
+        else if (x <= 0x10ffff) {
+                /* UCS-4 character. Either store directly, or as
+                   surrogate pair. */
+#ifdef Py_UNICODE_WIDE
+                *p++ = (Py_UNICODE) x;
+#else
+                x -= 0x10000L;
+                *p++ = 0xD800 + (Py_UNICODE) (x >> 10);
+                *p++ = 0xDC00 + (Py_UNICODE) (x & 0x03FF);
+#endif
+        } else {
+            endinpos = s-starts;
+            outpos = p-PyUnicode_AS_UNICODE(v);
             if (unicode_decode_call_errorhandler(
                     errors, &errorHandler,
                     "rawunicodeescape", "\\Uxxxxxxxx out of range",
@@ -2282,8 +2296,6 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
                    (PyObject **)&v, &outpos, &p))
                    goto onError;
         }
-#endif
-       *p++ = x;
        nextByte:
        ;
     }
@@ -2337,6 +2349,32 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
             *p++ = hexdigit[ch & 15];
         }
         else
+#else
+       /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
+       if (ch >= 0xD800 && ch < 0xDC00) {
+           Py_UNICODE ch2;
+           Py_UCS4 ucs;
+
+           ch2 = *s++;
+           size--;
+           if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
+               ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
+               *p++ = '\\';
+               *p++ = 'U';
+               *p++ = hexdigit[(ucs >> 28) & 0xf];
+               *p++ = hexdigit[(ucs >> 24) & 0xf];
+               *p++ = hexdigit[(ucs >> 20) & 0xf];
+               *p++ = hexdigit[(ucs >> 16) & 0xf];
+               *p++ = hexdigit[(ucs >> 12) & 0xf];
+               *p++ = hexdigit[(ucs >> 8) & 0xf];
+               *p++ = hexdigit[(ucs >> 4) & 0xf];
+               *p++ = hexdigit[ucs & 0xf];
+               continue;
+           }
+           /* Fall through: isolated surrogates are copied as-is */
+           s--;
+           size++;
+       }
 #endif
        /* Map 16-bit characters to '\uxxxx' */
        if (ch >= 256) {