Issue #5057: fix a bug in the peepholer that led to non-portable pyc files between...

diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py
index d05fea6da78057d05252dc60b23e5b8ccf04214a..3e8b7ae9a4d93b2dc6d96496ed9b4630f04ad3d9 100644 (file)
--- a/Lib/test/test_peepholer.py
+++ b/Lib/test/test_peepholer.py
@@ -137,6 +137,24 @@ class TestTranforms(unittest.TestCase):
         asm = dis_single('a="x"*1000')
         self.assertIn('(1000)', asm)
 
+    def test_binary_subscr_on_unicode(self):
+        # valid code get optimized
+        asm = dis_single('u"foo"[0]')
+        self.assertIn("(u'f')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
+        asm = dis_single('u"\u0061\uffff"[1]')
+        self.assertIn("(u'\\uffff')", asm)
+        self.assertNotIn('BINARY_SUBSCR', asm)
+
+        # invalid code doesn't get optimized
+        # out of range
+        asm = dis_single('u"fuu"[10]')
+        self.assertIn('BINARY_SUBSCR', asm)
+        # non-BMP char (see #5057)
+        asm = dis_single('u"\U00012345"[0]')
+        self.assertIn('BINARY_SUBSCR', asm)
+
+
     def test_folding_of_unaryops_on_constants(self):
         for line, elem in (
             ('`1`', "('1')"),                       # unary convert

diff --git a/Misc/NEWS b/Misc/NEWS
index 7510225f2310fa61890b2a60292ba797b97e45a5..3f75b1e9a3a43ba6908554a91d3a2d22a2c18a75 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -9,6 +9,10 @@ What's New in Python 2.7.2?
 Core and Builtins
 -----------------
 
+- Issue #5057: fix a bug in the peepholer that led to non-portable pyc files
+  between narrow and wide builds while optimizing BINARY_SUBSCR on non-BMP
+  chars (e.g. u"\U00012345"[0]).
+
 - Issue #11650: PyOS_StdioReadline() retries fgets() if it was interrupted
   (EINTR), for example if the program is stopped with CTRL+z on Mac OS X. Patch
   written by Charles-Francois Natali.

diff --git a/Python/peephole.c b/Python/peephole.c
index 313eb439e6164a1426df89991bdabe35b888f0a7..433fe27d0e93ecb55090d03548cf31cc9fda58ac 100644 (file)
--- a/Python/peephole.c
+++ b/Python/peephole.c
@@ -129,6 +129,24 @@ fold_binops_on_constants(unsigned char *codestr, PyObject *consts)
             break;
         case BINARY_SUBSCR:
             newconst = PyObject_GetItem(v, w);
+            /* #5057: if v is unicode, there might be differences between
+               wide and narrow builds in cases like u'\U00012345'[0].
+               Wide builds will return a non-BMP char, whereas narrow builds
+               will return a surrogate.  In both the cases skip the
+               optimization in order to produce compatible pycs.
+             */
+            if (newconst != NULL &&
+                PyUnicode_Check(v) && PyUnicode_Check(newconst)) {
+                Py_UNICODE ch = PyUnicode_AS_UNICODE(newconst)[0];
+#ifdef Py_UNICODE_WIDE
+                if (ch > 0xFFFF) {
+#else
+                if (ch >= 0xD800 && ch <= 0xDFFF) {
+#endif
+                    Py_DECREF(newconst);
+                    return 0;
+                }
+            }
             break;
         case BINARY_LSHIFT:
             newconst = PyNumber_Lshift(v, w);