Issue #1882: when compiling code from a string, encoding cookies in the

second line of code were not always recognized correctly.
 (backport from rev. 60168)

diff --git a/Lib/test/test_pep263.py b/Lib/test/test_pep263.py
index 3b09c120d3b0db062124ac6f6a22f266efe76a20..1a85f3baff784216ef6be777bad5394f26e76902 100644 (file)
--- a/Lib/test/test_pep263.py
+++ b/Lib/test/test_pep263.py
@@ -1,5 +1,5 @@
 #! -*- coding: koi8-r -*-\r
-# This file is marked as binary in the CVS, to prevent MacCVS from recoding it.\r
+# This file is marked as binary in SVN, to prevent MacCVS from recoding it.\r
 \r
 import unittest\r
 from test import test_support\r
@@ -16,6 +16,14 @@ class PEP263Test(unittest.TestCase):
             '\\\xd0\x9f'\r
         )\r
 \r
+    def test_compilestring(self):\r
+        # see #1882\r
+        c = compile("\n# coding: utf-8\nu = u'\xc3\xb3'\n", "dummy", "exec")\r
+        d = {}\r
+        exec c in d\r
+        self.assertEqual(d['u'], u'\xf3')\r
+\r
+\r
 def test_main():\r
     test_support.run_unittest(PEP263Test)\r
 \r

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index f2c80eb63e5c0682d20cc0c774f925fdf6163ec1..1533502c837f24f2b50bc2b7669b8afe12c21afe 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -585,6 +585,7 @@ decode_str(const char *str, struct tok_state *tok)
 {
        PyObject* utf8 = NULL;
        const char *s;
+       char *newl[2] = {NULL, NULL};
        int lineno = 0;
        tok->enc = NULL;
        tok->str = str;
@@ -603,13 +604,23 @@ decode_str(const char *str, struct tok_state *tok)
        for (s = str;; s++) {
                if (*s == '\0') break;
                else if (*s == '\n') {
+                       newl[lineno] = s;
                        lineno++;
                        if (lineno == 2) break;
                }
        }
        tok->enc = NULL;
-       if (!check_coding_spec(str, s - str, tok, buf_setreadl))
-               return error_ret(tok);
+       /* need to check line 1 and 2 separately since check_coding_spec
+          assumes a single line as input */
+       if (newl[0]) {
+               if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
+                       return error_ret(tok);
+               if (tok->enc == NULL && newl[1]) {
+                       if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
+                                              tok, buf_setreadl))
+                               return error_ret(tok);
+               }
+       }
 #ifdef Py_USING_UNICODE
        if (tok->enc != NULL) {
                assert(utf8 == NULL);