Issue #10778: decoding_fgets() decodes the filename from the filesystem

encoding instead of UTF-8.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 28dcea1085a38c50f9bf624966ad4be0d29d5aea..441d05a9bbc8cb09f2e43b1e9ee6708cda9f6e79 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -545,6 +545,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
 {
     char *line = NULL;
     int badchar = 0;
+    PyObject *filename;
     for (;;) {
         if (tok->decoding_state == STATE_NORMAL) {
             /* We already have a codec associated with
@@ -585,12 +586,16 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
     if (badchar) {
         /* Need to add 1 to the line number, since this line
            has not been counted, yet.  */
-        PyErr_Format(PyExc_SyntaxError,
-            "Non-UTF-8 code starting with '\\x%.2x' "
-            "in file %.200s on line %i, "
-            "but no encoding declared; "
-            "see http://python.org/dev/peps/pep-0263/ for details",
-            badchar, tok->filename, tok->lineno + 1);
+        filename = PyUnicode_DecodeFSDefault(tok->filename);
+        if (filename != NULL) {
+            PyErr_Format(PyExc_SyntaxError,
+                    "Non-UTF-8 code starting with '\\x%.2x' "
+                    "in file %.200U on line %i, "
+                    "but no encoding declared; "
+                    "see http://python.org/dev/peps/pep-0263/ for details",
+                    badchar, filename, tok->lineno + 1);
+            Py_DECREF(filename);
+        }
         return error_ret(tok);
     }
 #endif