bpo-2180: Treat line continuation at EOF as a `SyntaxError` (GH-13401)

This makes the parser consistent with the tokenize module (already the case
in `pypy`).

sample
------

```python
x = 5\
```

before
------

```console
$ python3 t.py
$ python3 -mtokenize t.py
t.py:2:0: error: EOF in multi-line statement
```

after
-----

```console
$ ./python t.py
  File "t.py", line 3
    x = 5\

         ^
SyntaxError: unexpected EOF while parsing
$ ./python -m tokenize t.py
t.py:2:0: error: EOF in multi-line statement
```

https://bugs.python.org/issue2180

diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py
index 7baa7ae57c6ff413d4144e1b9e180ff5fe5f6708..a091ceaa25bc4fc2b67a76c1fbd38951e5b5c658 100644 (file)
--- a/Lib/test/test_eof.py
+++ b/Lib/test/test_eof.py
@@ -1,7 +1,9 @@
 """test script for a few new invalid token catches"""
 
-import unittest
+import sys
 from test import support
+from test.support import script_helper
+import unittest
 
 class EOFTestCase(unittest.TestCase):
     def test_EOFC(self):
@@ -24,5 +26,27 @@ class EOFTestCase(unittest.TestCase):
         else:
             raise support.TestFailed
 
+    def test_line_continuation_EOF(self):
+        """A contination at the end of input must be an error; bpo2180."""
+        expect = 'unexpected EOF while parsing (<string>, line 1)'
+        with self.assertRaises(SyntaxError) as excinfo:
+            exec('x = 5\\')
+        self.assertEqual(str(excinfo.exception), expect)
+        with self.assertRaises(SyntaxError) as excinfo:
+            exec('\\')
+        self.assertEqual(str(excinfo.exception), expect)
+
+    @unittest.skipIf(not sys.executable, "sys.executable required")
+    def test_line_continuation_EOF_from_file_bpo2180(self):
+        """Ensure tok_nextc() does not add too many ending newlines."""
+        with support.temp_dir() as temp_dir:
+            file_name = script_helper.make_script(temp_dir, 'foo', '\\')
+            rc, out, err = script_helper.assert_python_failure(file_name)
+            self.assertIn(b'unexpected EOF while parsing', err)
+
+            file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\')
+            rc, out, err = script_helper.assert_python_failure(file_name)
+            self.assertIn(b'unexpected EOF while parsing', err)
+
 if __name__ == "__main__":
     unittest.main()

diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst
new file mode 100644 (file)
index 0000000..a2207c1
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-17-18-34-30.bpo-2180.aBqHeW.rst
@@ -0,0 +1 @@
+Treat line continuation at EOF as a ``SyntaxError`` by Anthony Sottile.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5dc2ae65c42daf67aa6fbd9066ec6aa9aacfb347..e52d498d5542d5b8dfc7f12b40ed4211ab3f9508 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -983,7 +983,8 @@ tok_nextc(struct tok_state *tok)
                         return EOF;
                     /* Last line does not end in \n,
                        fake one */
-                    strcpy(tok->inp, "\n");
+                    if (tok->inp[-1] != '\n')
+                        strcpy(tok->inp, "\n");
                 }
                 tok->inp = strchr(tok->inp, '\0');
                 done = tok->inp[-1] == '\n';
@@ -1674,6 +1675,14 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
             tok->cur = tok->inp;
             return ERRORTOKEN;
         }
+        c = tok_nextc(tok);
+        if (c == EOF) {
+            tok->done = E_EOF;
+            tok->cur = tok->inp;
+            return ERRORTOKEN;
+        } else {
+            tok_backup(tok, c);
+        }
         tok->cont_line = 1;
         goto again; /* Read next line */
     }