[3.6] bpo-31852: Fix segfault caused by using the async soft keyword (GH-4122)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 10e0ad807798e154e4842f27bd5c7d379b6e7f56..ef02342244c5ae2bb3f597e938a8c5b783a2f15a 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -630,6 +630,11 @@ def"', """\
     NAME       'async'       (1, 0) (1, 5)
     OP         '='           (1, 6) (1, 7)
     NUMBER     '1'           (1, 8) (1, 9)
+    """)
+
+        self.check_tokenize("async\\", """\
+    ERRORTOKEN '\\\\'          (1, 5) (1, 6)
+    NAME       'async'       (1, 0) (1, 5)
     """)
 
         self.check_tokenize("a = (async = 1)", """\

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst
new file mode 100644 (file)
index 0000000..d72f41b
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-10-27-19-18-44.bpo-31852.P_4cVr.rst
@@ -0,0 +1,2 @@
+Fix a segmentation fault caused by a combination of the async soft keyword
+and continuation lines.

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ff65f2a735903c2d6d90391ac772b4ce881d60f0..ab72f6145f28795ed6160603f8b58d1582ec793b 100644 (file)
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1563,6 +1563,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                 /* The current token is 'async'.
                    Look ahead one token.*/
 
+                int async_def_prev = tok->async_def;
+                tok->async_def = 2;
+
                 struct tok_state ahead_tok;
                 char *ahead_tok_start = NULL, *ahead_tok_end = NULL;
                 int ahead_tok_kind;
@@ -1581,6 +1584,9 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
                     tok->async_def = 1;
                     return ASYNC;
                 }
+                else{
+                    tok->async_def = async_def_prev;
+                }
             }
         }
 
@@ -1844,6 +1850,10 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
     /* Line continuation */
     if (c == '\\') {
         c = tok_nextc(tok);
+        if (tok->async_def == 2) {
+            tok->done = E_SYNTAX;
+            return ERRORTOKEN;
+        }
         if (c != '\n') {
             tok->done = E_LINECONT;
             tok->cur = tok->inp;