bpo-33260: Regenerate token.py after removing ASYNC and AWAIT. (GH-6447)

(cherry picked from commit d08972fdb92cad6b813d22205752c97ea18df060)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index fb92ad0f07c21ed1f5c2bf9bb3505b3916b8db3c..7fe989b90524119434d9707873cc40e73cb4d9c0 100644 (file)
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -172,7 +172,7 @@ Common syntax elements for comprehensions are:
 
 .. productionlist::
    comprehension: `expression` `comp_for`
-   comp_for: [ASYNC] "for" `target_list` "in" `or_test` [`comp_iter`]
+   comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`]
    comp_iter: `comp_for` | `comp_if`
    comp_if: "if" `expression_nocond` [`comp_iter`]
 

diff --git a/Lib/token.py b/Lib/token.py
index 091f80bf191b9a8163fc7ea8cc860aec50338d8f..ba132059abf5ee5d0edb56c0d61057b7126f9e6d 100644 (file)
--- a/Lib/token.py
+++ b/Lib/token.py
@@ -65,14 +65,12 @@ RARROW = 51
 ELLIPSIS = 52
 # Don't forget to update the table _PyParser_TokenNames in tokenizer.c!
 OP = 53
-AWAIT = 54
-ASYNC = 55
-ERRORTOKEN = 56
+ERRORTOKEN = 54
 # These aren't used by the C tokenizer but are needed for tokenize.py
-COMMENT = 57
-NL = 58
-ENCODING = 59
-N_TOKENS = 60
+COMMENT = 55
+NL = 56
+ENCODING = 57
+N_TOKENS = 58
 # Special definitions for cooperation with parser
 NT_OFFSET = 256
 #--end constants--

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index f5c6ac7f5e055924438f8e27767abd449aadf322..6528b90061286312b2740acd863f6a32c857b714 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -284,7 +284,7 @@ class Untokenizer:
                 self.encoding = tokval
                 continue
 
-            if toknum in (NAME, NUMBER, ASYNC, AWAIT):
+            if toknum in (NAME, NUMBER):
                 tokval += ' '
 
             # Insert a space between two consecutive strings