Add new tokens // and //=, in support of PEP 238.

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 33c5e41d7ae40d65ced3c9b3e0e2cb4eb4dfa20d..b952b365c2a856142157fb9caef38608641b8248 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -78,6 +78,7 @@ String = group(r"[uU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
 # longest operators first (e.g., if = came before ==, == would get
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
+                 r"//=?",
                  r"[+\-*/%&|^=<>]=?",
                  r"~")