#16152: fix tokenize to ignore whitespace at the end of the code when no newline...

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index b6a9ca1e16821263432465cbb2cc48bf95be8e45..f9652ce7b27c0842369ea72516f80f13a8e6e2e6 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -552,6 +552,11 @@ Evil tabs
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
 
+Pathological whitespace (http://bugs.python.org/issue16152)
+    >>> dump_tokens("@          ")
+    ENCODING   'utf-8'       (0, 0) (0, 0)
+    OP         '@'           (1, 0) (1, 1)
+
 Non-ascii identifiers
 
     >>> dump_tokens("Örter = 'places'\\ngrün = 'green'")

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 59081d3579082dcf38b26a1181d8a805e9657c8f..29c9e29b30f20f59956730e2ce931803d8778048 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -108,7 +108,7 @@ ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
                 r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 def _compile(expr):
@@ -473,6 +473,8 @@ def _tokenize(readline, encoding):
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
                 token, initial = line[start:end], line[start]
 
                 if (initial in numchars or                  # ordinary number

diff --git a/Misc/ACKS b/Misc/ACKS
index a070be811c1d7f3f70581d52ed17ccec6aba48ba..1095733839314e1f03bb10ed0b0d5e4001bfcfc1 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -71,6 +71,7 @@ Des Barry
 Ulf Bartelt
 Don Bashford
 Nick Bastin
+Ned Batchelder
 Jeff Bauer
 Mike Bayer
 Michael R Bax

diff --git a/Misc/NEWS b/Misc/NEWS
index 307522523375d048e46b687a8f87d7ddfb9ce188..3ee301f8a893e247137cfc898bef889b3a32fdde 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -143,6 +143,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
+  no newline is found.  Patch by Ned Batchelder.
+
 - Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
   Patch by Todd Rovito.