#16152: fix tokenize to ignore whitespace at the end of the code when no newline...

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index a51e7818670d7ac2acab3fd4d0fe98df0dbee69b..489f68fade956a5347b7dde4521ec6908544b704 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -550,6 +550,10 @@ Evil tabs
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+
+Pathological whitespace (http://bugs.python.org/issue16152)
+    >>> dump_tokens("@          ")
+    OP         '@'           (1, 0) (1, 1)
 """
 
 

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 1cba6e5d9e30de24c740b280ca30a73e0b67fe8b..ca7b07493cfe0b30979b172f225bee8b5a814e33 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -95,7 +95,7 @@ ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
                 r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 tokenprog, pseudoprog, single3prog, double3prog = map(
@@ -362,6 +362,8 @@ def generate_tokens(readline):
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
                 token, initial = line[start:end], line[start]
 
                 if initial in numchars or \

diff --git a/Misc/ACKS b/Misc/ACKS
index 129a116d8dbefd7d7c573a214ddd794da02ad838..01bc45ecb81643ab69f8e2ff275082089ff48b9d 100644 (file)
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -64,6 +64,7 @@ Des Barry
 Ulf Bartelt
 Don Bashford
 Nick Bastin
+Ned Batchelder
 Jeff Bauer
 Mike Bayer
 Michael R Bax

diff --git a/Misc/NEWS b/Misc/NEWS
index da211014ac3f0a2488207edf9376572ce0b51bb1..fedcd35c8992ac4155eac5afea7ccaa7c339a5a7 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -130,6 +130,9 @@ Core and Builtins
 Library
 -------
 
+- Issue #16152: fix tokenize to ignore whitespace at the end of the code when
+  no newline is found.  Patch by Ned Batchelder.
+
 - Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu
   Patch by Todd Rovito.