SF bug #1224621: tokenize module does not detect inconsistent dedents

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 2ce435f585d58509d9097f8aed0f878bc44fbc57..d3c1cc498d2633db8e99e5207d5d17ab2461bed8 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,4 +1,4 @@
-from test.test_support import verbose, findfile, is_resource_enabled
+from test.test_support import verbose, findfile, is_resource_enabled, TestFailed
 import os, glob, random
 from tokenize import (tokenize, generate_tokens, untokenize,
                       NUMBER, NAME, OP, STRING)
@@ -41,6 +41,24 @@ for f in testfiles:
     test_roundtrip(f)
 
 
+###### Test detecton of IndentationError ######################
+
+from cStringIO import StringIO
+
+sampleBadText = """
+def foo():
+    bar
+  baz
+"""
+
+try:
+    for tok in generate_tokens(StringIO(sampleBadText).readline):
+        pass
+except IndentationError:
+    pass
+else:
+    raise TestFailed("Did not detect IndentationError:")
+
 
 ###### Test example in the docs ###############################
 

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index b29da6b7adae211289066eb5cb9a47e4ea6a0c23..2b40e6f31bb499f3761059c891ac3d42b1ae03a1 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -271,6 +271,9 @@ def generate_tokens(readline):
                 indents.append(column)
                 yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
             while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level")
                 indents = indents[:-1]
                 yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 80f7dc79333ce227761c0d521ccdaf2b74f604e5..1fcd798ca2881f7d047426c5131212907b99c160 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -147,6 +147,9 @@ Extension Modules
 Library
 -------
 
+- The tokenize module now detects and reports indentation errors.
+  Bug #1224621.
+
 - The tokenize module has a new untokenize() function to support a full
   roundtrip from lexed tokens back to Python sourcecode.  In addition,
   the generate_tokens() function now accepts a callable argument that