Untokenize: An logically incorrect assert tested user input validity.

Replace it with correct logic that raises ValueError for bad input.
Issues #8478 and #12691 reported the incorrect logic.
Add an Untokenize test case and an initial test method.

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 489f68fade956a5347b7dde4521ec6908544b704..a4963f8c634d0f63f7a6aba4b034d255522fdd10 100644 (file)
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -559,9 +559,10 @@ Pathological whitespace (http://bugs.python.org/issue16152)
 
 from test import test_support
 from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name)
+                     STRING, ENDMARKER, tok_name, Untokenizer)
 from StringIO import StringIO
 import os
+from unittest import TestCase
 
 def dump_tokens(s):
     """Print out the tokens in s in a table format.
@@ -614,12 +615,25 @@ def decistmt(s):
     return untokenize(result)
 
 
+class UntokenizeTest(TestCase):
+
+    def test_bad_input_order(self):
+        u = Untokenizer()
+        u.prev_row = 2
+        u.prev_col = 2
+        with self.assertRaises(ValueError) as cm:
+            u.add_whitespace((1,3))
+        self.assertEqual(cm.exception.args[0],
+                'start (1,3) precedes previous end (2,2)')
+        self.assertRaises(ValueError, u.add_whitespace, (2,1))
+
 __test__ = {"doctests" : doctests, 'decistmt': decistmt}
 
 
 def test_main():
     from test import test_tokenize
     test_support.run_doctest(test_tokenize, True)
+    test_support.run_unittest(UntokenizeTest)
 
 if __name__ == "__main__":
     test_main()

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index db02134ab561a8e598acbd8ae3a24be53bb14457..a5bfa5adf516059cadeb46cd97c3ef2d849b6f5c 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -184,7 +184,9 @@ class Untokenizer:
 
     def add_whitespace(self, start):
         row, col = start
-        assert row <= self.prev_row
+        if row < self.prev_row or row == self.prev_row and col < self.prev_col:
+            raise ValueError("start ({},{}) precedes previous end ({},{})"
+                             .format(row, col, self.prev_row, self.prev_col))
         col_offset = col - self.prev_col
         if col_offset:
             self.tokens.append(" " * col_offset)