From 7751a34400d05d125de8d8f23339756f8d3f774d Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 17 Feb 2014 16:45:38 -0500 Subject: [PATCH] Untokenize: An logically incorrect assert tested user input validity. Replace it with correct logic that raises ValueError for bad input. Issues #8478 and #12691 reported the incorrect logic. Add an Untokenize test case and an initial test method. --- Lib/test/test_tokenize.py | 16 +++++++++++++++- Lib/tokenize.py | 4 +++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 489f68fade..a4963f8c63 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -559,9 +559,10 @@ Pathological whitespace (http://bugs.python.org/issue16152) from test import test_support from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP, - STRING, ENDMARKER, tok_name) + STRING, ENDMARKER, tok_name, Untokenizer) from StringIO import StringIO import os +from unittest import TestCase def dump_tokens(s): """Print out the tokens in s in a table format. @@ -614,12 +615,25 @@ def decistmt(s): return untokenize(result) +class UntokenizeTest(TestCase): + + def test_bad_input_order(self): + u = Untokenizer() + u.prev_row = 2 + u.prev_col = 2 + with self.assertRaises(ValueError) as cm: + u.add_whitespace((1,3)) + self.assertEqual(cm.exception.args[0], + 'start (1,3) precedes previous end (2,2)') + self.assertRaises(ValueError, u.add_whitespace, (2,1)) + __test__ = {"doctests" : doctests, 'decistmt': decistmt} def test_main(): from test import test_tokenize test_support.run_doctest(test_tokenize, True) + test_support.run_unittest(UntokenizeTest) if __name__ == "__main__": test_main() diff --git a/Lib/tokenize.py b/Lib/tokenize.py index db02134ab5..a5bfa5adf5 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -184,7 +184,9 @@ class Untokenizer: def add_whitespace(self, start): row, col = start - assert row <= self.prev_row + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) col_offset = col - self.prev_col if col_offset: self.tokens.append(" " * col_offset) -- 2.50.1