From eabfe8cc0e061081d1cbcef4895a99cf7520e8d1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 28 Jun 2015 13:05:19 -0400 Subject: [PATCH] Issue #20387: Backport fix from Python 3.4 --- Lib/tokenize.py | 17 +++++++++++++++++ Misc/NEWS | 3 +++ 2 files changed, 20 insertions(+) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 661ddeb8e6..d426cd2df5 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -198,6 +198,8 @@ class Untokenizer: def untokenize(self, iterable): it = iter(iterable) + indents = [] + startline = False for t in it: if len(t) == 2: self.compat(t, it) @@ -205,6 +207,21 @@ class Untokenizer: tok_type, token, start, end, line = t if tok_type == ENDMARKER: break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False self.add_whitespace(start) self.tokens.append(token) self.prev_row, self.prev_col = end diff --git a/Misc/NEWS b/Misc/NEWS index 9418ad257d..92e08a9219 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -30,6 +30,9 @@ Core and Builtins Library ------- +- Issue #20387: Restore semantic round-trip correctness in tokenize/untokenize + for tab-indented blocks. + - Issue #24456: Fixed possible buffer over-read in adpcm2lin() and lin2adpcm() functions of the audioop module. Fixed SystemError when the state is not a tuple. Fixed possible memory leak. -- 2.50.1