From c2d384dbd7c6ed9bdfaac45f05b463263c743ee7 Mon Sep 17 00:00:00 2001 From: =?utf8?q?=C5=81ukasz=20Langa?= Date: Mon, 23 Apr 2018 01:07:11 -0700 Subject: [PATCH] bpo-33338: [tokenize] Minor code cleanup (#6573) This change contains minor things that make diffing between Lib/tokenize.py and Lib/lib2to3/pgen2/tokenize.py cleaner. --- Lib/tokenize.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 6528b90061..40e6a8b929 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -28,7 +28,6 @@ from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections from io import TextIOWrapper -from itertools import chain import itertools as _itertools import re import sys @@ -278,7 +277,7 @@ class Untokenizer: startline = token[0] in (NEWLINE, NL) prevstring = False - for tok in chain([token], iterable): + for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] if toknum == ENCODING: self.encoding = tokval @@ -475,13 +474,10 @@ def tokenize(readline): The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ - # This import is here to avoid problems when the itertools module is not - # built yet and tokenize is imported. - from itertools import chain, repeat encoding, consumed = detect_encoding(readline) - rl_gen = iter(readline, b"") - empty = repeat(b"") - return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding) + empty = _itertools.repeat(b"") + rl_gen = _itertools.chain(consumed, iter(readline, b""), empty) + return _tokenize(rl_gen.__next__, encoding) def _tokenize(readline, encoding): @@ -496,7 +492,7 @@ def _tokenize(readline, encoding): # BOM will already have been stripped. encoding = "utf-8" yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') - while True: # loop over lines in stream + while True: # loop over lines in stream try: line = readline() except StopIteration: @@ -581,7 +577,7 @@ def _tokenize(readline, encoding): continue token, initial = line[start:end], line[start] - if (initial in numchars or # ordinary number + if (initial in numchars or # ordinary number (initial == '.' and token != '.' and token != '...')): yield TokenInfo(NUMBER, token, spos, epos, line) elif initial in '\r\n': @@ -667,7 +663,8 @@ def main(): # Helper error handling routines def perror(message): - print(message, file=sys.stderr) + sys.stderr.write(message) + sys.stderr.write('\n') def error(message, filename=None, location=None): if location: -- 2.40.0