From 21db77e396c00c0490b6344a130bdbcef62bfa73 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 14 Nov 2009 16:27:26 +0000 Subject: [PATCH] simply by using itertools.chain() --- Lib/tokenize.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 9d2a6bb88b..b8ee2c854f 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -377,17 +377,12 @@ def tokenize(readline): The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. """ + # This import is here to avoid problems when the itertools module is not + # built yet and tokenize is imported. + from itertools import chain encoding, consumed = detect_encoding(readline) - def readline_generator(consumed): - for line in consumed: - yield line - while True: - try: - yield readline() - except StopIteration: - return - chained = readline_generator(consumed) - return _tokenize(chained.__next__, encoding) + rl_iter = iter(readline, "") + return _tokenize(chain(consumed, rl_iter).__next__, encoding) def _tokenize(readline, encoding): -- 2.40.0