From: Benjamin Peterson <benjamin@python.org>
Date: Sat, 14 Nov 2009 16:27:26 +0000 (+0000)
Subject: simply by using itertools.chain()
X-Git-Tag: v3.2a1~2190
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=21db77e396c00c0490b6344a130bdbcef62bfa73;p=python

simply by using itertools.chain()
---

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 9d2a6bb88b..b8ee2c854f 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -377,17 +377,12 @@ def tokenize(readline):
     The first token sequence will always be an ENCODING token
     which tells you which encoding was used to decode the bytes stream.
     """
+    # This import is here to avoid problems when the itertools module is not
+    # built yet and tokenize is imported.
+    from itertools import chain
     encoding, consumed = detect_encoding(readline)
-    def readline_generator(consumed):
-        for line in consumed:
-            yield line
-        while True:
-            try:
-                yield readline()
-            except StopIteration:
-                return
-    chained = readline_generator(consumed)
-    return _tokenize(chained.__next__, encoding)
+    rl_iter = iter(readline, "")
+    return _tokenize(chain(consumed, rl_iter).__next__, encoding)
 
 
 def _tokenize(readline, encoding):