From: Benjamin Peterson <benjamin@python.org>
Date: Sat, 14 Nov 2009 18:09:17 +0000 (+0000)
Subject: use some more itertools magic to make '' be yielded after readline is done
X-Git-Tag: v3.2a1~2189
X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=81dd8b9594d88ff1d2c8f5efea687645bbc36d6f;p=python

use some more itertools magic to make '' be yielded after readline is done
---

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index b8ee2c854f..f82922b7f0 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -379,10 +379,11 @@ def tokenize(readline):
     """
     # This import is here to avoid problems when the itertools module is not
     # built yet and tokenize is imported.
-    from itertools import chain
+    from itertools import chain, repeat
     encoding, consumed = detect_encoding(readline)
-    rl_iter = iter(readline, "")
-    return _tokenize(chain(consumed, rl_iter).__next__, encoding)
+    rl_gen = iter(readline, b"")
+    empty = repeat(b"")
+    return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
 
 
 def _tokenize(readline, encoding):