Merged revisions 88498 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/branches/py3k

........
  r88498 | brett.cannon | 2011-02-21 19:25:12 -0800 (Mon, 21 Feb 2011) | 8 lines

  Issue #11074: Make 'tokenize' so it can be reloaded.

  The module stored away the 'open' object as found in the global namespace
  (which fell through to the built-in namespace) since it defined its own 'open'.
  Problem is that if you reloaded the module it then grabbed the 'open' defined
  in the previous load, leading to code that infinite recursed. Switched to
  simply call builtins.open directly.
........

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 506aa6a42f91ee8c5b2bb5f94296718d7c57228c..f575e9bc237a9258254c29391889cae3e4137572 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -24,6 +24,7 @@ __author__ = 'Ka-Ping Yee <ping@lfw.org>'
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
+import builtins
 import re
 import sys
 from token import *
@@ -335,13 +336,11 @@ def detect_encoding(readline):
     return default, [first, second]
 
 
-_builtin_open = open
-
 def open(filename):
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
-    buffer = _builtin_open(filename, 'rb')
+    buffer = builtins.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
     text = TextIOWrapper(buffer, encoding, line_buffering=True)

diff --git a/Misc/NEWS b/Misc/NEWS
index 39abedcd83d38009a2da714c7681bf39d3cd1c7e..a5ac7c104e11084fe18ea6c0856f06a89aebd881 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -15,6 +15,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #11074: Make 'tokenize' so it can be reloaded.
+
 - Issue #4681: Allow mmap() to work on file sizes and offsets larger than
   4GB, even on 32-bit builds.  Initial patch by Ross Lagerwall, adapted for
   32-bit Windows.