- Issue #11250: Back port fix from 3.3 branch, so that 2to3 can handle files

  with line feeds.  This was ported from the sandbox to the 3.3 branch, but
  didn't make it into 3.2.

- Re-enable lib2to3's test_parser.py tests, though with an expected failure
  (see issue 13125).

diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py
index bb538d537788507ffebc3fd7a4e2f1ab64678b36..0a259e90afd83844c1311de5e2f379e88af05e61 100644 (file)
--- a/Lib/lib2to3/patcomp.py
+++ b/Lib/lib2to3/patcomp.py
@@ -11,6 +11,7 @@ The compiler compiles a pattern to a pytree.*Pattern instance.
 __author__ = "Guido van Rossum <guido@python.org>"
 
 # Python imports
+import io
 import os
 
 # Fairly local imports
@@ -32,7 +33,7 @@ class PatternSyntaxError(Exception):
 def tokenize_wrapper(input):
     """Tokenizes a string suppressing significant whitespace."""
     skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
-    tokens = tokenize.generate_tokens(driver.generate_lines(input).__next__)
+    tokens = tokenize.generate_tokens(io.StringIO(input).readline)
     for quintuple in tokens:
         type, value, start, end, line_text = quintuple
         if type not in skip:

diff --git a/Lib/lib2to3/pgen2/driver.py b/Lib/lib2to3/pgen2/driver.py
index ee77a13899bfe3116325303356c7518eb8b6c5e4..e7828ffe160f37c145de6aee16a99b34bd141eda 100644 (file)
--- a/Lib/lib2to3/pgen2/driver.py
+++ b/Lib/lib2to3/pgen2/driver.py
@@ -17,6 +17,7 @@ __all__ = ["Driver", "load_grammar"]
 
 # Python imports
 import codecs
+import io
 import os
 import logging
 import sys
@@ -101,18 +102,10 @@ class Driver(object):
 
     def parse_string(self, text, debug=False):
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.generate_tokens(generate_lines(text).__next__)
+        tokens = tokenize.generate_tokens(io.StringIO(text).readline)
         return self.parse_tokens(tokens, debug)
 
 
-def generate_lines(text):
-    """Generator that behaves like readline without using StringIO."""
-    for line in text.splitlines(True):
-        yield line
-    while True:
-        yield ""
-
-
 def load_grammar(gt="Grammar.txt", gp=None,
                  save=True, force=False, logger=None):
     """Load the grammar (maybe from a pickle)."""

diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index ce39e41700474c4a89ce10ed4405c1c9e2302dfe..f32404cc1bdf3315eb736254744e79bb7274a0d5 100644 (file)
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -14,10 +14,21 @@ from .support import driver, test_dir
 
 # Python imports
 import os
+import unittest
 
 # Local imports
 from lib2to3.pgen2 import tokenize
 from ..pgen2.parse import ParseError
+from lib2to3.pygram import python_symbols as syms
+
+
+class TestDriver(support.TestCase):
+
+    def test_formfeed(self):
+        s = """print 1\n\x0Cprint 2\n"""
+        t = driver.parse_string(s)
+        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
+        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
 
 
 class GrammarTest(support.TestCase):
@@ -147,19 +158,22 @@ class TestParserIdempotency(support.TestCase):
 
     """A cut-down version of pytree_idempotency.py."""
 
+    # Issue 13125
+    @unittest.expectedFailure
     def test_all_project_files(self):
         for filepath in support.all_project_files():
             with open(filepath, "rb") as fp:
                 encoding = tokenize.detect_encoding(fp.readline)[0]
             self.assertTrue(encoding is not None,
                             "can't detect encoding for %s" % filepath)
-            with open(filepath, "r") as fp:
+            with open(filepath, "r", encoding=encoding) as fp:
                 source = fp.read()
-                source = source.decode(encoding)
-            tree = driver.parse_string(source)
+            try:
+                tree = driver.parse_string(source)
+            except ParseError as err:
+                print('ParseError on file', filepath, err)
+                continue
             new = str(tree)
-            if encoding:
-                new = new.encode(encoding)
             if diff(filepath, new):
                 self.fail("Idempotency failed: %s" % filepath)
 
@@ -202,14 +216,14 @@ class TestLiterals(GrammarTest):
         self.validate(s)
 
 
-def diff(fn, result, encoding):
-    f = open("@", "w")
-    try:
-        f.write(result.encode(encoding))
-    finally:
-        f.close()
+def diff(fn, result):
     try:
+        with open('@', 'w') as f:
+            f.write(str(result))
         fn = fn.replace('"', '\\"')
         return os.system('diff -u "%s" @' % fn)
     finally:
-        os.remove("@")
+        try:
+            os.remove("@")
+        except OSError:
+            pass

diff --git a/Lib/test/test_lib2to3.py b/Lib/test/test_lib2to3.py
index 0d6f9a36a6eadeb2a088c2a2a9308348b53a273b..1afaf70ffa499380ef9e97deeffed055a5a71f8c 100644 (file)
--- a/Lib/test/test_lib2to3.py
+++ b/Lib/test/test_lib2to3.py
@@ -1,6 +1,7 @@
 # Skipping test_parser and test_all_fixers
 # because of running
 from lib2to3.tests import (test_fixers, test_pytree, test_util, test_refactor,
+                           test_parser,
                            test_main as test_main_)
 import unittest
 from test.support import run_unittest
@@ -9,7 +10,7 @@ def suite():
     tests = unittest.TestSuite()
     loader = unittest.TestLoader()
     for m in (test_fixers, test_pytree,test_util, test_refactor,
-              test_main_):
+              test_parser, test_main_):
         tests.addTests(loader.loadTestsFromModule(m))
     return tests
 

diff --git a/Misc/NEWS b/Misc/NEWS
index 44435fef36d50021af719303cea50751683c6a08..f49097d000c2b0abbcc469aac05139bd59116898 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -39,6 +39,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #11250: Back port fix from 3.3 branch, so that 2to3 can handle files
+  with line feeds.  This was ported from the sandbox to the 3.3 branch, but
+  didn't make it into 3.2.
+
 - Issue #7367: Fix pkgutil.walk_paths to skip directories whose
   contents cannot be read.
 
@@ -101,6 +105,9 @@ Tests
 
 - Issue #12821: Fix test_fcntl failures on OpenBSD 5.
 
+- Re-enable lib2to3's test_parser.py tests, though with an expected failure
+  (see issue 13125).
+
 Extension Modules
 -----------------