Issue2495: tokenize.untokenize did not insert space between two consecutive string...

"" "" becomes """", which is invalid code.

Backport of r61979.

diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
index b78a223475d02e4202458edc0b76323ad8598823..9a6cebe1f52fe6c432843c05f0d94f031cb69f75 100644 (file)
--- a/Lib/test/output/test_tokenize
+++ b/Lib/test/output/test_tokenize
@@ -656,4 +656,10 @@ test_tokenize
 177,11-177,15: NAME    'pass'
 177,15-177,16: NEWLINE '\n'
 178,0-178,1:   NL      '\n'
-179,0-179,0:   ENDMARKER       ''
+179,0-179,13:  COMMENT '# Issue 2495\n'
+180,0-180,1:   NAME    'x'
+180,2-180,3:   OP      '='
+180,4-180,6:   STRING  "''"
+180,7-180,9:   STRING  "''"
+180,9-180,10:  NEWLINE '\n'
+181,0-181,0:   ENDMARKER       ''

diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenize_tests.txt
index 4ef3bf134aa20adff50b9beb3723cfc80dbec05e..c0950968ee1d6c247f35a9da5d3d167a6ae276b1 100644 (file)
--- a/Lib/test/tokenize_tests.txt
+++ b/Lib/test/tokenize_tests.txt
@@ -176,3 +176,5 @@ x = sys.modules['time'].time()
 @staticmethod
 def foo(): pass
 
+# Issue 2495
+x = '' ''

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index a9be4cfe03e0b52a7dd281fecf2745f06579d541..0db3867283cd8406ca041e2af632a6af38ac4639 100644 (file)
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -171,11 +171,12 @@ def untokenize(iterable):
         t1 = [tok[:2] for tok in generate_tokens(f.readline)]
         newcode = untokenize(t1)
         readline = iter(newcode.splitlines(1)).next
-        t2 = [tok[:2] for tokin generate_tokens(readline)]
+        t2 = [tok[:2] for tok in generate_tokens(readline)]
         assert t1 == t2
     """
 
     startline = False
+    prevstring = False
     indents = []
     toks = []
     toks_append = toks.append
@@ -185,6 +186,14 @@ def untokenize(iterable):
         if toknum in (NAME, NUMBER):
             tokval += ' '
 
+        # Insert a space between two consecutive strings
+        if toknum == STRING:
+            if prevstring:
+                tokval = ' ' + tokval
+            prevstring = True
+        else:
+            prevstring = False
+
         if toknum == INDENT:
             indents.append(tokval)
             continue

diff --git a/Misc/NEWS b/Misc/NEWS
index 0a182bd2f7d4a5edcc62477f65e21e310901a0b2..695e71b6f5452389cb6fcc1f1182b54f04ecb5f0 100644 (file)
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -27,6 +27,10 @@ Core and builtins
 Library
 -------
 
+- Issue #2495: tokenize.untokenize now inserts a space between two consecutive
+  string literals; previously, ["" ""] was rendered as [""""], which is
+  incorrect python code.
+
 - Issue #2482: Make sure that the coefficient of a Decimal is always
   stored as a str instance, not as a unicode instance.  This ensures
   that str(Decimal) is always an instance of str.  This fixes a