From 198e353735be424ee7e21381504a6ab13d2f5fa7 Mon Sep 17 00:00:00 2001
From: Amaury Forgeot d'Arc <amauryfa@gmail.com>
Date: Thu, 27 Mar 2008 23:41:59 +0000
Subject: [PATCH] Issue2495: tokenize.untokenize did not insert space between
 two consecutive string literals: "" "" becomes """", which is invalid code.

Backport of r61979.
---
 Lib/test/output/test_tokenize |  8 +++++++-
 Lib/test/tokenize_tests.txt   |  2 ++
 Lib/tokenize.py               | 11 ++++++++++-
 Misc/NEWS                     |  4 ++++
 4 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
index b78a223475..9a6cebe1f5 100644
--- a/Lib/test/output/test_tokenize
+++ b/Lib/test/output/test_tokenize
@@ -656,4 +656,10 @@ test_tokenize
 177,11-177,15:	NAME	'pass'
 177,15-177,16:	NEWLINE	'\n'
 178,0-178,1:	NL	'\n'
-179,0-179,0:	ENDMARKER	''
+179,0-179,13:	COMMENT	'# Issue 2495\n'
+180,0-180,1:	NAME	'x'
+180,2-180,3:	OP	'='
+180,4-180,6:	STRING	"''"
+180,7-180,9:	STRING	"''"
+180,9-180,10:	NEWLINE	'\n'
+181,0-181,0:	ENDMARKER	''
diff --git a/Lib/test/tokenize_tests.txt b/Lib/test/tokenize_tests.txt
index 4ef3bf134a..c0950968ee 100644
--- a/Lib/test/tokenize_tests.txt
+++ b/Lib/test/tokenize_tests.txt
@@ -176,3 +176,5 @@ x = sys.modules['time'].time()
 @staticmethod
 def foo(): pass
 
+# Issue 2495
+x = '' ''
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index a9be4cfe03..0db3867283 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -171,11 +171,12 @@ def untokenize(iterable):
         t1 = [tok[:2] for tok in generate_tokens(f.readline)]
         newcode = untokenize(t1)
         readline = iter(newcode.splitlines(1)).next
-        t2 = [tok[:2] for tokin generate_tokens(readline)]
+        t2 = [tok[:2] for tok in generate_tokens(readline)]
         assert t1 == t2
     """
 
     startline = False
+    prevstring = False
     indents = []
     toks = []
     toks_append = toks.append
@@ -185,6 +186,14 @@ def untokenize(iterable):
         if toknum in (NAME, NUMBER):
             tokval += ' '
 
+        # Insert a space between two consecutive strings
+        if toknum == STRING:
+            if prevstring:
+                tokval = ' ' + tokval
+            prevstring = True
+        else:
+            prevstring = False
+
         if toknum == INDENT:
             indents.append(tokval)
             continue
diff --git a/Misc/NEWS b/Misc/NEWS
index 0a182bd2f7..695e71b6f5 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -27,6 +27,10 @@ Core and builtins
 Library
 -------
 
+- Issue #2495: tokenize.untokenize now inserts a space between two consecutive
+  string literals; previously, ["" ""] was rendered as [""""], which is
+  incorrect python code.
+
 - Issue #2482: Make sure that the coefficient of a Decimal is always
   stored as a str instance, not as a unicode instance.  This ensures
   that str(Decimal) is always an instance of str.  This fixes a
-- 
2.40.0