bpo-23894: make lib2to3 recognize f-strings (#1733)

Note: this doesn't unpack f-strings into the underlying JoinedStr AST.

Ideally we'd fully implement JoinedStr here but given its additional
complexity, I think this is worth bandaiding as is. This unblocks tools like
https://github.com/google/yapf to format 3.6 syntax using f-strings.

diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py
index 9a0cc1ea4086c7120b02b7765ba67e07aa155f34..45afc5f4e53fcf7d0f1f602638804a93726d50c1 100644 (file)
--- a/Lib/lib2to3/pgen2/tokenize.py
+++ b/Lib/lib2to3/pgen2/tokenize.py
@@ -74,7 +74,7 @@ Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 # Tail end of """ string.
 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-_litprefix = r"(?:[uUrRbB]|[rR][bB]|[bBuU][rR])?"
+_litprefix = r"(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?"
 Triple = group(_litprefix + "'''", _litprefix + '"""')
 # Single-line ' or " string.
 String = group(_litprefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
@@ -110,12 +110,14 @@ endprogs = {"'": re.compile(Single), '"': re.compile(Double),
             "r'''": single3prog, 'r"""': double3prog,
             "u'''": single3prog, 'u"""': double3prog,
             "b'''": single3prog, 'b"""': double3prog,
+            "f'''": single3prog, 'f"""': double3prog,
             "ur'''": single3prog, 'ur"""': double3prog,
             "br'''": single3prog, 'br"""': double3prog,
             "rb'''": single3prog, 'rb"""': double3prog,
             "R'''": single3prog, 'R"""': double3prog,
             "U'''": single3prog, 'U"""': double3prog,
             "B'''": single3prog, 'B"""': double3prog,
+            "F'''": single3prog, 'F"""': double3prog,
             "uR'''": single3prog, 'uR"""': double3prog,
             "Ur'''": single3prog, 'Ur"""': double3prog,
             "UR'''": single3prog, 'UR"""': double3prog,
@@ -127,6 +129,7 @@ endprogs = {"'": re.compile(Single), '"': re.compile(Double),
             "RB'''": single3prog, 'RB"""': double3prog,
             'r': None, 'R': None,
             'u': None, 'U': None,
+            'f': None, 'F': None,
             'b': None, 'B': None}
 
 triple_quoted = {}
@@ -134,6 +137,7 @@ for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "u'''", 'u"""', "U'''", 'U"""',
           "b'''", 'b"""', "B'''", 'B"""',
+          "f'''", 'f"""', "F'''", 'F"""',
           "ur'''", 'ur"""', "Ur'''", 'Ur"""',
           "uR'''", 'uR"""', "UR'''", 'UR"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
@@ -146,6 +150,7 @@ for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "u'", 'u"', "U'", 'U"',
           "b'", 'b"', "B'", 'B"',
+          "f'", 'f"', "F'", 'F"',
           "ur'", 'ur"', "Ur'", 'Ur"',
           "uR'", 'uR"', "UR'", 'UR"',
           "br'", 'br"', "Br'", 'Br"',

diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index c79611db56522a6bced8f6376798c9d42c230e55..3f7ab9714e38f9822a3b061e2631281275a01d1e 100644 (file)
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -344,6 +344,7 @@ class TestStringLiterals(GrammarTest):
         "r'", 'r"', "R'", 'R"',
         "u'", 'u"', "U'", 'U"',
         "b'", 'b"', "B'", 'B"',
+        "f'", 'f"', "F'", 'F"',
         "ur'", 'ur"', "Ur'", 'Ur"',
         "uR'", 'uR"', "UR'", 'UR"',
         "br'", 'br"', "Br'", 'Br"',