bpo-10544: Disallow "yield" in comprehensions and generator expressions. (GH-4564)

diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst
index fb92ad0f07c21ed1f5c2bf9bb3505b3916b8db3c..151062ba17535a1c20911eb83cd3abf2e0cf0f16 100644 (file)
--- a/Doc/reference/expressions.rst
+++ b/Doc/reference/expressions.rst
@@ -196,8 +196,7 @@ they may depend on the values obtained from the leftmost iterable. For example:
 
 To ensure the comprehension always results in a container of the appropriate
 type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly
-nested scope (in Python 3.7, such expressions emit :exc:`DeprecationWarning`
-when compiled, in Python 3.8+ they will emit :exc:`SyntaxError`).
+nested scope.
 
 Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for`
 clause may be used to iterate over a :term:`asynchronous iterator`.
@@ -214,8 +213,8 @@ See also :pep:`530`.
 .. versionadded:: 3.6
    Asynchronous comprehensions were introduced.
 
-.. deprecated:: 3.7
-   ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+   ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _lists:
@@ -350,9 +349,7 @@ The parentheses can be omitted on calls with only one argument.  See section
 
 To avoid interfering with the expected operation of the generator expression
 itself, ``yield`` and ``yield from`` expressions are prohibited in the
-implicitly defined generator (in Python 3.7, such expressions emit
-:exc:`DeprecationWarning` when compiled, in Python 3.8+ they will emit
-:exc:`SyntaxError`).
+implicitly defined generator.
 
 If a generator expression contains either :keyword:`async for`
 clauses or :keyword:`await` expressions it is called an
@@ -368,8 +365,8 @@ which is an asynchronous iterator (see :ref:`async-iterators`).
    only appear in :keyword:`async def` coroutines.  Starting
    with 3.7, any function can use asynchronous generator expressions.
 
-.. deprecated:: 3.7
-   ``yield`` and ``yield from`` deprecated in the implicitly nested scope.
+.. versionchanged:: 3.8
+   ``yield`` and ``yield from`` prohibited in the implicitly nested scope.
 
 
 .. _yieldexpr:
@@ -401,12 +398,10 @@ coroutine function to be an asynchronous generator. For example::
 
 Due to their side effects on the containing scope, ``yield`` expressions
 are not permitted as part of the implicitly defined scopes used to
-implement comprehensions and generator expressions (in Python 3.7, such
-expressions emit :exc:`DeprecationWarning` when compiled, in Python 3.8+
-they will emit :exc:`SyntaxError`)..
+implement comprehensions and generator expressions.
 
-.. deprecated:: 3.7
-   Yield expressions deprecated in the implicitly nested scopes used to
+.. versionchanged:: 3.8
+   Yield expressions prohibited in the implicitly nested scopes used to
    implement comprehensions and generator expressions.
 
 Generator functions are described below, while asynchronous generator

diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst
index 8a3f9b0f7014984ed100b1f873c03b0b741d67d3..c4063ad7674454c102bf6dbd335051f5c6ebc163 100644 (file)
--- a/Doc/whatsnew/3.8.rst
+++ b/Doc/whatsnew/3.8.rst
@@ -113,6 +113,15 @@ This section lists previously described changes and other bugfixes
 that may require changes to your code.
 
 
+Changes in Python behavior
+--------------------------
+
+* Yield expressions (both ``yield`` and ``yield from`` clauses) are now disallowed
+  in comprehensions and generator expressions (aside from the iterable expression
+  in the leftmost :keyword:`for` clause).
+  (Contributed by Serhiy Storchaka in :issue:`10544`.)
+
+
 Changes in the Python API
 -------------------------
 

diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index 22868d4ba1a3890eac466c7d8bc75cbe07a1ddef..6c9e31a22c10ac6274a3f523a8a636703b4b74f3 100644 (file)
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1061,8 +1061,8 @@ def make_bad_fd():
         file.close()
         unlink(TESTFN)
 
-def check_syntax_error(testcase, statement, *, lineno=None, offset=None):
-    with testcase.assertRaises(SyntaxError) as cm:
+def check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None):
+    with testcase.assertRaisesRegex(SyntaxError, errtext) as cm:
         compile(statement, '<test string>', 'exec')
     err = cm.exception
     testcase.assertIsNotNone(err.lineno)

diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
index 88c22b89d444a3f93cb398c3fc1731904d806914..d89bfdc063352dbfbd1d7d0c66a7ff3616a5141e 100644 (file)
--- a/Lib/test/test_grammar.py
+++ b/Lib/test/test_grammar.py
@@ -251,6 +251,8 @@ class CNS:
 
 class GrammarTests(unittest.TestCase):
 
+    check_syntax_error = check_syntax_error
+
     # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
     # XXX can't test in a script -- this rule is only used when interactive
 
@@ -920,15 +922,7 @@ class GrammarTests(unittest.TestCase):
         def g(): [x for x in [(yield 1)]]
         def g(): [x for x in [(yield from ())]]
 
-        def check(code, warntext):
-            with self.assertWarnsRegex(DeprecationWarning, warntext):
-                compile(code, '<test string>', 'exec')
-            import warnings
-            with warnings.catch_warnings():
-                warnings.filterwarnings('error', category=DeprecationWarning)
-                with self.assertRaisesRegex(SyntaxError, warntext):
-                    compile(code, '<test string>', 'exec')
-
+        check = self.check_syntax_error
         check("def g(): [(yield x) for x in ()]",
               "'yield' inside list comprehension")
         check("def g(): [x for x in () if not (yield x)]",

diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst
new file mode 100644 (file)
index 0000000..404f12c
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-26-00-59-22.bpo-10544.fHOM3V.rst
@@ -0,0 +1,2 @@
+Yield expressions are now disallowed in comprehensions and generator
+expressions except the expression for the outermost iterable.

diff --git a/Python/symtable.c b/Python/symtable.c
index bbac25cf3767aa5bd941a3a26788c435f4a1c002..ac14058fefd238d51b900d803528a34c7dcb1e18 100644 (file)
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1754,35 +1754,18 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
         VISIT(st, expr, value);
     VISIT(st, expr, elt);
     if (st->st_cur->ste_generator) {
-        PyObject *msg = PyUnicode_FromString(
+        PyErr_SetString(PyExc_SyntaxError,
             (e->kind == ListComp_kind) ? "'yield' inside list comprehension" :
             (e->kind == SetComp_kind) ? "'yield' inside set comprehension" :
             (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" :
             "'yield' inside generator expression");
-        if (msg == NULL) {
-            symtable_exit_block(st, (void *)e);
-            return 0;
-        }
-        if (PyErr_WarnExplicitObject(PyExc_DeprecationWarning,
-                msg, st->st_filename, st->st_cur->ste_lineno,
-                NULL, NULL) == -1)
-        {
-            if (PyErr_ExceptionMatches(PyExc_DeprecationWarning)) {
-                /* Replace the DeprecationWarning exception with a SyntaxError
-                   to get a more accurate error report */
-                PyErr_Clear();
-                PyErr_SetObject(PyExc_SyntaxError, msg);
-                PyErr_SyntaxLocationObject(st->st_filename,
-                                           st->st_cur->ste_lineno,
-                                           st->st_cur->ste_col_offset);
-            }
-            Py_DECREF(msg);
-            symtable_exit_block(st, (void *)e);
-            return 0;
-        }
-        Py_DECREF(msg);
+        PyErr_SyntaxLocationObject(st->st_filename,
+                                   st->st_cur->ste_lineno,
+                                   st->st_cur->ste_col_offset);
+        symtable_exit_block(st, (void *)e);
+        return 0;
     }
-    st->st_cur->ste_generator |= is_generator;
+    st->st_cur->ste_generator = is_generator;
     return symtable_exit_block(st, (void *)e);
 }