From: Serhiy Storchaka Date: Sat, 2 Dec 2017 19:00:09 +0000 (+0200) Subject: [2.7] bpo-10544: Deprecate "yield" in comprehensions and generator expressions in... X-Git-Tag: v2.7.15rc1~105 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=65d1887170fb278c10a836e9e4319cae4707f524;p=python [2.7] bpo-10544: Deprecate "yield" in comprehensions and generator expressions in Py3k mode. (GH-4579) (#4676) --- diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 0f0da72b19..cabf548ec3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -847,8 +847,8 @@ def make_bad_fd(): file.close() unlink(TESTFN) -def check_syntax_error(testcase, statement, lineno=None, offset=None): - with testcase.assertRaises(SyntaxError) as cm: +def check_syntax_error(testcase, statement, errtext='', lineno=None, offset=None): + with testcase.assertRaisesRegexp(SyntaxError, errtext) as cm: compile(statement, '', 'exec') err = cm.exception if lineno is not None: diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 5e1a3e52d5..0f7bf19abb 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -1524,13 +1524,7 @@ Yield by itself yields None: [None] - -An obscene abuse of a yield expression within a generator expression: - ->>> list((yield 21) for i in range(4)) -[21, None, 21, None, 21, None, 21, None] - -And a more sane, but still weird usage: +Yield is allowed only in the outermost iterable in generator expression: >>> def f(): list(i for i in [(yield 26)]) >>> type(f()) @@ -1571,7 +1565,7 @@ SyntaxError: 'yield' outside function >>> def f(): return lambda x=(yield): 1 Traceback (most recent call last): ... -SyntaxError: 'return' with argument inside generator (, line 1) +SyntaxError: 'return' with argument inside generator (, line 1) >>> def f(): x = yield = y Traceback (most recent call last): @@ -1784,7 +1778,7 @@ enclosing function a generator: >>> type(f()) ->>> def f(): x=(i for i in (yield) if (yield)) +>>> def f(): x=(i for i in (yield) if i) >>> type(f()) diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 5f77c1d018..fc675c35a3 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -493,6 +493,46 @@ hello world def testYield(self): check_syntax_error(self, "class foo:yield 1") + def test_yield_in_comprehensions(self): + # Check yield in comprehensions + def g(): [x for x in [(yield 1)]] + + def check(code, warntext): + with check_py3k_warnings((warntext, DeprecationWarning)): + compile(code, '', 'exec') + if sys.py3kwarning: + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings('error', category=DeprecationWarning) + with self.assertRaises(SyntaxError) as cm: + compile(code, '', 'exec') + self.assertIn(warntext, str(cm.exception)) + + check("def g(): [(yield x) for x in ()]", + "'yield' inside list comprehension") + check("def g(): [x for x in () if not (yield x)]", + "'yield' inside list comprehension") + check("def g(): [y for x in () for y in [(yield x)]]", + "'yield' inside list comprehension") + check("def g(): {(yield x) for x in ()}", + "'yield' inside set comprehension") + check("def g(): {(yield x): x for x in ()}", + "'yield' inside dict comprehension") + check("def g(): {x: (yield x) for x in ()}", + "'yield' inside dict comprehension") + check("def g(): ((yield x) for x in ())", + "'yield' inside generator expression") + with check_py3k_warnings(("'yield' inside list comprehension", + DeprecationWarning)): + check_syntax_error(self, "class C: [(yield x) for x in ()]") + check("class C: ((yield x) for x in ())", + "'yield' inside generator expression") + with check_py3k_warnings(("'yield' inside list comprehension", + DeprecationWarning)): + check_syntax_error(self, "[(yield x) for x in ()]") + check("((yield x) for x in ())", + "'yield' inside generator expression") + def testRaise(self): # 'raise' test [',' test] try: raise RuntimeError, 'just testing' diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst new file mode 100644 index 0000000000..d0d3a75c4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-11-27-08-37-34.bpo-10544.07nioT.rst @@ -0,0 +1,4 @@ +Yield expressions are now deprecated in comprehensions and generator +expressions when checking Python 3 compatibility. They are still +permitted in the definition of the outermost iterable, as that is +evaluated directly in the enclosing scope. diff --git a/Python/symtable.c b/Python/symtable.c index 3b4247b415..21790b1cd1 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -162,12 +162,14 @@ PyTypeObject PySTEntry_Type = { }; static int symtable_analyze(struct symtable *st); -static int symtable_warn(struct symtable *st, char *msg, int lineno); +static int symtable_warn(struct symtable *st, + PyObject *warn, const char *msg, int lineno); static int symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, void *ast, int lineno); static int symtable_exit_block(struct symtable *st, void *ast); static int symtable_visit_stmt(struct symtable *st, stmt_ty s); static int symtable_visit_expr(struct symtable *st, expr_ty s); +static int symtable_visit_listcomp(struct symtable *st, expr_ty e); static int symtable_visit_genexp(struct symtable *st, expr_ty s); static int symtable_visit_setcomp(struct symtable *st, expr_ty e); static int symtable_visit_dictcomp(struct symtable *st, expr_ty e); @@ -796,14 +798,18 @@ symtable_analyze(struct symtable *st) static int -symtable_warn(struct symtable *st, char *msg, int lineno) +symtable_warn(struct symtable *st, PyObject *warn, const char *msg, int lineno) { - if (PyErr_WarnExplicit(PyExc_SyntaxWarning, msg, st->st_filename, - lineno, NULL, NULL) < 0) { - if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + if (lineno < 0) { + lineno = st->st_cur->ste_lineno; + } + if (PyErr_WarnExplicit(warn, msg, st->st_filename, lineno, NULL, NULL) < 0) { + if (PyErr_ExceptionMatches(warn)) { + /* Replace the warning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); PyErr_SetString(PyExc_SyntaxError, msg); - PyErr_SyntaxLocation(st->st_filename, - st->st_cur->ste_lineno); + PyErr_SyntaxLocation(st->st_filename, lineno); } return 0; } @@ -1153,7 +1159,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) PyOS_snprintf(buf, sizeof(buf), GLOBAL_AFTER_USE, c_name); - if (!symtable_warn(st, buf, s->lineno)) + if (!symtable_warn(st, PyExc_SyntaxWarning, buf, s->lineno)) return 0; } if (!symtable_add_def(st, name, DEF_GLOBAL)) @@ -1221,8 +1227,8 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_SEQ(st, expr, e->v.Set.elts); break; case ListComp_kind: - VISIT(st, expr, e->v.ListComp.elt); - VISIT_SEQ(st, comprehension, e->v.ListComp.generators); + if (!symtable_visit_listcomp(st, e)) + return 0; break; case GeneratorExp_kind: if (!symtable_visit_genexp(st, e)) @@ -1420,12 +1426,11 @@ symtable_visit_alias(struct symtable *st, alias_ty a) return r; } else { - if (st->st_cur->ste_type != ModuleBlock) { - int lineno = st->st_cur->ste_lineno; - if (!symtable_warn(st, IMPORT_STAR_WARNING, lineno)) { - Py_DECREF(store_name); - return 0; - } + if (st->st_cur->ste_type != ModuleBlock && + !symtable_warn(st, PyExc_SyntaxWarning, IMPORT_STAR_WARNING, -1)) + { + Py_DECREF(store_name); + return 0; } st->st_cur->ste_unoptimized |= OPT_IMPORT_STAR; Py_DECREF(store_name); @@ -1509,7 +1514,10 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, 0)) { return 0; } - st->st_cur->ste_generator = is_generator; + /* In order to check for yield expressions under '-3', we clear + the generator flag, and restore it at the end */ + is_generator |= st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; /* Outermost iter is received as an argument */ if (!symtable_implicit_arg(st, 0)) { symtable_exit_block(st, (void *)e); @@ -1527,9 +1535,55 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, if (value) VISIT_IN_BLOCK(st, expr, value, (void*)e); VISIT_IN_BLOCK(st, expr, elt, (void*)e); + if (Py_Py3kWarningFlag && st->st_cur->ste_generator) { + const char *msg = ( + (e->kind == SetComp_kind) ? "'yield' inside set comprehension" : + (e->kind == DictComp_kind) ? "'yield' inside dict comprehension" : + "'yield' inside generator expression"); + if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) { + symtable_exit_block(st, (void *)e); + return 0; + } + } + st->st_cur->ste_generator |= is_generator; return symtable_exit_block(st, (void *)e); } +static int +symtable_visit_listcomp(struct symtable *st, expr_ty e) +{ + asdl_seq *generators = e->v.ListComp.generators; + int i, is_generator; + /* In order to check for yield expressions under '-3', we clear + the generator flag, and restore it at the end */ + is_generator = st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; + VISIT(st, expr, e->v.ListComp.elt); + for (i = 0; i < asdl_seq_LEN(generators); i++) { + comprehension_ty lc = (comprehension_ty)asdl_seq_GET(generators, i); + VISIT(st, expr, lc->target); + if (i == 0 && !st->st_cur->ste_generator) { + /* 'yield' in the outermost iterator doesn't cause a warning */ + VISIT(st, expr, lc->iter); + is_generator |= st->st_cur->ste_generator; + st->st_cur->ste_generator = 0; + } + else { + VISIT(st, expr, lc->iter); + } + VISIT_SEQ(st, expr, lc->ifs); + } + + if (Py_Py3kWarningFlag && st->st_cur->ste_generator) { + const char *msg = "'yield' inside list comprehension"; + if (!symtable_warn(st, PyExc_DeprecationWarning, msg, -1)) { + return 0; + } + } + st->st_cur->ste_generator |= is_generator; + return 1; +} + static int symtable_visit_genexp(struct symtable *st, expr_ty e) {