From: Matthias Bussonnier Date: Tue, 21 May 2019 20:12:03 +0000 (-0700) Subject: bpo-34616: Add PyCF_ALLOW_TOP_LEVEL_AWAIT to allow top-level await (GH-13148) X-Git-Tag: v3.8.0b1~268 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=565b4f1ac7304d1e690c404ca8316f383ba60862;p=python bpo-34616: Add PyCF_ALLOW_TOP_LEVEL_AWAIT to allow top-level await (GH-13148) Co-Authored-By: Yury Selivanov --- diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 613e4f74ac..1a9a8b5bee 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -257,6 +257,12 @@ are always available. They are listed here in alphabetical order. can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on the :class:`~__future__._Feature` instance in the :mod:`__future__` module. + The optional argument *flags* also controls whether the compiled source is + allowed to contain top-level ``await``, ``async for`` and ``async with``. + When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code + object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively + executed via ``await eval(code_object)``. + The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as given by :option:`-O` options. Explicit levels are ``0`` (no optimization; @@ -290,6 +296,10 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. + .. versionadded:: 3.8 + ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable + support for top-level ``await``, ``async for``, and ``async with``. + .. class:: complex([real[, imag]]) diff --git a/Include/compile.h b/Include/compile.h index 1370867867..a833caa06b 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -23,6 +23,7 @@ PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); #define PyCF_ONLY_AST 0x0400 #define PyCF_IGNORE_COOKIE 0x0800 #define PyCF_TYPE_COMMENTS 0x1000 +#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000 #ifndef Py_LIMITED_API typedef struct { diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 5674ea89b1..4a358e89d1 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1,6 +1,7 @@ # Python test set -- built-in functions import ast +import asyncio import builtins import collections import decimal @@ -18,9 +19,14 @@ import types import unittest import warnings from contextlib import ExitStack +from inspect import CO_COROUTINE +from itertools import product +from textwrap import dedent +from types import AsyncGeneratorType, FunctionType from operator import neg from test.support import ( - EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink) + EnvironmentVarGuard, TESTFN, check_warnings, swap_attr, unlink, + maybe_get_event_loop_policy) from test.support.script_helper import assert_python_ok from unittest.mock import MagicMock, patch try: @@ -358,6 +364,71 @@ class BuiltinTest(unittest.TestCase): rv = ns['f']() self.assertEqual(rv, tuple(expected)) + def test_compile_top_level_await(self): + """Test whether code some top level await can be compiled. + + Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag set, + and make sure the generated code object has the CO_COROUTINE flag set in + order to execute it with `await eval(.....)` instead of exec, or via a + FunctionType. + """ + + # helper function just to check we can run top=level async-for + async def arange(n): + for i in range(n): + yield i + + modes = ('single', 'exec') + code_samples = ['''a = await asyncio.sleep(0, result=1)''', + '''async for i in arange(1): + a = 1''', + '''async with asyncio.Lock() as l: + a = 1'''] + policy = maybe_get_event_loop_policy() + try: + for mode, code_sample in product(modes,code_samples): + source = dedent(code_sample) + with self.assertRaises(SyntaxError, msg=f"{source=} {mode=}"): + compile(source, '?' , mode) + + co = compile(source, + '?', + mode, + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + + self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE, + msg=f"{source=} {mode=}") + + + # test we can create and advance a function type + globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange} + async_f = FunctionType(co, globals_) + asyncio.run(async_f()) + self.assertEqual(globals_['a'], 1) + + # test we can await-eval, + globals_ = {'asyncio': asyncio, 'a':0, 'arange': arange} + asyncio.run(eval(co, globals_)) + self.assertEqual(globals_['a'], 1) + finally: + asyncio.set_event_loop_policy(policy) + + def test_compile_async_generator(self): + """ + With the PyCF_ALLOW_TOP_LEVEL_AWAIT flag added in 3.8, we want to + make sure AsyncGenerators are still properly not marked with CO_COROUTINE + """ + code = dedent("""async def ticker(): + for i in range(10): + yield i + await asyncio.sleep(0)""") + + co = compile(code, '?', 'exec', flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT) + glob = {} + exec(co, glob) + self.assertEqual(type(glob['ticker']()), AsyncGeneratorType) + + def test_delattr(self): sys.spam = 1 delattr(sys, 'spam') diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst new file mode 100644 index 0000000000..c264d21bd0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-17-12-37.bpo-34616.0Y0_9r.rst @@ -0,0 +1 @@ +The ``compile()`` builtin functions now support the ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` flag, which allow to compile sources that contains top-level ``await``, ``async with`` or ``async for``. This is useful to evaluate async-code from with an already async functions; for example in a custom REPL. \ No newline at end of file diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 4091b6db63..cb0e6d7f9d 100644 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1000,6 +1000,8 @@ class ASTModuleVisitor(PickleVisitor): self.emit("if (!m) return NULL;", 1) self.emit("d = PyModule_GetDict(m);", 1) self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1) + self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0)', 1) + self.emit("return NULL;", 2) self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1) self.emit("return NULL;", 2) self.emit('if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0)', 1) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index cb53a41cdf..5527505844 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -8776,6 +8776,8 @@ PyInit__ast(void) if (!m) return NULL; d = PyModule_GetDict(m); if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL; + if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) + return NULL; if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0) return NULL; if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) diff --git a/Python/compile.c b/Python/compile.c index 63b2456bb3..734e8401ff 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2609,7 +2609,9 @@ static int compiler_async_for(struct compiler *c, stmt_ty s) { basicblock *start, *except, *end; - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { + if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){ + c->u->u_ste->ste_coroutine = 1; + } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { return compiler_error(c, "'async for' outside async function"); } @@ -4564,7 +4566,9 @@ compiler_async_with(struct compiler *c, stmt_ty s, int pos) withitem_ty item = asdl_seq_GET(s->v.AsyncWith.items, pos); assert(s->kind == AsyncWith_kind); - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION) { + if (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT){ + c->u->u_ste->ste_coroutine = 1; + } else if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION){ return compiler_error(c, "'async with' outside async function"); } @@ -4773,12 +4777,16 @@ compiler_visit_expr1(struct compiler *c, expr_ty e) ADDOP(c, YIELD_FROM); break; case Await_kind: - if (c->u->u_ste->ste_type != FunctionBlock) - return compiler_error(c, "'await' outside function"); + if (!(c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT)){ + if (c->u->u_ste->ste_type != FunctionBlock){ + return compiler_error(c, "'await' outside function"); + } - if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && - c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) - return compiler_error(c, "'await' outside async function"); + if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION && + c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION){ + return compiler_error(c, "'await' outside async function"); + } + } VISIT(c, expr, e->v.Await.value); ADDOP(c, GET_AWAITABLE); @@ -5712,6 +5720,12 @@ compute_code_flags(struct compiler *c) /* (Only) inherit compilerflags in PyCF_MASK */ flags |= (c->c_flags->cf_flags & PyCF_MASK); + if ((c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) && + ste->ste_coroutine && + !ste->ste_generator) { + flags |= CO_COROUTINE; + } + return flags; }