Skip to content

Commit

Permalink
pythongh-121637: Syntax error for optimized-away incorrect await (pyt…
Browse files Browse the repository at this point in the history 
…hon#121656)

Co-authored-by: Bénédikt Tran <[email protected]>
  • Loading branch information
@JelleZijlstra @picnixz
JelleZijlstra and picnixz authored Jul 22, 2024
1 parent 69f2dc5 commit 2762c6c
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 55 deletions.
5 changes: 5 additions & 0 deletions Doc/whatsnew/3.14.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,11 @@ New Features
Other Language Changes
======================

* Incorrect usage of :keyword:`await` and asynchronous comprehensions
is now detected even if the code is optimized away by the :option:`-O`
command line option. For example, ``python -O -c 'assert await 1'``
now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.)

* Added class methods :meth:`float.from_number` and :meth:`complex.from_number`
to convert a number to :class:`float` or :class:`complex` type correspondingly.
They raise an error if the argument is a string.
Expand Down
70 changes: 45 additions & 25 deletions Lib/test/test_builtin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import random
import re
import sys
import textwrap
import traceback
import types
import typing
Expand Down Expand Up @@ -412,7 +413,7 @@ def test_compile_top_level_await_no_coro(self):
"socket.accept is broken"
)
def test_compile_top_level_await(self):
"""Test whether code some top level await can be compiled.
"""Test whether code with top level await can be compiled.
Make sure it compiles only with the PyCF_ALLOW_TOP_LEVEL_AWAIT flag
set, and make sure the generated code object has the CO_COROUTINE flag
Expand All @@ -426,6 +427,7 @@ async def arange(n):
yield i

modes = ('single', 'exec')
optimizations = (-1, 0, 1, 2)
code_samples = [
'''a = await asyncio.sleep(0, result=1)''',
'''async for i in arange(1):
Expand All @@ -438,34 +440,52 @@ async def arange(n):
'''a = [x async for x in arange(2) async for x in arange(2)][1]''',
'''a = [x async for x in (x async for x in arange(5))][1]''',
'''a, = [1 for x in {x async for x in arange(1)}]''',
'''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]'''
'''a = [await asyncio.sleep(0, x) async for x in arange(2)][1]''',
# gh-121637: Make sure we correctly handle the case where the
# async code is optimized away
'''assert not await asyncio.sleep(0); a = 1''',
'''assert [x async for x in arange(1)]; a = 1''',
'''assert {x async for x in arange(1)}; a = 1''',
'''assert {x: x async for x in arange(1)}; a = 1''',
'''
if (a := 1) and __debug__:
async with asyncio.Lock() as l:
pass
''',
'''
if (a := 1) and __debug__:
async for x in arange(2):
pass
''',
]
policy = maybe_get_event_loop_policy()
try:
for mode, code_sample in product(modes, code_samples):
source = dedent(code_sample)
with self.assertRaises(
SyntaxError, msg=f"source={source} mode={mode}"):
compile(source, '?', mode)

co = compile(source,
'?',
mode,
flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT)

self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
msg=f"source={source} mode={mode}")
for mode, code_sample, optimize in product(modes, code_samples, optimizations):
with self.subTest(mode=mode, code_sample=code_sample, optimize=optimize):
source = dedent(code_sample)
with self.assertRaises(
SyntaxError, msg=f"source={source} mode={mode}"):
compile(source, '?', mode, optimize=optimize)

# test we can create and advance a function type
globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange}
async_f = FunctionType(co, globals_)
asyncio.run(async_f())
self.assertEqual(globals_['a'], 1)

# test we can await-eval,
globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange}
asyncio.run(eval(co, globals_))
self.assertEqual(globals_['a'], 1)
co = compile(source,
'?',
mode,
flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT,
optimize=optimize)

self.assertEqual(co.co_flags & CO_COROUTINE, CO_COROUTINE,
msg=f"source={source} mode={mode}")

# test we can create and advance a function type
globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange}
async_f = FunctionType(co, globals_)
asyncio.run(async_f())
self.assertEqual(globals_['a'], 1)

# test we can await-eval,
globals_ = {'asyncio': asyncio, 'a': 0, 'arange': arange}
asyncio.run(eval(co, globals_))
self.assertEqual(globals_['a'], 1)
finally:
asyncio.set_event_loop_policy(policy)

Expand Down
4 changes: 4 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2024-07-12-18-18-44.gh-issue-121297.67VE7b.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Previously, incorrect usage of :keyword:`await` or asynchronous
comprehensions in code removed by the :option:`-O` option was not flagged by
the Python compiler. Now, such code raises :exc:`SyntaxError`. Patch by
Jelle Zijlstra.
37 changes: 14 additions & 23 deletions Python/compile.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5675,14 +5675,16 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
PyCodeObject *co = NULL;
inlined_comprehension_state inline_state = {NULL, NULL, NULL, NO_LABEL, NO_LABEL};
comprehension_ty outermost;
#ifndef NDEBUG
int scope_type = c->u->u_scope_type;
int is_top_level_await = IS_TOP_LEVEL_AWAIT(c);
#endif
PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e);
if (entry == NULL) {
goto error;
}
int is_inlined = entry->ste_comp_inlined;
int is_async_generator = entry->ste_coroutine;
int is_async_comprehension = entry->ste_coroutine;

location loc = LOC(e);

Expand All @@ -5697,22 +5699,17 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,
}
else {
if (compiler_enter_scope(c, name, COMPILER_SCOPE_COMPREHENSION,
(void *)e, e->lineno, NULL) < 0)
{
(void *)e, e->lineno, NULL) < 0) {
goto error;
}
}
Py_CLEAR(entry);

if (is_async_generator && type != COMP_GENEXP &&
scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
scope_type != COMPILER_SCOPE_COMPREHENSION &&
!is_top_level_await)
{
compiler_error(c, loc, "asynchronous comprehension outside of "
"an asynchronous function");
goto error_in_scope;
}
assert (!is_async_comprehension ||
type == COMP_GENEXP ||
scope_type == COMPILER_SCOPE_ASYNC_FUNCTION ||
scope_type == COMPILER_SCOPE_COMPREHENSION ||
is_top_level_await);

if (type != COMP_GENEXP) {
int op;
Expand Down Expand Up @@ -5777,7 +5774,7 @@ compiler_comprehension(struct compiler *c, expr_ty e, int type,

ADDOP_I(c, loc, CALL, 0);

if (is_async_generator && type != COMP_GENEXP) {
if (is_async_comprehension && type != COMP_GENEXP) {
ADDOP_I(c, loc, GET_AWAITABLE, 0);
ADDOP_LOAD_CONST(c, loc, Py_None);
ADD_YIELD_FROM(c, loc, 1);
Expand Down Expand Up @@ -6138,16 +6135,10 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
ADD_YIELD_FROM(c, loc, 0);
break;
case Await_kind:
if (!IS_TOP_LEVEL_AWAIT(c)){
if (!_PyST_IsFunctionLike(SYMTABLE_ENTRY(c))) {
return compiler_error(c, loc, "'await' outside function");
}

if (c->u->u_scope_type != COMPILER_SCOPE_ASYNC_FUNCTION &&
c->u->u_scope_type != COMPILER_SCOPE_COMPREHENSION) {
return compiler_error(c, loc, "'await' outside async function");
}
}
assert(IS_TOP_LEVEL_AWAIT(c) || (_PyST_IsFunctionLike(SYMTABLE_ENTRY(c)) && (
c->u->u_scope_type == COMPILER_SCOPE_ASYNC_FUNCTION ||
c->u->u_scope_type == COMPILER_SCOPE_COMPREHENSION
)));

VISIT(c, expr, e->v.Await.value);
ADDOP_I(c, loc, GET_AWAITABLE, 0);
Expand Down
46 changes: 39 additions & 7 deletions Python/symtable.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,10 @@
#define DUPLICATE_TYPE_PARAM \
"duplicate type parameter '%U'"

#define ASYNC_WITH_OUTISDE_ASYNC_FUNC \
#define ASYNC_WITH_OUTSIDE_ASYNC_FUNC \
"'async with' outside async function"

#define ASYNC_FOR_OUTISDE_ASYNC_FUNC \
#define ASYNC_FOR_OUTSIDE_ASYNC_FUNC \
"'async for' outside async function"

#define LOCATION(x) SRC_LOCATION_FROM_AST(x)
Expand All @@ -82,6 +82,8 @@
PyErr_RangedSyntaxLocationObject((FNAME), \
(L).lineno, (L).col_offset + 1, (L).end_lineno, (L).end_col_offset + 1)

#define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine)

static PySTEntryObject *
ste_new(struct symtable *st, identifier name, _Py_block_ty block,
void *key, _Py_SourceLocation loc)
Expand Down Expand Up @@ -1660,12 +1662,18 @@ check_import_from(struct symtable *st, stmt_ty s)
return 1;
}

static bool
allows_top_level_await(struct symtable *st)
{
return (st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
st->st_cur->ste_type == ModuleBlock;
}


static void
maybe_set_ste_coroutine_for_module(struct symtable *st, stmt_ty s)
{
if ((st->st_future->ff_features & PyCF_ALLOW_TOP_LEVEL_AWAIT) &&
(st->st_cur->ste_type == ModuleBlock))
{
if (allows_top_level_await(st)) {
st->st_cur->ste_coroutine = 1;
}
}
Expand Down Expand Up @@ -2054,15 +2062,15 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
}
case AsyncWith_kind:
maybe_set_ste_coroutine_for_module(st, s);
if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTISDE_ASYNC_FUNC, LOCATION(s))) {
if (!symtable_raise_if_not_coroutine(st, ASYNC_WITH_OUTSIDE_ASYNC_FUNC, LOCATION(s))) {
VISIT_QUIT(st, 0);
}
VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
break;
case AsyncFor_kind:
maybe_set_ste_coroutine_for_module(st, s);
if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTISDE_ASYNC_FUNC, LOCATION(s))) {
if (!symtable_raise_if_not_coroutine(st, ASYNC_FOR_OUTSIDE_ASYNC_FUNC, LOCATION(s))) {
VISIT_QUIT(st, 0);
}
VISIT(st, expr, s->v.AsyncFor.target);
Expand Down Expand Up @@ -2279,6 +2287,20 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
VISIT_QUIT(st, 0);
}
if (!allows_top_level_await(st)) {
if (!_PyST_IsFunctionLike(st->st_cur)) {
PyErr_SetString(PyExc_SyntaxError,
"'await' outside function");
SET_ERROR_LOCATION(st->st_filename, LOCATION(e));
VISIT_QUIT(st, 0);
}
if (!IS_ASYNC_DEF(st) && st->st_cur->ste_comprehension == NoComprehension) {
PyErr_SetString(PyExc_SyntaxError,
"'await' outside async function");
SET_ERROR_LOCATION(st->st_filename, LOCATION(e));
VISIT_QUIT(st, 0);
}
}
VISIT(st, expr, e->v.Await.value);
st->st_cur->ste_coroutine = 1;
break;
Expand Down Expand Up @@ -2798,6 +2820,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e,
if (!symtable_exit_block(st)) {
return 0;
}
if (is_async &&
!IS_ASYNC_DEF(st) &&
st->st_cur->ste_comprehension == NoComprehension &&
!allows_top_level_await(st))
{
PyErr_SetString(PyExc_SyntaxError, "asynchronous comprehension outside of "
"an asynchronous function");
SET_ERROR_LOCATION(st->st_filename, LOCATION(e));
return 0;
}
if (is_async) {
st->st_cur->ste_coroutine = 1;
}
Expand Down

0 comments on commit 2762c6c

Please sign in to comment.