From 5b4cc3ef4d36f6c3ea50bc2311f84c61065a05c6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 3 Sep 2026 10:01:08 +0100 Subject: [PATCH 1/9] [mypyc] Add am_send slot and use PyIter_Send for await/yield from Generated generator and coroutine classes now install an am_send slot that adapts __mypyc_generator_helper__ to CPython's C send protocol. Yielded values use PYGEN_NEXT, normal returns use PYGEN_RETURN and an out parameter, and errors use PYGEN_ERROR. StopIteration escaping the helper also maps to PYGEN_RETURN, preserving the known mypyc bug where an explicit StopIteration in a generator body is treated as normal completion. CPyIter_Send now wraps PyIter_Send and returns normal completion through an out parameter. The generic await/yield from path uses it for the initial advance and ordinary resumed sends, avoiding StopIteration overhead. Statically known native children retain the direct generator-helper fast path. asyncio's C Task also reaches generated coroutines through the new am_send slot. Across 8 interleaved benchmark rounds on CPython 3.14, median gains are 25-26% for dynamically typed immediate awaits, 33-37% for dynamically typed suspended awaits, 20% for one statically typed suspended await, and 26-30% for two. Native coroutine chains of length 5 and 20 improve by about 10% and 3-5%. Immediate static awaits are unchanged, and uvloop results are inconclusive. Since the baseline already optimizes statically known resumed children, chain20 improves from 1.75x to 1.83x interpreted speed with the GIL and from 1.63x to 1.67x with free threading. --- mypyc/codegen/emitclass.py | 5 ++ mypyc/codegen/emitmodule.py | 3 +- mypyc/codegen/emitwrapper.py | 39 ++++++++++++++++ mypyc/common.py | 1 + mypyc/irbuild/classdef.py | 3 +- mypyc/irbuild/for_helpers.py | 3 +- mypyc/irbuild/generator.py | 8 +++- mypyc/irbuild/prepare.py | 5 +- mypyc/irbuild/statement.py | 71 +++++++++++++++-------------- mypyc/lib-rt/CPy.h | 2 +- mypyc/lib-rt/misc_ops.c | 20 ++++++-- mypyc/primitives/misc_ops.py | 14 +++--- mypyc/test-data/run-async.test | 63 +++++++++++++++++++++++++ mypyc/test-data/run-generators.test | 63 +++++++++++++++++++++++++ 14 files changed, 245 insertions(+), 55 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 9a1b2efca8156..648f282c3361e 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -14,6 +14,7 @@ ) from mypyc.codegen.emitfunc import native_function_header from mypyc.codegen.emitwrapper import ( + generate_am_send_wrapper, generate_bin_op_wrapper, generate_bool_wrapper, generate_contains_wrapper, @@ -29,6 +30,7 @@ BITMAP_BITS, BITMAP_TYPE, CPYFUNCTION_NAME, + GENERATOR_HELPER_NAME, IS_FREE_THREADED, MYPYC_DEFAULTS_SETUP, NATIVE_PREFIX, @@ -149,6 +151,9 @@ def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__await__": ("am_await", native_slot), "__aiter__": ("am_aiter", native_slot), "__anext__": ("am_anext", native_slot), + # Generator classes get an am_send slot based on the generator helper method, + # so that PyIter_Send can drive them without raising StopIteration. + GENERATOR_HELPER_NAME: ("am_send", generate_am_send_wrapper), } SIDE_TABLES = [ diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index ccf8081251a5e..b8039dcca823c 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -47,6 +47,7 @@ from mypyc.codegen.literals import Literals from mypyc.common import ( EXT_SUFFIX, + GENERATOR_HELPER_NAME, IS_FREE_THREADED, MODULE_PREFIX, PREFIX, @@ -75,7 +76,7 @@ from mypyc.ir.rtypes import RType from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper -from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME, load_type_map +from mypyc.irbuild.prepare import load_type_map from mypyc.namegen import NameGenerator, exported_name from mypyc.options import CompilerOptions from mypyc.transform.copy_propagation import do_copy_propagation diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index bc255f67cb75f..a6858613805e5 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -595,6 +595,45 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: return name +def generate_am_send_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: + """Generate an am_send slot for a generator class based on the generator helper method. + + This implements the C-level send protocol (used by PyIter_Send), which reports + normal completion without raising StopIteration. + """ + name = f"{DUNDER_PREFIX}am_send{cl.name_prefix(emitter.names)}" + emitter.emit_line( + f"static PySendResult {name}(PyObject *self, PyObject *arg, PyObject **result) {{" + ) + emitter.emit_line("PyObject *stop_iter_value = NULL;") + emitter.emit_line( + "PyObject *retval = {}{}{}(self, Py_None, Py_None, Py_None, arg, &stop_iter_value);".format( + emitter.get_group_prefix(fn.decl), NATIVE_PREFIX, fn.cname(emitter.names) + ) + ) + emitter.emit_line("if (retval != NULL) {") + emitter.emit_line("*result = retval;") + emitter.emit_line("return PYGEN_NEXT;") + emitter.emit_line("}") + emitter.emit_line("if (stop_iter_value == NULL) {") + # The helper raised an exception instead of using the out parameter. A + # StopIteration still signals normal completion (e.g. if the generator has + # already completed, or if the body raised StopIteration explicitly). + emitter.emit_line("if (PyErr_ExceptionMatches(PyExc_StopIteration)) {") + emitter.emit_line("stop_iter_value = CPy_FetchStopIterationValue();") + emitter.emit_line("}") + emitter.emit_line("if (stop_iter_value == NULL) {") + emitter.emit_line("*result = NULL;") + emitter.emit_line("return PYGEN_ERROR;") + emitter.emit_line("}") + emitter.emit_line("}") + emitter.emit_line("*result = stop_iter_value;") + emitter.emit_line("return PYGEN_RETURN;") + emitter.emit_line("}") + + return name + + def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: """Generates a wrapper for native __bool__ methods.""" name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}" diff --git a/mypyc/common.py b/mypyc/common.py index f76a3f189dc4f..fea09582d5f5d 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -20,6 +20,7 @@ ENV_ATTR_NAME: Final = "__mypyc_env__" NEXT_LABEL_ATTR_NAME: Final = "__mypyc_next_label__" +GENERATOR_HELPER_NAME: Final = "__mypyc_generator_helper__" TEMP_ATTR_NAME: Final = "__mypyc_temp__" LAMBDA_NAME: Final = "__mypyc_lambda__" PROPSET_PREFIX: Final = "__mypyc_setter__" diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 61587b46161f7..3eb617cf86550 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -30,7 +30,7 @@ is_class_var, ) from mypy.types import Instance, UnboundType, get_proper_type -from mypyc.common import MYPYC_DEFAULTS_SETUP, PROPSET_PREFIX +from mypyc.common import GENERATOR_HELPER_NAME, MYPYC_DEFAULTS_SETUP, PROPSET_PREFIX from mypyc.ir.class_ir import ClassIR, NonExtClassInfo from mypyc.ir.func_ir import FuncDecl, FuncSignature from mypyc.ir.ops import ( @@ -58,7 +58,6 @@ handle_non_ext_method, load_type, ) -from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.util import ( dataclass_type, default_attr_name, diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index cecc69a07e274..00315781c6840 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -28,7 +28,7 @@ Var, ) from mypy.types import LiteralType, TupleType, get_proper_type, get_proper_types -from mypyc.common import IS_FREE_THREADED +from mypyc.common import GENERATOR_HELPER_NAME, IS_FREE_THREADED from mypyc.ir.ops import ( ERR_NEVER, BasicBlock, @@ -68,7 +68,6 @@ ) from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.constant_fold import constant_fold_expr -from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple from mypyc.irbuild.vec import vec_append, vec_create, vec_get_item_unsafe, vec_init_item_unsafe from mypyc.primitives.dict_ops import ( diff --git a/mypyc/irbuild/generator.py b/mypyc/irbuild/generator.py index e5d0bf3aa565e..40791e8326f8d 100644 --- a/mypyc/irbuild/generator.py +++ b/mypyc/irbuild/generator.py @@ -13,7 +13,12 @@ from collections.abc import Callable from mypy.nodes import ARG_OPT, FuncDef, Var -from mypyc.common import ENV_ATTR_NAME, GENERATOR_ATTRIBUTE_PREFIX, NEXT_LABEL_ATTR_NAME +from mypyc.common import ( + ENV_ATTR_NAME, + GENERATOR_ATTRIBUTE_PREFIX, + GENERATOR_HELPER_NAME, + NEXT_LABEL_ATTR_NAME, +) from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FuncDecl, FuncIR from mypyc.ir.ops import ( @@ -51,7 +56,6 @@ setup_func_for_recursive_call, ) from mypyc.irbuild.nonlocalcontrol import ExceptNonlocalControl, gen_generator_func_cleanup -from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTargetAttr from mypyc.primitives.exc_ops import ( error_catch_op, diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index fdca2fd49e6f5..99ce7de8df1f7 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -15,7 +15,7 @@ from collections import defaultdict from collections.abc import Iterable -from typing import Final, NamedTuple +from typing import NamedTuple from mypy.build import Graph from mypy.nodes import ( @@ -42,6 +42,7 @@ from mypy.types import Instance, Type, get_proper_type from mypyc.common import ( FAST_PREFIX, + GENERATOR_HELPER_NAME, MYPYC_DEFAULTS_SETUP, PROPSET_PREFIX, SELF_NAME, @@ -83,8 +84,6 @@ from mypyc.options import CompilerOptions from mypyc.sametype import is_same_type -GENERATOR_HELPER_NAME: Final = "__mypyc_generator_helper__" - def build_type_map( mapper: Mapper, diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index d978265339c45..bc9fe03d55b9a 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,7 +49,12 @@ YieldExpr, YieldFromExpr, ) -from mypyc.common import KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, TEMP_ATTR_NAME +from mypyc.common import ( + GENERATOR_HELPER_NAME, + KEEP_ALIVE_SHORT_LIVED, + KEEP_ALIVE_WHOLE_EXPRESSION, + TEMP_ATTR_NAME, +) from mypyc.ir.ops import ( ERR_NEVER, NAMESPACE_MODULE, @@ -100,7 +105,6 @@ FinallyNonlocalControl, TryFinallyNonlocalControl, ) -from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.specialize import apply_dunder_specialization from mypyc.irbuild.targets import ( AssignmentTarget, @@ -121,10 +125,9 @@ reraise_exception_op, restore_exc_info_op, ) -from mypyc.primitives.generic_ops import iter_op, next_raw_op, py_delattr_op +from mypyc.primitives.generic_ops import iter_op, py_delattr_op from mypyc.primitives.librt_threading_ops import lock_acquire_op, lock_release_op from mypyc.primitives.misc_ops import ( - check_stop_op, coro_op, get_native_attrs_op, import_from_many_op, @@ -1355,9 +1358,16 @@ def emit_yield_from_or_await( helper_method ) - # Register where a native child stores its return value instead of raising - # StopIteration (only used on the fast path). - stop_iter_val = Register(object_rprimitive) if fast_path else None + # Neither the direct helper call nor CPyIter_Send raises StopIteration on normal + # completion. Instead they store the return value through a PyObject ** argument, + # which must be cleared before each call. + stop_iter_val = Register(object_rprimitive) + + def clear_stop_iter_val() -> Value: + """Reset the stop iteration value and return a pointer to it.""" + err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) + builder.assign(stop_iter_val, err, line) + return builder.add(LoadAddress(object_pointer_rprimitive, stop_iter_val)) def native_step(sent: Value) -> Value: """Advance a native generator/coroutine child by calling its helper directly. @@ -1366,12 +1376,9 @@ def native_step(sent: Value) -> Value: raised. On normal completion the return value is stored in stop_iter_val, which is set to the error value if a real exception was raised instead. """ - assert stop_iter_val is not None obj = builder.read(iter_reg, line) nn = builder.none_object() - err = builder.add(LoadErrorValue(object_rprimitive, undefines=True)) - builder.assign(stop_iter_val, err, line) - ptr = builder.add(LoadAddress(object_pointer_rprimitive, stop_iter_val)) + ptr = clear_stop_iter_val() m = MethodCall(obj, helper_method, [nn, nn, nn, sent, ptr], line) # Generators have custom error handling, so disable normal error handling. m.error_kind = ERR_NEVER @@ -1384,19 +1391,19 @@ def native_step(sent: Value) -> Value: # an extra PyObject ** argument to helper where the stop iteration value is stored. _y_init = native_step(builder.none_object()) else: - _y_init = builder.call_c(next_raw_op, [builder.read(iter_reg, line)], line) + # Use PyIter_Send (via CPyIter_Send) for the initial advance as well, so that + # a natively compiled iterator can complete through its am_send slot without + # raising StopIteration, even if we don't know the type statically. + ptr = clear_stop_iter_val() + _y_init = builder.call_c( + send_op, [builder.read(iter_reg, line), builder.none_object(), ptr], line + ) builder.add(Branch(_y_init, stop_block, main_block, Branch.IS_ERROR)) builder.activate_block(stop_block) - if fast_path: - assert stop_iter_val is not None - builder.primitive_op(propagate_if_error_op, [stop_iter_val], line) - builder.assign(result, stop_iter_val, line) - else: - # Try extracting a return value from a StopIteration and return it. - # If it wasn't, this reraises the exception. - builder.assign(result, builder.call_c(check_stop_op, [], line), line) + builder.primitive_op(propagate_if_error_op, [stop_iter_val], line) + builder.assign(result, stop_iter_val, line) # Clear the spilled iterator/coroutine so that it will be freed. # Otherwise, the freeing of the spilled register would likely be delayed. err = builder.add(LoadErrorValue(iter_reg.type)) @@ -1446,10 +1453,13 @@ def else_body() -> None: # completion doesn't have to go through .send() and StopIteration. _y = native_step(builder.read(received_reg, line)) else: - # Do a next() or a .send(). It will return NULL on exception - # but it won't automatically propagate. + # Do a next() or a .send(). It will return NULL on exception or on normal + # completion, but it won't automatically propagate an error. + ptr = clear_stop_iter_val() _y = builder.call_c( - send_op, [builder.read(iter_reg, line), builder.read(received_reg, line)], line + send_op, + [builder.read(iter_reg, line), builder.read(received_reg, line), ptr], + line, ) ok, stop = BasicBlock(), BasicBlock() builder.add(Branch(_y, stop, ok, Branch.IS_ERROR)) @@ -1459,17 +1469,12 @@ def else_body() -> None: builder.assign(to_yield_reg, _y, line) builder.nonlocal_control[-1].gen_continue(builder, line) + # The iterator completed (or raised). If it completed normally, the + # return value is the result of the await/yield from. Otherwise + # propagate the exception. builder.activate_block(stop) - if fast_path: - assert stop_iter_val is not None - # The child either returned a value through the out pointer, or raised - # a real exception (in which case this propagates it). - builder.primitive_op(propagate_if_error_op, [stop_iter_val], line) - builder.assign(result, stop_iter_val, line) - else: - # Try extracting a return value from a StopIteration and return it. - # If it wasn't, this rereaises the exception. - builder.assign(result, builder.call_c(check_stop_op, [], line), line) + builder.primitive_op(propagate_if_error_op, [stop_iter_val], line) + builder.assign(result, stop_iter_val, line) builder.nonlocal_control[-1].gen_break(builder, line) builder.push_loop_stack(loop_block, done_block) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 502c8c3bc1ff9..8d2335fead0c2 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -992,7 +992,7 @@ static inline PyObject *CPy_TYPE(PyObject *obj) { PyObject *CPy_CalculateMetaclass(PyObject *type, PyObject *o); PyObject *CPy_GetCoro(PyObject *obj); PyObject *CPyGen_AlreadyExecutingError(int is_coroutine); -PyObject *CPyIter_Send(PyObject *iter, PyObject *val); +PyObject *CPyIter_Send(PyObject *iter, PyObject *val, PyObject **stop_iter_value); int CPy_YieldFromErrorHandle(PyObject *iter, PyObject **outp); PyObject *CPy_FetchStopIterationValue(void); PyObject *CPyType_FromTemplate(PyObject *template_, diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 621237063ae08..39461cc1c3bdc 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -31,14 +31,24 @@ PyObject *CPyGen_AlreadyExecutingError(int is_coroutine) return NULL; } -PyObject *CPyIter_Send(PyObject *iter, PyObject *val) +PyObject *CPyIter_Send(PyObject *iter, PyObject *val, PyObject **stop_iter_value) { // Do a send, or a next if second arg is None. // (This behavior is to match the PEP 380 spec for yield from.) - if (Py_IsNone(val)) { - return CPyIter_Next(iter); - } else { - return PyObject_CallMethodOneArg(iter, mypyc_interned_str.send, val); + // + // If the iterator yields a value, return it. If it completes normally, + // return NULL and store the return value in *stop_iter_value, without + // raising StopIteration. If it raises, return NULL and leave + // *stop_iter_value untouched (the caller initializes it to NULL). + PyObject *result; + switch (PyIter_Send(iter, val, &result)) { + case PYGEN_NEXT: + return result; + case PYGEN_RETURN: + *stop_iter_value = result; + return NULL; + default: + return NULL; } } diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 7b78b61f50e26..fbb27470c1630 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -74,13 +74,15 @@ error_kind=ERR_MAGIC, ) -# Do obj.send(value), or a next(obj) if second arg is None. -# (This behavior is to match the PEP 380 spec for yield from.) -# Like next_raw_op, don't swallow StopIteration, -# but also don't propagate an error. -# Can return NULL: see next_op. +# Do obj.send(value), or a next(obj) if second arg is None, using the +# PyIter_Send C API. (This behavior is to match the PEP 380 spec for yield from.) +# +# Returns the yielded value, or NULL if the iterator completed or raised. On +# normal completion the return value is stored via the third (PyObject **) +# argument instead of raising StopIteration. The caller must initialize the +# pointed-to value to NULL, since it's only assigned on normal completion. send_op = custom_op( - arg_types=[object_rprimitive, object_rprimitive], + arg_types=[object_rprimitive, object_rprimitive, object_pointer_rprimitive], return_type=object_rprimitive, c_function_name="CPyIter_Send", error_kind=ERR_NEVER, diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index bdffc658486a1..0f30bac982c60 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2498,3 +2498,66 @@ def test_close_suspended_native_child() -> None: coro.close() [typing fixtures/typing-full.pyi] + +[case testRunAsyncNativeCoroutineDrivenByInterpretedCode] +from typing import Any, Generator + +class Suspend: + def __await__(self) -> Generator[Any, Any, None]: + yield None + +async def inc(x: int) -> int: + return x + 1 + +async def suspend_then_inc(x: int) -> int: + await Suspend() + return x + 1 + +async def nested(x: int) -> int: + return await suspend_then_inc(x) + await inc(x) + +class MyError(Exception): + pass + +async def fails() -> int: + await Suspend() + raise MyError() + +[file driver.py] +# Interpreted 'await' and asyncio drive native coroutines through the C-level +# am_send slot (PyIter_Send), which reports completion without raising +# StopIteration. +import asyncio + +from native import inc, suspend_then_inc, nested, fails, MyError + +async def main() -> None: + assert await inc(1) == 2 + assert await suspend_then_inc(1) == 2 + assert await nested(3) == 8 + assert await asyncio.gather(inc(5), nested(5)) == [6, 12] + try: + await fails() + except MyError: + pass + else: + assert False + +asyncio.run(main()) + +# Resuming a completed native coroutine raises StopIteration. +coro = inc(1) +try: + coro.send(None) +except StopIteration as e: + assert e.value == 2 +else: + assert False +try: + coro.send(None) +except StopIteration: + pass +else: + assert False + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 8e09fb881310d..b794cad10fb76 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1138,3 +1138,66 @@ def test_generator_method_override_still_works() -> None: box[0] = g with assertRaises(ValueError, "generator already executing"): next(g) + +[case testYieldFromNativeGeneratorInInterpretedCode] +from typing import Generator + +def native(n: int) -> Generator[int, int, str]: + x = yield n + yield x + 1 + return "done" + +def exhausted() -> Generator[int, None, int]: + yield 1 + return 5 + +def raises_stop_iteration() -> Generator[int, None, object]: + if int() == 1: + yield 1 + raise StopIteration("seven") + +def native_yield_from_stop_iteration() -> Generator[int, None, object]: + x = yield from raises_stop_iteration() + return x + +[file driver.py] +# Interpreted 'yield from' drives native generators through the C-level +# am_send slot (PyIter_Send), which reports completion without raising +# StopIteration. +from native import ( + native, exhausted, raises_stop_iteration, native_yield_from_stop_iteration +) + +def outer(gen): + x = yield from gen + return x + +def run(gen, sends): + g = outer(gen) + yields = [] + try: + value = g.send(None) + for s in sends: + yields.append(value) + value = g.send(s) + yields.append(value) + except StopIteration as e: + return yields, e.value + assert False, "generator did not stop" + +assert run(native(1), [10, None]) == ([1, 11], "done") + +# An already exhausted native generator completes with None, like a +# CPython generator does. +g = exhausted() +assert next(g) == 1 +try: + next(g) +except StopIteration as e: + assert e.value == 5 +assert run(g, []) == ([], None) + +# StopIteration raised in a generator body is treated as a return value by +# 'yield from', both in interpreted and in native code. +assert run(raises_stop_iteration(), []) == ([], "seven") +assert run(native_yield_from_stop_iteration(), []) == ([], "seven") From 4d8e810e553e9c98b6f3c87a1284fda77f7c1743 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 10:17:55 +0100 Subject: [PATCH 2/9] Update tests --- mypyc/test-data/fixtures/testutil.py | 33 ++++++++++++ mypyc/test-data/run-async.test | 65 +++++++++++++++++++++--- mypyc/test-data/run-generators.test | 75 ++++++++++++++++++++++++++-- 3 files changed, 164 insertions(+), 9 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 60e99256c5671..9aeb0cc26e21d 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -86,6 +86,39 @@ def run_generator(gen: Generator[T, V, U], F = TypeVar('F', bound=Callable) +# Result codes of CPython's C-level send protocol (PySendResult). +PYGEN_RETURN: Final = 0 +PYGEN_ERROR: Final = -1 +PYGEN_NEXT: Final = 1 + + +def pyiter_send(it: Any, arg: Any) -> Tuple[int, Any]: + """Resume an iterator using PyIter_Send, the C-level send protocol. + + This calls the am_send slot, if the type of 'it' has one, so it's the only way to + exercise am_send from Python. It's the path asyncio's C Task and compiled + 'await'/'yield from' use to resume a generator or coroutine. Note that the + interpreter's own SEND opcode does not use it on Python 3.12 and later. + + Return (result code, value), where the code is PYGEN_NEXT for a yielded value and + PYGEN_RETURN for normal completion -- completion is reported without raising + StopIteration. If the iterator raises, the exception propagates, since ctypes + re-raises whatever exception is pending after the call. + """ + import ctypes + + send = ctypes.pythonapi.PyIter_Send + send.argtypes = [ctypes.py_object, ctypes.py_object, ctypes.POINTER(ctypes.py_object)] + send.restype = ctypes.c_int + result = ctypes.py_object() + code = send(it, arg, ctypes.byref(result)) + assert code in (PYGEN_NEXT, PYGEN_RETURN), f"unexpected PySendResult {code}" + try: + value = result.value + except ValueError: + assert False, f"PyIter_Send returned {code} but left *result NULL" + return code, value + class async_val(Awaitable[V], Generic[T, V]): def __init__(self, val: T) -> None: diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 0f30bac982c60..95f5059043cb1 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2524,9 +2524,11 @@ async def fails() -> int: raise MyError() [file driver.py] -# Interpreted 'await' and asyncio drive native coroutines through the C-level -# am_send slot (PyIter_Send), which reports completion without raising -# StopIteration. +# Interpreted 'await' resumes a native coroutine through send(), i.e. through the +# Python-level protocol, since the SEND opcode calls the send method directly on +# Python 3.12 and later. asyncio's C Task uses PyIter_Send instead, so it goes through +# the C-level am_send slot, which reports completion without raising StopIteration. +# See also testAmSendSlotOnNativeCoroutine. import asyncio from native import inc, suspend_then_inc, nested, fails, MyError @@ -2545,7 +2547,6 @@ async def main() -> None: asyncio.run(main()) -# Resuming a completed native coroutine raises StopIteration. coro = inc(1) try: coro.send(None) @@ -2553,11 +2554,63 @@ except StopIteration as e: assert e.value == 2 else: assert False + +# Resuming a completed native coroutine raises a bare StopIteration, which means that +# awaiting it a second time evaluates to None. +# +# This is a known deviation from Python semantics, not the desired behavior: CPython +# raises RuntimeError("cannot reuse already awaited coroutine") instead. This assertion +# pins down the current behavior so that a future fix shows up as a test failure here. +# It's pre-existing and independent of the am_send slot, which faithfully reports what +# the generator helper does. try: coro.send(None) -except StopIteration: - pass +except StopIteration as e: + assert e.value is None else: assert False [typing fixtures/typing-full.pyi] + +[case testAmSendSlotOnNativeCoroutine] +from typing import Any, Generator + +class Suspend: + def __await__(self) -> Generator[Any, Any, int]: + x = yield None + return x + +async def inc(x: int) -> int: + return x + 1 + +async def suspends(x: int) -> int: + y = await Suspend() + return x + y + +class MyError(Exception): + pass + +async def fails() -> int: + raise MyError() + +[file driver.py] +# Coroutine classes have an am_send slot implementing CPython's C-level send protocol, +# which reports normal completion as PYGEN_RETURN instead of raising StopIteration. +# asyncio's C Task resumes coroutines through it, so drive it directly via PyIter_Send +# here. +from native import inc, suspends, fails, MyError +from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN + +# Completing without suspending. +assert pyiter_send(inc(1), None) == (PYGEN_RETURN, 2) + +# Suspending once, then completing with a value sent in. +coro = suspends(5) +assert pyiter_send(coro, None) == (PYGEN_NEXT, None) +assert pyiter_send(coro, 3) == (PYGEN_RETURN, 8) + +# An exception raised in the coroutine body is reported as PYGEN_ERROR. +with assertRaises(MyError): + pyiter_send(fails(), None) + +[typing fixtures/typing-full.pyi] diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index b794cad10fb76..f6fa65fc824a7 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1161,9 +1161,10 @@ def native_yield_from_stop_iteration() -> Generator[int, None, object]: return x [file driver.py] -# Interpreted 'yield from' drives native generators through the C-level -# am_send slot (PyIter_Send), which reports completion without raising -# StopIteration. +# Interpreted 'yield from' resumes a native generator through __next__ or send(), +# i.e. through the Python-level protocol, since the SEND opcode calls tp_iternext or +# the send method directly on Python 3.12 and later. See testAmSendSlotOnNativeGenerator +# for coverage of the C-level send protocol. from native import ( native, exhausted, raises_stop_iteration, native_yield_from_stop_iteration ) @@ -1199,5 +1200,73 @@ assert run(g, []) == ([], None) # StopIteration raised in a generator body is treated as a return value by # 'yield from', both in interpreted and in native code. +# +# This is a known deviation from Python semantics, not the desired behavior: PEP 479 +# requires the StopIteration to be replaced with RuntimeError("generator raised +# StopIteration"), which mypyc doesn't implement. These assertions pin down the current +# behavior so that a future fix shows up as a test failure here. It's pre-existing and +# not specific to 'yield from': raising StopIteration in a native generator body also +# silently ends a 'for' loop over it. assert run(raises_stop_iteration(), []) == ([], "seven") assert run(native_yield_from_stop_iteration(), []) == ([], "seven") +assert list(raises_stop_iteration()) == [] + +[case testAmSendSlotOnNativeGenerator] +from typing import Generator, Iterable + +def gen() -> Generator[int, int, str]: + x = yield 1 + return "ret" + str(x) + +def gen_no_yield() -> Generator[int, None, int]: + if int() == 1: + yield 1 + return 5 + +def gen_raises() -> Generator[int, None, int]: + if int() == 1: + yield 1 + raise ValueError("boom") + +def gen_delegate(it: Iterable[int]) -> Generator[int, None, None]: + yield from it + +[file driver.py] +# Generator classes have an am_send slot implementing CPython's C-level send protocol, +# which reports normal completion as PYGEN_RETURN instead of raising StopIteration. +# asyncio's C Task and compiled 'await'/'yield from' resume generators through it, so +# drive it directly via PyIter_Send here. +from native import gen, gen_no_yield, gen_raises, gen_delegate +from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN + +g = gen() +assert pyiter_send(g, None) == (PYGEN_NEXT, 1) +assert pyiter_send(g, 3) == (PYGEN_RETURN, "ret3") +# An exhausted generator completes with None, like a CPython generator does. +assert pyiter_send(g, None) == (PYGEN_RETURN, None) + +# Completing without ever yielding. +assert pyiter_send(gen_no_yield(), None) == (PYGEN_RETURN, 5) + +# An exception raised in the generator body is reported as PYGEN_ERROR. +with assertRaises(ValueError, "boom"): + pyiter_send(gen_raises(), None) + +# A closed generator completes with None. +g = gen() +assert pyiter_send(g, None) == (PYGEN_NEXT, 1) +g.close() +assert pyiter_send(g, None) == (PYGEN_RETURN, None) + +# Resuming a generator that is currently executing is an error. +box = [] + +def reenter(): + with assertRaises(ValueError, "generator already executing"): + pyiter_send(box[0], None) + yield 2 + +g2 = gen_delegate(reenter()) +box.append(g2) +assert pyiter_send(g2, None) == (PYGEN_NEXT, 2) +assert pyiter_send(g2, None) == (PYGEN_RETURN, None) From 693c1de8b4192fe565deddbe678baf1a3187feaa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 10:28:49 +0100 Subject: [PATCH 3/9] Update comments --- mypyc/test-data/run-async.test | 12 ++++-------- mypyc/test-data/run-generators.test | 13 ++++--------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 95f5059043cb1..97ae4b8ed67cf 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2555,14 +2555,10 @@ except StopIteration as e: else: assert False -# Resuming a completed native coroutine raises a bare StopIteration, which means that -# awaiting it a second time evaluates to None. -# -# This is a known deviation from Python semantics, not the desired behavior: CPython -# raises RuntimeError("cannot reuse already awaited coroutine") instead. This assertion -# pins down the current behavior so that a future fix shows up as a test failure here. -# It's pre-existing and independent of the am_send slot, which faithfully reports what -# the generator helper does. +# This is a mypyc bug: unlike CPython, which rejects reuse with RuntimeError("cannot +# reuse already awaited coroutine"), native coroutines permit reuse after they have +# been awaited. Resuming a completed native coroutine raises a bare StopIteration, so +# awaiting it again evaluates to None. try: coro.send(None) except StopIteration as e: diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index f6fa65fc824a7..e56aab85c54a1 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1198,15 +1198,10 @@ except StopIteration as e: assert e.value == 5 assert run(g, []) == ([], None) -# StopIteration raised in a generator body is treated as a return value by -# 'yield from', both in interpreted and in native code. -# -# This is a known deviation from Python semantics, not the desired behavior: PEP 479 -# requires the StopIteration to be replaced with RuntimeError("generator raised -# StopIteration"), which mypyc doesn't implement. These assertions pin down the current -# behavior so that a future fix shows up as a test failure here. It's pre-existing and -# not specific to 'yield from': raising StopIteration in a native generator body also -# silently ends a 'for' loop over it. +# This is a mypyc bug: native generators treat StopIteration raised in the generator +# body as a return value instead of applying the PEP 479 conversion to +# RuntimeError("generator raised StopIteration"). Thus 'yield from' receives the +# exception's value and a 'for' loop silently ends. assert run(raises_stop_iteration(), []) == ([], "seven") assert run(native_yield_from_stop_iteration(), []) == ([], "seven") assert list(raises_stop_iteration()) == [] From ffe62050009609a71f655fca6b7c8f5f67f3caf0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 10:44:59 +0100 Subject: [PATCH 4/9] Update docstring --- mypyc/test-data/fixtures/testutil.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 9aeb0cc26e21d..8a7bb759be676 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -93,17 +93,16 @@ def run_generator(gen: Generator[T, V, U], def pyiter_send(it: Any, arg: Any) -> Tuple[int, Any]: - """Resume an iterator using PyIter_Send, the C-level send protocol. + """Resume an iterator through CPython's C-level PyIter_Send protocol. - This calls the am_send slot, if the type of 'it' has one, so it's the only way to - exercise am_send from Python. It's the path asyncio's C Task and compiled - 'await'/'yield from' use to resume a generator or coroutine. Note that the - interpreter's own SEND opcode does not use it on Python 3.12 and later. + PyIter_Send calls the iterator type's am_send slot when available, allowing tests to + exercise the slot directly from Python. asyncio's C Task and compiled 'await' and + 'yield from' expressions use this path to resume generators and coroutines, but the + SEND opcode does not on Python 3.12 and later. - Return (result code, value), where the code is PYGEN_NEXT for a yielded value and - PYGEN_RETURN for normal completion -- completion is reported without raising - StopIteration. If the iterator raises, the exception propagates, since ctypes - re-raises whatever exception is pending after the call. + Return (code, value), where code is PYGEN_NEXT for a yielded value and PYGEN_RETURN + for normal completion. Completion does not raise StopIteration. Other exceptions + propagate because ctypes re-raises the exception left pending by PyIter_Send. """ import ctypes From 27187f42e3802fd985eab4750cdbcd724cb6bf6c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 11:02:35 +0100 Subject: [PATCH 5/9] Update comments --- mypyc/test-data/fixtures/testutil.py | 7 ++++--- mypyc/test-data/run-generators.test | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 8a7bb759be676..1dc6d1cdf9100 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -96,9 +96,10 @@ def pyiter_send(it: Any, arg: Any) -> Tuple[int, Any]: """Resume an iterator through CPython's C-level PyIter_Send protocol. PyIter_Send calls the iterator type's am_send slot when available, allowing tests to - exercise the slot directly from Python. asyncio's C Task and compiled 'await' and - 'yield from' expressions use this path to resume generators and coroutines, but the - SEND opcode does not on Python 3.12 and later. + exercise the slot directly from Python. asyncio's C Task and mypyc's generic 'await' + and 'yield from' path use this protocol, while mypyc's native fast path calls the + generator helper directly. The SEND opcode also bypasses the protocol on Python 3.12 + and later. Return (code, value), where code is PYGEN_NEXT for a yielded value and PYGEN_RETURN for normal completion. Completion does not raise StopIteration. Other exceptions diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index e56aab85c54a1..64ad1e1e62eb4 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1229,8 +1229,9 @@ def gen_delegate(it: Iterable[int]) -> Generator[int, None, None]: [file driver.py] # Generator classes have an am_send slot implementing CPython's C-level send protocol, # which reports normal completion as PYGEN_RETURN instead of raising StopIteration. -# asyncio's C Task and compiled 'await'/'yield from' resume generators through it, so -# drive it directly via PyIter_Send here. +# mypyc's generic 'await' and 'yield from' path resumes generators through this slot, +# while the native fast path calls the mypyc generator helper directly. Drive the slot +# directly via PyIter_Send here. from native import gen, gen_no_yield, gen_raises, gen_delegate from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN From 347cae0151147fb3d3608b361a8d7c47b840fea8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 11:08:47 +0100 Subject: [PATCH 6/9] Update comments --- mypyc/codegen/emitclass.py | 4 ++-- mypyc/codegen/emitwrapper.py | 8 ++++---- mypyc/irbuild/statement.py | 9 ++++----- mypyc/test-data/run-async.test | 10 +++++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 648f282c3361e..1a73d5426b851 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -151,8 +151,8 @@ def dunder_attr_slot(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: "__await__": ("am_await", native_slot), "__aiter__": ("am_aiter", native_slot), "__anext__": ("am_anext", native_slot), - # Generator classes get an am_send slot based on the generator helper method, - # so that PyIter_Send can drive them without raising StopIteration. + # Generator and coroutine classes get an am_send slot based on the generator helper + # method, so that PyIter_Send can drive them without raising StopIteration. GENERATOR_HELPER_NAME: ("am_send", generate_am_send_wrapper), } diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index a6858613805e5..c6855e38a724c 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -596,7 +596,7 @@ def generate_len_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: def generate_am_send_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: - """Generate an am_send slot for a generator class based on the generator helper method. + """Generate an am_send slot for a generator or coroutine class. This implements the C-level send protocol (used by PyIter_Send), which reports normal completion without raising StopIteration. @@ -616,9 +616,9 @@ def generate_am_send_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: emitter.emit_line("return PYGEN_NEXT;") emitter.emit_line("}") emitter.emit_line("if (stop_iter_value == NULL) {") - # The helper raised an exception instead of using the out parameter. A - # StopIteration still signals normal completion (e.g. if the generator has - # already completed, or if the body raised StopIteration explicitly). + # The helper raised an exception instead of using the out parameter. StopIteration + # signals normal completion when the generator or coroutine is already exhausted. + # Due to a mypyc bug, an explicit StopIteration raised in the body does so as well. emitter.emit_line("if (PyErr_ExceptionMatches(PyExc_StopIteration)) {") emitter.emit_line("stop_iter_value = CPy_FetchStopIterationValue();") emitter.emit_line("}") diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index bc9fe03d55b9a..7ebfc2b4ca29e 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1340,8 +1340,7 @@ def emit_yield_from_or_await( # This allows two optimizations: # 1) No need to call CPy_GetCoro() or iter() since for native generators # it just returns the generator object (implemented here). - # 2) Instead of calling next(), call generator helper method directly, - # since next() just calls __next__ which calls the helper method. + # 2) Call the generator helper method directly instead of using PyIter_Send. iter_val: Value = val else: get_op = coro_op if is_await else iter_op @@ -1391,9 +1390,9 @@ def native_step(sent: Value) -> Value: # an extra PyObject ** argument to helper where the stop iteration value is stored. _y_init = native_step(builder.none_object()) else: - # Use PyIter_Send (via CPyIter_Send) for the initial advance as well, so that - # a natively compiled iterator can complete through its am_send slot without - # raising StopIteration, even if we don't know the type statically. + # Use PyIter_Send (via CPyIter_Send) so that a natively compiled iterator can + # complete through its am_send slot without raising StopIteration, even if we + # don't know the type statically. ptr = clear_stop_iter_val() _y_init = builder.call_c( send_op, [builder.read(iter_reg, line), builder.none_object(), ptr], line diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 97ae4b8ed67cf..0a63978d24f1a 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2524,11 +2524,11 @@ async def fails() -> int: raise MyError() [file driver.py] -# Interpreted 'await' resumes a native coroutine through send(), i.e. through the -# Python-level protocol, since the SEND opcode calls the send method directly on -# Python 3.12 and later. asyncio's C Task uses PyIter_Send instead, so it goes through -# the C-level am_send slot, which reports completion without raising StopIteration. -# See also testAmSendSlotOnNativeCoroutine. +# Interpreted 'await' resumes a native coroutine through __next__ or send(), i.e. through +# the Python-level protocol, since the SEND opcode calls tp_iternext or the send method +# directly on Python 3.12 and later. asyncio's C Task uses PyIter_Send instead, so it +# goes through the C-level am_send slot, which reports completion without raising +# StopIteration. See also testAmSendSlotOnNativeCoroutine. import asyncio from native import inc, suspend_then_inc, nested, fails, MyError From 4165c2f2cc27d7350388638425047cdcdda9bd34 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 13:06:23 +0100 Subject: [PATCH 7/9] Update comments --- mypyc/test-data/run-async.test | 29 ++++++++++++++++------------- mypyc/test-data/run-generators.test | 29 +++++++++++++++-------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 0a63978d24f1a..082f667622250 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2524,20 +2524,17 @@ async def fails() -> int: raise MyError() [file driver.py] -# Interpreted 'await' resumes a native coroutine through __next__ or send(), i.e. through -# the Python-level protocol, since the SEND opcode calls tp_iternext or the send method -# directly on Python 3.12 and later. asyncio's C Task uses PyIter_Send instead, so it -# goes through the C-level am_send slot, which reports completion without raising -# StopIteration. See also testAmSendSlotOnNativeCoroutine. import asyncio from native import inc, suspend_then_inc, nested, fails, MyError async def main() -> None: + # On Python 3.12 and later, interpreted 'await' uses the SEND opcode, which calls + # tp_iternext or the send method directly. Exercise this Python-level protocol for + # immediate, suspended, nested, and exceptional completion. assert await inc(1) == 2 assert await suspend_then_inc(1) == 2 assert await nested(3) == 8 - assert await asyncio.gather(inc(5), nested(5)) == [6, 12] try: await fails() except MyError: @@ -2545,8 +2542,15 @@ async def main() -> None: else: assert False + # asyncio.gather implicitly schedules each native coroutine as a C Task. Tasks use + # PyIter_Send, which reaches the coroutine through its am_send slot. + # testAmSendSlotOnNativeCoroutine exercises that slot directly. + assert await asyncio.gather(inc(5), nested(5)) == [6, 12] + asyncio.run(main()) +# The Python-level send method reports normal completion by raising StopIteration with +# the coroutine's return value. coro = inc(1) try: coro.send(None) @@ -2590,22 +2594,21 @@ async def fails() -> int: raise MyError() [file driver.py] -# Coroutine classes have an am_send slot implementing CPython's C-level send protocol, -# which reports normal completion as PYGEN_RETURN instead of raising StopIteration. -# asyncio's C Task resumes coroutines through it, so drive it directly via PyIter_Send -# here. from native import inc, suspends, fails, MyError from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN -# Completing without suspending. +# Coroutine classes expose an am_send slot for CPython's C-level send protocol. Drive +# the slot directly through PyIter_Send, as asyncio's C Task does. + +# Immediate completion returns PYGEN_RETURN instead of raising StopIteration. assert pyiter_send(inc(1), None) == (PYGEN_RETURN, 2) -# Suspending once, then completing with a value sent in. +# A suspension returns PYGEN_NEXT; the next send completes with PYGEN_RETURN. coro = suspends(5) assert pyiter_send(coro, None) == (PYGEN_NEXT, None) assert pyiter_send(coro, 3) == (PYGEN_RETURN, 8) -# An exception raised in the coroutine body is reported as PYGEN_ERROR. +# PYGEN_ERROR leaves the body exception pending, so ctypes propagates it. with assertRaises(MyError): pyiter_send(fails(), None) diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 64ad1e1e62eb4..d8fced07ff1a2 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1161,10 +1161,6 @@ def native_yield_from_stop_iteration() -> Generator[int, None, object]: return x [file driver.py] -# Interpreted 'yield from' resumes a native generator through __next__ or send(), -# i.e. through the Python-level protocol, since the SEND opcode calls tp_iternext or -# the send method directly on Python 3.12 and later. See testAmSendSlotOnNativeGenerator -# for coverage of the C-level send protocol. from native import ( native, exhausted, raises_stop_iteration, native_yield_from_stop_iteration ) @@ -1186,6 +1182,10 @@ def run(gen, sends): return yields, e.value assert False, "generator did not stop" +# On Python 3.12 and later, interpreted 'yield from' uses the SEND opcode, which calls +# tp_iternext or the send method directly. Exercise this Python-level protocol for +# yielding, receiving a sent value, and returning. testAmSendSlotOnNativeGenerator +# exercises the C-level send protocol separately. assert run(native(1), [10, None]) == ([1, 11], "done") # An already exhausted native generator completes with None, like a @@ -1227,24 +1227,25 @@ def gen_delegate(it: Iterable[int]) -> Generator[int, None, None]: yield from it [file driver.py] -# Generator classes have an am_send slot implementing CPython's C-level send protocol, -# which reports normal completion as PYGEN_RETURN instead of raising StopIteration. -# mypyc's generic 'await' and 'yield from' path resumes generators through this slot, -# while the native fast path calls the mypyc generator helper directly. Drive the slot -# directly via PyIter_Send here. from native import gen, gen_no_yield, gen_raises, gen_delegate from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN +# Generator classes expose an am_send slot for CPython's C-level send protocol. The +# generic compiled 'await' and 'yield from' path reaches this slot through PyIter_Send; +# the native fast path calls the mypyc generator helper directly. Drive the slot through +# PyIter_Send here. + +# Completion without yielding returns PYGEN_RETURN. +assert pyiter_send(gen_no_yield(), None) == (PYGEN_RETURN, 5) + +# A yield returns PYGEN_NEXT; the next send completes with PYGEN_RETURN. Further sends +# to the exhausted generator complete with None, like a CPython generator. g = gen() assert pyiter_send(g, None) == (PYGEN_NEXT, 1) assert pyiter_send(g, 3) == (PYGEN_RETURN, "ret3") -# An exhausted generator completes with None, like a CPython generator does. assert pyiter_send(g, None) == (PYGEN_RETURN, None) -# Completing without ever yielding. -assert pyiter_send(gen_no_yield(), None) == (PYGEN_RETURN, 5) - -# An exception raised in the generator body is reported as PYGEN_ERROR. +# PYGEN_ERROR leaves the body exception pending, so ctypes propagates it. with assertRaises(ValueError, "boom"): pyiter_send(gen_raises(), None) From b0a7afe7a1dd7e656b1abd0a2446142902348506 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 13:13:16 +0100 Subject: [PATCH 8/9] Update comments --- mypyc/test-data/fixtures/testutil.py | 11 ++++++----- mypyc/test-data/run-async.test | 13 +++++-------- mypyc/test-data/run-generators.test | 13 +++++-------- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index 1dc6d1cdf9100..587625145b019 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -95,11 +95,12 @@ def run_generator(gen: Generator[T, V, U], def pyiter_send(it: Any, arg: Any) -> Tuple[int, Any]: """Resume an iterator through CPython's C-level PyIter_Send protocol. - PyIter_Send calls the iterator type's am_send slot when available, allowing tests to - exercise the slot directly from Python. asyncio's C Task and mypyc's generic 'await' - and 'yield from' path use this protocol, while mypyc's native fast path calls the - generator helper directly. The SEND opcode also bypasses the protocol on Python 3.12 - and later. + PyIter_Send calls the iterator type's am_send slot when available. asyncio's C Task, + mypyc's generic 'await' and 'yield from' paths, and Python 3.11's SEND opcode use this + protocol. On Python 3.12 and later, SEND bypasses PyIter_Send: it has an inline fast + path for exact CPython generators and coroutines, and otherwise calls tp_iternext + when sending None and the send method for other values. mypyc's native fast path + calls the generator helper directly. Return (code, value), where code is PYGEN_NEXT for a yielded value and PYGEN_RETURN for normal completion. Completion does not raise StopIteration. Other exceptions diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 082f667622250..38a25581878ef 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2529,9 +2529,8 @@ import asyncio from native import inc, suspend_then_inc, nested, fails, MyError async def main() -> None: - # On Python 3.12 and later, interpreted 'await' uses the SEND opcode, which calls - # tp_iternext or the send method directly. Exercise this Python-level protocol for - # immediate, suspended, nested, and exceptional completion. + # Exercise native coroutines through interpreted 'await', including immediate, + # suspended, nested, and exceptional completion. assert await inc(1) == 2 assert await suspend_then_inc(1) == 2 assert await nested(3) == 8 @@ -2542,9 +2541,8 @@ async def main() -> None: else: assert False - # asyncio.gather implicitly schedules each native coroutine as a C Task. Tasks use - # PyIter_Send, which reaches the coroutine through its am_send slot. - # testAmSendSlotOnNativeCoroutine exercises that slot directly. + # asyncio.gather schedules each native coroutine as a C Task, which drives it through + # PyIter_Send and its am_send slot. The next case exercises the slot directly. assert await asyncio.gather(inc(5), nested(5)) == [6, 12] asyncio.run(main()) @@ -2597,8 +2595,7 @@ async def fails() -> int: from native import inc, suspends, fails, MyError from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN -# Coroutine classes expose an am_send slot for CPython's C-level send protocol. Drive -# the slot directly through PyIter_Send, as asyncio's C Task does. +# Drive a native coroutine's am_send slot through PyIter_Send, as asyncio's C Task does. # Immediate completion returns PYGEN_RETURN instead of raising StopIteration. assert pyiter_send(inc(1), None) == (PYGEN_RETURN, 2) diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index d8fced07ff1a2..0e88e286ca4c5 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1182,10 +1182,8 @@ def run(gen, sends): return yields, e.value assert False, "generator did not stop" -# On Python 3.12 and later, interpreted 'yield from' uses the SEND opcode, which calls -# tp_iternext or the send method directly. Exercise this Python-level protocol for -# yielding, receiving a sent value, and returning. testAmSendSlotOnNativeGenerator -# exercises the C-level send protocol separately. +# Exercise a native generator through interpreted 'yield from', including yielding, +# receiving a sent value, and returning. assert run(native(1), [10, None]) == ([1, 11], "done") # An already exhausted native generator completes with None, like a @@ -1230,10 +1228,9 @@ def gen_delegate(it: Iterable[int]) -> Generator[int, None, None]: from native import gen, gen_no_yield, gen_raises, gen_delegate from testutil import assertRaises, pyiter_send, PYGEN_NEXT, PYGEN_RETURN -# Generator classes expose an am_send slot for CPython's C-level send protocol. The -# generic compiled 'await' and 'yield from' path reaches this slot through PyIter_Send; -# the native fast path calls the mypyc generator helper directly. Drive the slot through -# PyIter_Send here. +# Drive a native generator's am_send slot through PyIter_Send. The generic compiled +# 'await' and 'yield from' path does the same; the native fast path calls the generator +# helper directly. # Completion without yielding returns PYGEN_RETURN. assert pyiter_send(gen_no_yield(), None) == (PYGEN_RETURN, 5) From 8abd45b37120dc44b3a506c21020a3ae139e7ddc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 8 Sep 2026 15:06:08 +0100 Subject: [PATCH 9/9] Update comments --- mypyc/codegen/emitwrapper.py | 3 ++- mypyc/test-data/run-async.test | 8 ++++---- mypyc/test-data/run-generators.test | 8 ++++---- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mypyc/codegen/emitwrapper.py b/mypyc/codegen/emitwrapper.py index c6855e38a724c..3098a3a20367f 100644 --- a/mypyc/codegen/emitwrapper.py +++ b/mypyc/codegen/emitwrapper.py @@ -618,7 +618,8 @@ def generate_am_send_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str: emitter.emit_line("if (stop_iter_value == NULL) {") # The helper raised an exception instead of using the out parameter. StopIteration # signals normal completion when the generator or coroutine is already exhausted. - # Due to a mypyc bug, an explicit StopIteration raised in the body does so as well. + # TODO: Convert an explicit StopIteration raised in the body to RuntimeError instead + # of treating it as normal completion. emitter.emit_line("if (PyErr_ExceptionMatches(PyExc_StopIteration)) {") emitter.emit_line("stop_iter_value = CPy_FetchStopIterationValue();") emitter.emit_line("}") diff --git a/mypyc/test-data/run-async.test b/mypyc/test-data/run-async.test index 38a25581878ef..971902494d9d5 100644 --- a/mypyc/test-data/run-async.test +++ b/mypyc/test-data/run-async.test @@ -2557,10 +2557,10 @@ except StopIteration as e: else: assert False -# This is a mypyc bug: unlike CPython, which rejects reuse with RuntimeError("cannot -# reuse already awaited coroutine"), native coroutines permit reuse after they have -# been awaited. Resuming a completed native coroutine raises a bare StopIteration, so -# awaiting it again evaluates to None. +# TODO: Reject reuse with RuntimeError("cannot reuse already awaited coroutine") like +# CPython. Native coroutines currently permit reuse after they have been awaited. +# Resuming a completed native coroutine raises a bare StopIteration, so awaiting it +# again evaluates to None. try: coro.send(None) except StopIteration as e: diff --git a/mypyc/test-data/run-generators.test b/mypyc/test-data/run-generators.test index 0e88e286ca4c5..67c569bf0b23c 100644 --- a/mypyc/test-data/run-generators.test +++ b/mypyc/test-data/run-generators.test @@ -1196,10 +1196,10 @@ except StopIteration as e: assert e.value == 5 assert run(g, []) == ([], None) -# This is a mypyc bug: native generators treat StopIteration raised in the generator -# body as a return value instead of applying the PEP 479 conversion to -# RuntimeError("generator raised StopIteration"). Thus 'yield from' receives the -# exception's value and a 'for' loop silently ends. +# TODO: Apply the PEP 479 conversion to RuntimeError("generator raised StopIteration"). +# Native generators currently treat StopIteration raised in the generator body as a +# return value, so 'yield from' receives the exception's value and a 'for' loop +# silently ends. assert run(raises_stop_iteration(), []) == ([], "seven") assert run(native_yield_from_stop_iteration(), []) == ([], "seven") assert list(raises_stop_iteration()) == []