From 7990de7495e616813e0221a871a2bd6bc364755b Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 17:45:41 -0700 Subject: [PATCH 1/7] [mypyc] add support for contextlib.contextmanager --- mypyc/irbuild/statement.py | 110 ++++- mypyc/lib-rt/CPy.h | 5 + mypyc/lib-rt/exc_ops.c | 164 +++++++ mypyc/primitives/exc_ops.py | 52 ++- mypyc/test-data/run-functions.test | 667 +++++++++++++++++++++++++++++ 5 files changed, 994 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index d60de0d90e1c..659d85467bd9 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -49,6 +49,7 @@ YieldExpr, YieldFromExpr, ) +from mypy.types import Instance, get_proper_type from mypyc.common import ( GENERATOR_HELPER_NAME, KEEP_ALIVE_SHORT_LIVED, @@ -114,6 +115,10 @@ AssignmentTargetTuple, ) from mypyc.primitives.exc_ops import ( + contextmanager_bind_op, + contextmanager_enter_op, + contextmanager_exit_exception_op, + contextmanager_exit_normal_op, error_catch_op, exc_matches_op, get_exc_info_op, @@ -1125,6 +1130,40 @@ def transform_with( is_async: bool, line: int, ) -> None: + + node: mypy.nodes.SymbolNode | None = None + receiver_expr: Expression | None = None + if isinstance(expr, mypy.nodes.CallExpr) and isinstance(expr.callee, mypy.nodes.RefExpr): + callee = expr.callee + node = callee.node + if isinstance(callee, mypy.nodes.MemberExpr) and not builder.is_module_member_expr(callee): + node = None + # Keep the experimental method path to ordinary, non-suspending calls. + if ( + all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) + and not find_walrus_targets(expr) + and not expr_has_suspend(expr) + ): + receiver_type = get_proper_type(builder.types[callee.expr]) + if isinstance(receiver_type, Instance): + symbol = receiver_type.type.get(callee.name) + if symbol is not None: + node = symbol.node + receiver_expr = callee.expr + + if ( + not is_async + and isinstance(expr, mypy.nodes.CallExpr) + and isinstance(expr.callee, mypy.nodes.RefExpr) + and isinstance(dec := node, mypy.nodes.Decorator) + and not (receiver_expr is not None and (dec.func.is_static or dec.func.is_class)) + and len(dec.decorators) == 1 + and isinstance(dec1 := dec.decorators[0], mypy.nodes.RefExpr) + and dec1.node + and dec1.node.fullname == "contextlib.contextmanager" + ): + return _transform_with_contextmanager(builder, expr, target, body, line, receiver_expr) + # This is basically a straight transcription of the Python code in PEP 343. # I don't actually understand why a bunch of it is the way it is. # We could probably optimize the case where the manager is compiled by us, @@ -1136,7 +1175,7 @@ def transform_with( if not is_async and mgr_v.type == lock_rprimitive: transform_with_lock(builder, mgr_v, target, body, line) - return + return None # The methods may be inherited from a non-native class, and then they can't be # called natively. is_native = ( @@ -1240,6 +1279,75 @@ def finally_body() -> None: transform_try_finally_stmt(builder, try_body, finally_body, line) +def _transform_with_contextmanager( + builder: IRBuilder, + expr: mypy.nodes.CallExpr, + target: Lvalue | None, + with_body: GenFunc, + line: int, + receiver_expr: Expression | None, +) -> None: + assert isinstance(expr.callee, mypy.nodes.RefExpr) + if receiver_expr is None: + wrapped_call = mypy.nodes.CallExpr( + mypy.nodes.MemberExpr(expr.callee, "__wrapped__"), + expr.args, + expr.arg_kinds, + expr.arg_names, + ) + wrapped_call.line = line + gen = builder.accept(wrapped_call) + else: + assert isinstance(expr.callee, mypy.nodes.MemberExpr) + receiver = builder.accept(receiver_expr) + method = builder.py_get_attr(receiver, expr.callee.name, line) + wrapped = builder.py_get_attr(method, "__wrapped__", line) + wrapped = builder.call_c(contextmanager_bind_op, [method, wrapped], line) + args = [builder.accept(arg) for arg in expr.args] + gen = builder.py_call(wrapped, args, line, expr.arg_kinds, expr.arg_names) + + # Require exactly one yield: enter here, finish after a successful body. + mgr_target = builder.call_c(contextmanager_enter_op, [gen], line) + + exc = builder.ensure_register(builder.true()) + + # try: + # {body} + + def try_body() -> None: + if target: + builder.assign(builder.get_assignment_target(target), mgr_target, line) + with_body() + + def except_body() -> None: + builder.assign(exc, builder.false(), line) + suppressed = builder.call_c( + contextmanager_exit_exception_op, [builder.read(gen), *get_sys_exc_info(builder)], line + ) + out_block, reraise_block = BasicBlock(), BasicBlock() + builder.add_bool_branch(suppressed, out_block, reraise_block) + builder.activate_block(reraise_block) + builder.call_c(reraise_exception_op, [], NO_TRACEBACK_LINE_NO) + builder.add(Unreachable()) + builder.activate_block(out_block) + + handlers = [(None, None, except_body)] + + def finally_body() -> None: + out_block, exit_block = BasicBlock(), BasicBlock() + builder.add(Branch(builder.read(exc), exit_block, out_block, Branch.BOOL)) + builder.activate_block(exit_block) + builder.call_c(contextmanager_exit_normal_op, [builder.read(gen)], line) + builder.goto_and_activate(out_block) + + transform_try_finally_stmt( + builder, + lambda: transform_try_except(builder, try_body, handlers, None, line), + finally_body, + line, + ) + + def transform_with_stmt(builder: IRBuilder, o: WithStmt) -> None: # Generate separate logic for each expr in it, left to right def generate(i: int) -> None: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index dfb7e43ed769..94cc0960050e 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -1071,6 +1071,11 @@ bool CPy_ExceptionMatches(PyObject *type); PyObject *CPy_GetExcValue(void); tuple_T3OOO CPy_GetExcInfo(void); void _CPy_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback); +PyObject *CPyContextManager_Bind(PyObject *callable, PyObject *wrapped); +PyObject *CPyContextManager_Enter(PyObject *gen); +bool CPyContextManager_ExitNormal(PyObject *gen); +int CPyContextManager_ExitException(PyObject *gen, PyObject *type, + PyObject *value, PyObject *traceback); void CPyError_OutOfMemory(void); void CPy_TypeError(const char *expected, PyObject *value); void CPy_AddTraceback(const char *filename, const char *funcname, int line, PyObject *globals); diff --git a/mypyc/lib-rt/exc_ops.c b/mypyc/lib-rt/exc_ops.c index f6a79fa8197a..083696dc7f48 100644 --- a/mypyc/lib-rt/exc_ops.c +++ b/mypyc/lib-rt/exc_ops.c @@ -163,6 +163,170 @@ tuple_T3OOO CPy_GetExcInfo(void) { return ret; } +// Give a contextmanager's generator factory the same binding as its wrapper. +// The returned callable is owned; static methods remain unbound. +PyObject *CPyContextManager_Bind(PyObject *callable, PyObject *wrapped) { + if (PyMethod_Check(callable)) { + // Use the actual bound instance or class, not the attribute lookup's receiver. + return PyMethod_New(wrapped, PyMethod_GET_SELF(callable)); + } + Py_INCREF(wrapped); + return wrapped; +} + +// Implement contextlib's generator entry protocol. The returned value is owned. +PyObject *CPyContextManager_Enter(PyObject *gen) { + if (!PyIter_Check(gen)) { + PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", Py_TYPE(gen)->tp_name); + return NULL; + } + PyObject *value = Py_TYPE(gen)->tp_iternext(gen); + if (value != NULL) { + return value; + } + if (!PyErr_Occurred()) { + // Iterators may signal exhaustion without setting StopIteration. + PyErr_SetNone(PyExc_StopIteration); + } + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + // raise RuntimeError(...) from None, retaining the suppressed context. + tuple_T3OOO old_exc = CPy_CatchError(); + PyObject *error = PyObject_CallFunction(PyExc_RuntimeError, "s", "generator didn't yield"); + if (error != NULL) { + CPy_RaiseFrom(error, Py_None); + Py_DECREF(error); + } + CPy_RestoreExcInfo(old_exc); + Py_DECREF(old_exc.f0); + Py_DECREF(old_exc.f1); + Py_DECREF(old_exc.f2); + } + return NULL; +} + +// An extra yield is a protocol error. Always leave an exception pending. +static void contextmanager_exit_error(PyObject *gen, const char *message) { + PyErr_SetString(PyExc_RuntimeError, message); +#if PY_VERSION_HEX >= 0x030B0000 + // Since Python 3.11, contextlib closes the generator in a finally block. + // Make the protocol error the handled exception so cleanup sees it and + // any cleanup failure chains to it in the same way. + tuple_T3OOO old_exc = CPy_CatchError(); + PyObject *type, *value, *traceback; + PyErr_GetExcInfo(&type, &value, &traceback); + PyObject *closed = PyObject_CallMethod(gen, "close", NULL); + if (closed != NULL) { + Py_DECREF(closed); + // A native generator can change the handled exception during close(). + // Reraise the saved protocol error, as a Python finally block does. + PyErr_Restore(type, value, traceback); + } else { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + } + CPy_RestoreExcInfo(old_exc); + Py_DECREF(old_exc.f0); + Py_DECREF(old_exc.f1); + Py_DECREF(old_exc.f2); +#endif +} + +// Finish a contextmanager whose body succeeded. Return false on error. +bool CPyContextManager_ExitNormal(PyObject *gen) { + if (!PyIter_Check(gen)) { + PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", Py_TYPE(gen)->tp_name); + return false; + } + PyObject *value = Py_TYPE(gen)->tp_iternext(gen); + if (value == NULL) { + if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + return true; + } + return false; + } + Py_DECREF(value); + contextmanager_exit_error(gen, "generator didn't stop"); + return false; +} + +// Match contextlib._GeneratorContextManager.__exit__ for a failed with body. +// The borrowed exception arguments are already normalized by CPy_CatchError. +// Return 1 to suppress, 0 to propagate the original, or -1 with a new error pending. +// Preserve the caller's handled exception on every path. +int CPyContextManager_ExitException(PyObject *gen, PyObject *type, + PyObject *value, PyObject *traceback) { + // Native generators can change the handled exception when throw() unwinds + // an except block. Preserve the caller's state before interpreting the result. + PyObject *saved_type, *saved_value, *saved_traceback; + PyErr_GetExcInfo(&saved_type, &saved_value, &saved_traceback); +#if PY_VERSION_HEX >= 0x030C0000 + PyObject *yielded = PyObject_CallMethod(gen, "throw", "(O)", value); +#else + PyObject *yielded = PyObject_CallMethod(gen, "throw", "(OOO)", type, value, traceback); +#endif + PyErr_SetExcInfo(saved_type, saved_value, saved_traceback); + if (yielded != NULL) { + Py_DECREF(yielded); + contextmanager_exit_error(gen, "generator didn't stop after throw()"); + return -1; + } + + tuple_T3OOO old_exc = CPy_CatchError(); + PyObject *error = CPy_GetExcValue(); + int result = -1; + + if (CPy_ExceptionMatches(PyExc_StopIteration)) { + // A body StopIteration must not be mistaken for generator completion. + result = error != value; + goto done; + } + if (CPy_ExceptionMatches(PyExc_RuntimeError)) { + if (error == value) { + goto original; + } + // PEP 479 can wrap the original StopIteration (including subclasses). + int is_stop = PyObject_IsInstance(value, PyExc_StopIteration); + if (is_stop < 0) { + goto done; + } + if (is_stop) { + PyObject *cause = PyObject_GetAttrString(error, "__cause__"); + if (cause == NULL) { + goto done; + } + bool is_original = cause == value; + Py_DECREF(cause); + if (is_original) { + goto original; + } + } + } else if (error == value) { + goto original; + } + CPy_Reraise(); + goto done; + +original: +#if PY_VERSION_HEX >= 0x030B0000 + // contextlib removes throw() frames on these paths, but not on the + // same-StopIteration path above. Keep Python attribute semantics here. + if (PyObject_SetAttrString(value, "__traceback__", traceback) < 0) { + goto done; + } +#endif + result = 0; + +done: + Py_DECREF(error); + CPy_RestoreExcInfo(old_exc); + Py_DECREF(old_exc.f0); + Py_DECREF(old_exc.f1); + Py_DECREF(old_exc.f2); + return result; +} + void CPyError_OutOfMemory(void) { fprintf(stderr, "fatal: out of memory\n"); fflush(stderr); diff --git a/mypyc/primitives/exc_ops.py b/mypyc/primitives/exc_ops.py index 528a89ef2331..39b9c382345b 100644 --- a/mypyc/primitives/exc_ops.py +++ b/mypyc/primitives/exc_ops.py @@ -2,9 +2,50 @@ from __future__ import annotations -from mypyc.ir.ops import ERR_ALWAYS, ERR_FALSE, ERR_NEVER -from mypyc.ir.rtypes import bit_rprimitive, exc_rtuple, object_rprimitive, void_rtype -from mypyc.primitives.registry import custom_op, custom_primitive_op +from mypyc.ir.ops import ERR_ALWAYS, ERR_FALSE, ERR_MAGIC, ERR_NEVER +from mypyc.ir.rtypes import ( + bit_rprimitive, + c_int_rprimitive, + exc_rtuple, + object_rprimitive, + void_rtype, +) +from mypyc.primitives.registry import ERR_NEG_INT, custom_op, custom_primitive_op + +# Give a contextmanager's generator factory the same runtime method binding. +contextmanager_bind_op = custom_op( + arg_types=[object_rprimitive, object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyContextManager_Bind", + error_kind=ERR_MAGIC, +) + +# Enter a @contextmanager generator, requiring it to yield a value. +contextmanager_enter_op = custom_op( + arg_types=[object_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyContextManager_Enter", + error_kind=ERR_MAGIC, +) + +# Finish a @contextmanager generator after a successful with body. +# Returns true on success; generator/protocol errors propagate normally. +contextmanager_exit_normal_op = custom_op( + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyContextManager_ExitNormal", + error_kind=ERR_FALSE, +) + +# Finish a contextmanager after a body exception. True suppresses the exception; +# false leaves the original handled exception for the caller to reraise. +contextmanager_exit_exception_op = custom_op( + arg_types=[object_rprimitive] * 4, + return_type=c_int_rprimitive, + c_function_name="CPyContextManager_ExitException", + error_kind=ERR_NEG_INT, + truncated_type=bit_rprimitive, +) # If the argument is a class, raise an instance of the class. Otherwise, assume # that the argument is an exception object, and raise it. @@ -88,6 +129,11 @@ arg_types=[], return_type=exc_rtuple, c_function_name="CPy_CatchError", error_kind=ERR_NEVER ) +# Clear the current exception. +error_clear_op = custom_op( + arg_types=[], return_type=void_rtype, c_function_name="PyErr_Clear", error_kind=ERR_NEVER +) + # Restore an old "currently handled exception" returned from. # error_catch (by sticking it into sys.exc_info()) restore_exc_info_op = custom_op( diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index bb52eeb6d192..6c118ea65f88 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -1309,6 +1309,673 @@ def test_special_case() -> None: with f(): a.pop() +[case testContextManagerSpecialCaseSemantics] +from contextlib import contextmanager +from typing import Any, Generator, Iterator, cast +import traceback + +@contextmanager +def cm_no_yield() -> Iterator[None]: + if False: + yield + return + +@contextmanager +def cm_no_stop() -> Iterator[str]: + print("enter cm_no_stop") + yield "first" + print("after first") + yield "second" + +@contextmanager +def cm_value_error_suppress() -> Iterator[None]: + try: + yield + except ValueError as e: + print("suppress", str(e)) + +@contextmanager +def cm_throw_twice() -> Iterator[None]: + try: + yield + except ValueError: + print("throw yield") + yield + +@contextmanager +def cm_stop_passthrough() -> Iterator[None]: + yield + +@contextmanager +def cm_stop_suppress() -> Iterator[None]: + try: + yield + except StopIteration as e: + print("suppress stop", str(e)) + +@contextmanager +def cm_value() -> Iterator[int]: + print("enter cm_value") + try: + yield 1 + finally: + print("exit cm_value") + +@contextmanager +def cm_gen() -> Iterator[int]: + print("cm enter") + try: + yield 1 + finally: + print("cm exit") + +@contextmanager +def cm_log(tag: str) -> Iterator[None]: + print(tag, "enter") + try: + yield + finally: + print(tag, "exit") + +def test_no_yield() -> None: + try: + with cm_no_yield(): + print("body no yield") + except RuntimeError as e: + e_any = cast(Any, e) + print( + "no_yield", + str(e), + e_any.__cause__ is None, + e_any.__suppress_context__, + ) + +def test_no_stop() -> None: + try: + with cm_no_stop() as v: + print("body", v) + except RuntimeError as e: + print("no_stop", str(e)) + +def test_suppress_value_error() -> None: + with cm_value_error_suppress(): + raise ValueError("boom") + print("after suppress value") + +def test_throw_twice() -> None: + try: + with cm_throw_twice(): + raise ValueError("oops") + except RuntimeError as e: + print("throw_twice", str(e)) + +def test_stop_iteration_passthrough() -> None: + try: + with cm_stop_passthrough(): + raise StopIteration("stop") + except Exception as e: + print("stop passthrough", type(e).__name__, str(e)) + +def test_stop_iteration_suppress() -> None: + with cm_stop_suppress(): + raise StopIteration("stop2") + print("after suppress stop") + +def test_traceback() -> None: + try: + with cm_value(): + raise ValueError("trace") + except Exception as e: + e_any = cast(Any, e) + tb = e_any.__traceback__ + assert tb is not None + tb_mod = cast(Any, traceback) + names = [frame.name for frame in tb_mod.extract_tb(tb)] + print("traceback_has_cm_value", "cm_value" in names) + +def test_generator_with_yield() -> None: + def gen() -> Generator[int, None, None]: + with cm_gen() as v: + yield v + print("after yield in with") + + g = gen() + print(next(g)) + try: + next(g) + except StopIteration: + print("gen done") + +def test_return_in_with() -> None: + def inner() -> int: + with cm_log("ret"): + print("ret body") + return 5 + return 0 + + print("ret result", inner()) + +[file driver.py] +from native import ( + test_no_yield, + test_no_stop, + test_suppress_value_error, + test_throw_twice, + test_stop_iteration_passthrough, + test_stop_iteration_suppress, + test_traceback, + test_generator_with_yield, + test_return_in_with, +) + +test_no_yield() +test_no_stop() +test_suppress_value_error() +test_throw_twice() +test_stop_iteration_passthrough() +test_stop_iteration_suppress() +test_traceback() +test_generator_with_yield() +test_return_in_with() +[out] +no_yield generator didn't yield True True +enter cm_no_stop +body first +after first +no_stop generator didn't stop +suppress boom +after suppress value +throw yield +throw_twice generator didn't stop after throw() +stop passthrough StopIteration stop +suppress stop stop2 +after suppress stop +enter cm_value +exit cm_value +traceback_has_cm_value False +cm enter +1 +after yield in with +cm exit +gen done +ret enter +ret body +ret exit +ret result 5 + +[case testContextManagerRuntimePrimitives] +from contextlib import contextmanager +from typing import Any, Iterator, cast +import helper + +# C iterators can exhaust without setting StopIteration. +@contextmanager +def implicit(values: list[int]) -> Iterator[int]: + return iter(values) + +@contextmanager +def noniterator() -> Iterator[None]: + return cast(Any, 42) + +def test_entry_protocol() -> None: + try: + with implicit([]): + print('empty body') + except RuntimeError as error: + print(str(error), type(error.__context__).__name__) + try: + with noniterator(): + print('noniterator body') + except TypeError as error: + print(str(error)) + +# Other exceptions from next() must propagate unchanged at either boundary. +@contextmanager +def fail_entry(error: ValueError) -> Iterator[None]: + raise error + yield + +@contextmanager +def fail_exit(error: ValueError) -> Iterator[None]: + yield + raise error + +def test_failure_identity() -> None: + original = ValueError('original') + try: + with fail_entry(original): + print('entry body') + except ValueError as error: + assert error is original + print('entry error') + try: + with fail_exit(original): + print('exit body') + except ValueError as error: + assert error is original + print('exit error') + +# Protocol errors temporarily replace the handled exception. They must restore +# it and release the saved references, even after repeatedly closing generators. +@contextmanager +def missing() -> Iterator[None]: + if False: + yield + +@contextmanager +def twice(closed: list[str]) -> Iterator[int]: + try: + yield 1 + yield 2 + finally: + closed.append('closed') + +def failed_entry() -> None: + try: + with missing(): + assert False + except RuntimeError as error: + assert isinstance(error.__context__, StopIteration) + +def failed_exit() -> None: + closed: list[str] = [] + try: + with twice(closed): + pass + except RuntimeError: + if helper.closes_generator: + assert closed == ['closed'] + else: + assert False + +def test_outer_exception() -> None: + outer = ValueError('outer') + try: + raise outer + except ValueError: + # Also exercise normal exit when a C iterator exhausts without an error. + with implicit([7]) as value: + assert value == 7 + assert helper.exc_info()[1] is outer + assert helper.exc_info()[1] is outer + # Collect exception/traceback cycles before measuring retained references. + helper.collect() + before = helper.refcount(outer) + for i in range(20): + failed_entry() + helper.collect() + assert helper.refcount(outer) == before + assert helper.exc_info()[1] is outer + for i in range(20): + failed_exit() + helper.collect() + assert helper.refcount(outer) == before + assert helper.exc_info()[1] is outer + assert helper.exc_info()[1] is None + print('outer exception restored; saved references released') + +# A failure during close supersedes the protocol error but retains its context. +@contextmanager +def bad_close() -> Iterator[int]: + # Use a Python generator: native generator.close has different context chaining. + return helper.bad_close() + +def test_cleanup_error() -> None: + try: + with bad_close() as value: + print('body', value) + except ValueError as error: + current: BaseException | None = error + while current is not None: + print(f'{type(current).__name__}({current})') + current = current.__context__ + +[file helper.py] +from typing import Any, Iterator +import gc +import sys + +closes_generator: bool = sys.version_info >= (3, 11) # type: ignore + +def exc_info() -> Any: + return sys.exc_info() # type: ignore + +def collect() -> None: + gc.collect() + +def refcount(value: Any) -> int: + return sys.getrefcount(value) # type: ignore + +def bad_close() -> Iterator[int]: + try: + yield 1 + yield 2 + finally: + print('closing') + raise ValueError('close failed') + +[file driver.py] +import helper +from native import ( + test_entry_protocol, test_failure_identity, test_outer_exception, test_cleanup_error, +) + +test_entry_protocol() +test_failure_identity() +test_outer_exception() +# contextlib closes a generator after an extra yield starting in Python 3.11. +if helper.closes_generator: + test_cleanup_error() +[out] +generator didn't yield StopIteration +'int' object is not an iterator +entry error +exit body +exit error +outer exception restored; saved references released +[out version>=3.11] +generator didn't yield StopIteration +'int' object is not an iterator +entry error +exit body +exit error +outer exception restored; saved references released +body 1 +closing +ValueError(close failed) +GeneratorExit() +RuntimeError(generator didn't stop) + + +[case testContextManagerExceptionalExit] +from contextlib import contextmanager +from typing import Any, Iterator +import helper + +@contextmanager +def scope(generator: Iterator[None]) -> Iterator[None]: + return generator + +def invoke(generator: Iterator[None], error: BaseException, + captured: list[Any], reference: bool = False) -> None: + if reference: + # An ordinary call keeps contextlib as an oracle on the same generator type. + manager = scope(generator) + with manager: + helper.raise_body(error, captured) + else: + with scope(generator): + helper.raise_body(error, captured) + +# Native generators can propagate StopIteration directly instead of PEP 479 wrapping. +def native_passthrough() -> Iterator[None]: + yield + +# Unwinding this handler during throw() can change the native caller's exception state. +def native_handler() -> Iterator[None]: + try: + raise TypeError('earlier') + except BaseException: + yield + +[file helper.py] +from typing import Any, Iterator, cast +import sys + +closes_generator: bool = sys.version_info >= (3, 11) # type: ignore + +class Other(BaseException): + pass + +class StopSubclass(StopIteration): + pass + +def raise_body(error: BaseException, captured: list[Any]) -> None: + try: + raise error + except BaseException: + captured.append(cast(Any, error).__traceback__) + raise + +def passthrough() -> Iterator[None]: + yield + +def replace(error: BaseException) -> Iterator[None]: + try: + yield + except BaseException: + raise error + +def extra_yield(close_error: bool) -> Iterator[None]: + try: + try: + yield + except ValueError: + yield + finally: + if close_error: + raise ValueError('close failed') + +def suppress() -> Iterator[None]: + try: + yield + except ValueError: + pass + +[file driver.py] +import gc +import sys +import helper +from native import invoke, native_passthrough, native_handler, scope + + +def observe(generator, original, reference=False): + captured = [] + try: + invoke(generator, original, captured, reference) + except BaseException as caught: + names = [] + has_original_tail = False + tb = caught.__traceback__ + while tb is not None: + has_original_tail |= tb is captured[0] + # The C primitive has no Python frame corresponding to __exit__. + if tb.tb_frame.f_code.co_name != '__exit__': + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + chain = [] + error = caught + while error is not None: + chain.append(f'{type(error).__name__}({error})') + error = error.__context__ + closed = getattr(generator, 'gi_frame', None) is None + return caught is original, names, has_original_tail, chain, closed + raise AssertionError('unexpected suppression') + + +def compare(factory, error_type=ValueError): + direct = observe(factory(), error_type('body')) + reference = observe(factory(), error_type('body'), reference=True) + assert direct == reference, (direct, reference) + return direct + + +# These take different branches in contextlib; all must propagate the same object. +for error_type in [RuntimeError, helper.Other, helper.StopSubclass]: + result = compare(helper.passthrough, error_type) + assert result[0] and result[2] + print(error_type.__name__, 'identity and traceback match contextlib') +result = compare(native_passthrough, StopIteration) +assert result[0] +print('native StopIteration identity and traceback match contextlib') + +# Use an interpreted caller as the oracle: ordinary compiled with also relies on +# native handled-exception state. The interpreted generator isolates that state. +def propagate_from_handler(reference): + original = ValueError('body') + try: + if reference: + with scope(native_handler()): + raise original + else: + invoke(native_handler(), original, []) + except BaseException as caught: + yield caught is original + else: + yield False + + +def check_handler(): + for reference in [True, False]: + assert list(propagate_from_handler(reference)) == [True] + + +check_handler() +assert sys.exc_info()[1] is None +try: + raise KeyError('outer') +except KeyError as outer: + check_handler() + assert sys.exc_info()[1] is outer +assert sys.exc_info()[1] is None +print('native handler preserves body exception with and without an outer handler') + +# A different exception from throw() must replace the body's exception. +for replacement in [RuntimeError, helper.Other]: + result = compare(lambda: helper.replace(replacement('replacement'))) + assert not result[0] + print(*result[3], sep='\n') + +# A second yield requires close(), including its effects and any cleanup failure. +if helper.closes_generator: + for close_error in [False, True]: + result = compare(lambda: helper.extra_yield(close_error)) + print('closed', result[4]) + print(*result[3], sep='\n') + +# Suppression must restore the outer handled exception without retaining its refs. +outer = ValueError('outer') +try: + raise outer +except ValueError: + gc.collect() + before = sys.getrefcount(outer) + for i in range(20): + invoke(helper.suppress(), ValueError('body'), []) + assert sys.exc_info()[1] is outer + gc.collect() + assert sys.getrefcount(outer) == before + print('suppression restores outer exception and releases saved references') +assert sys.exc_info()[1] is None +[out] +RuntimeError identity and traceback match contextlib +Other identity and traceback match contextlib +StopSubclass identity and traceback match contextlib +native StopIteration identity and traceback match contextlib +native handler preserves body exception with and without an outer handler +RuntimeError(replacement) +ValueError(body) +Other(replacement) +ValueError(body) +suppression restores outer exception and releases saved references +[out version>=3.11] +RuntimeError identity and traceback match contextlib +Other identity and traceback match contextlib +StopSubclass identity and traceback match contextlib +native StopIteration identity and traceback match contextlib +native handler preserves body exception with and without an outer handler +RuntimeError(replacement) +ValueError(body) +Other(replacement) +ValueError(body) +closed True +RuntimeError(generator didn't stop after throw()) +ValueError(body) +closed True +ValueError(close failed) +GeneratorExit() +ValueError(body) +suppression restores outer exception and releases saved references + +[case testContextManagerRuntimeMethodBinding] +from contextlib import contextmanager +from typing import Callable, Iterator +from mypy_extensions import mypyc_attr + +@mypyc_attr(allow_interpreted_subclasses=True) +class Base: + @contextmanager + def cm(self, value: str) -> Iterator[str]: + yield 'base:' + value + +# The same Base-typed call must preserve the callable's actual binding. +def invoke(receiver: Callable[[], Base], argument: Callable[[], str]) -> None: + with receiver().cm(value=argument()) as value: + print('body', value) + +[file driver.py] +from contextlib import contextmanager +from native import Base, invoke + +# A classmethod binds to the runtime class; a staticmethod has no implicit receiver. + +class ClassOverride(Base): + @classmethod + @contextmanager + def cm(cls, value): + yield cls.__name__ + ':' + value + +invoke(ClassOverride, lambda: 'value') + +class StaticOverride(Base): + @staticmethod + @contextmanager + def cm(value): + yield 'static:' + value + +invoke(StaticOverride, lambda: 'value') + +# A descriptor can bind to another object. Lookup and arguments run once, in order. +class Target: + tag = 'target' + + @contextmanager + def cm(self, value): + print('enter', self.tag) + yield value + print('exit') + +target = Target() + +class Descriptor: + def __get__(self, instance, owner): + print('lookup') + return target.cm + +class Child(Base): + cm = Descriptor() + +obj = Child() + +def receiver(): + print('receiver') + return obj + +def argument(): + print('argument') + return 'keyword' + +invoke(receiver, argument) +[out] +body ClassOverride:value +body static:value +receiver +lookup +argument +enter target +body keyword +exit + [case testUnpackKwargsCompiled] from typing import TypedDict from typing_extensions import Unpack From 55d82de1e5c1886684a226bf4b7a54f84c526644 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 20:17:48 -0700 Subject: [PATCH 2/7] . --- mypyc/irbuild/statement.py | 68 ++++++++++++++++++++------------------ mypyc/lib-rt/exc_ops.c | 62 ++++++++++++++++++++++++++-------- 2 files changed, 85 insertions(+), 45 deletions(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 659d85467bd9..1762a88f44b3 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1131,43 +1131,35 @@ def transform_with( line: int, ) -> None: - node: mypy.nodes.SymbolNode | None = None - receiver_expr: Expression | None = None + if isinstance(expr, mypy.nodes.CallExpr) and isinstance(expr.callee, mypy.nodes.RefExpr): callee = expr.callee - node = callee.node + ctm_def_node: mypy.nodes.SymbolNode | None = callee.node + self_expr: Expression | None = None if isinstance(callee, mypy.nodes.MemberExpr) and not builder.is_module_member_expr(callee): - node = None + ctm_def_node = None # Keep the experimental method path to ordinary, non-suspending calls. - if ( - all(kind in (ARG_POS, ARG_NAMED) for kind in expr.arg_kinds) - and not find_walrus_targets(expr) - and not expr_has_suspend(expr) - ): - receiver_type = get_proper_type(builder.types[callee.expr]) - if isinstance(receiver_type, Instance): - symbol = receiver_type.type.get(callee.name) + if not find_walrus_targets(expr) and not expr_has_suspend(expr): + self_type = get_proper_type(builder.types[callee.expr]) + if isinstance(self_type, Instance): + symbol = self_type.type.get(callee.name) if symbol is not None: - node = symbol.node - receiver_expr = callee.expr - - if ( - not is_async - and isinstance(expr, mypy.nodes.CallExpr) - and isinstance(expr.callee, mypy.nodes.RefExpr) - and isinstance(dec := node, mypy.nodes.Decorator) - and not (receiver_expr is not None and (dec.func.is_static or dec.func.is_class)) - and len(dec.decorators) == 1 - and isinstance(dec1 := dec.decorators[0], mypy.nodes.RefExpr) - and dec1.node - and dec1.node.fullname == "contextlib.contextmanager" - ): - return _transform_with_contextmanager(builder, expr, target, body, line, receiver_expr) + ctm_def_node = symbol.node + self_expr = callee.expr + + if ( + not is_async + and isinstance(dec := ctm_def_node, mypy.nodes.Decorator) + and not (self_expr is not None and (dec.func.is_static or dec.func.is_class)) + and len(dec.decorators) == 1 + and isinstance(dec1 := dec.decorators[0], mypy.nodes.RefExpr) + and dec1.node + and dec1.node.fullname == "contextlib.contextmanager" + ): + return _transform_with_contextmanager(builder, expr, target, body, line, self_expr) # This is basically a straight transcription of the Python code in PEP 343. # I don't actually understand why a bunch of it is the way it is. - # We could probably optimize the case where the manager is compiled by us, - # but that is not our common case at all, so. al = "a" if is_async else "" @@ -1287,6 +1279,10 @@ def _transform_with_contextmanager( line: int, receiver_expr: Expression | None, ) -> None: + + # wrapped_call = {expr.callee}.__wrapped__(*args, **kwargs) + # where {expr.callee} is either a function or a method + assert isinstance(expr.callee, mypy.nodes.RefExpr) if receiver_expr is None: wrapped_call = mypy.nodes.CallExpr( @@ -1306,11 +1302,12 @@ def _transform_with_contextmanager( args = [builder.accept(arg) for arg in expr.args] gen = builder.py_call(wrapped, args, line, expr.arg_kinds, expr.arg_names) - # Require exactly one yield: enter here, finish after a successful body. + # try: + # mgr_target = next(gen) + # except StopIteration: + # raise RuntimeError("generator didn't yield") from None mgr_target = builder.call_c(contextmanager_enter_op, [gen], line) - exc = builder.ensure_register(builder.true()) - # try: # {body} @@ -1319,6 +1316,13 @@ def try_body() -> None: builder.assign(builder.get_assignment_target(target), mgr_target, line) with_body() + # except BaseException as e: + # {contextmanager_exit_exception_op} + # finally: + # {contextmanager_exit_normal_op} + + exc = builder.ensure_register(builder.true()) + def except_body() -> None: builder.assign(exc, builder.false(), line) suppressed = builder.call_c( diff --git a/mypyc/lib-rt/exc_ops.c b/mypyc/lib-rt/exc_ops.c index 083696dc7f48..e401f2be8f98 100644 --- a/mypyc/lib-rt/exc_ops.c +++ b/mypyc/lib-rt/exc_ops.c @@ -208,17 +208,15 @@ PyObject *CPyContextManager_Enter(PyObject *gen) { static void contextmanager_exit_error(PyObject *gen, const char *message) { PyErr_SetString(PyExc_RuntimeError, message); #if PY_VERSION_HEX >= 0x030B0000 - // Since Python 3.11, contextlib closes the generator in a finally block. - // Make the protocol error the handled exception so cleanup sees it and - // any cleanup failure chains to it in the same way. + // Since Python 3.11, contextlib closes the generator in a finally block tuple_T3OOO old_exc = CPy_CatchError(); PyObject *type, *value, *traceback; PyErr_GetExcInfo(&type, &value, &traceback); PyObject *closed = PyObject_CallMethod(gen, "close", NULL); if (closed != NULL) { Py_DECREF(closed); - // A native generator can change the handled exception during close(). - // Reraise the saved protocol error, as a Python finally block does. + // A native generator can change the handled exception during close() + // Reraise the saved protocol error, as a Python finally block does PyErr_Restore(type, value, traceback); } else { Py_XDECREF(type); @@ -234,11 +232,16 @@ static void contextmanager_exit_error(PyObject *gen, const char *message) { // Finish a contextmanager whose body succeeded. Return false on error. bool CPyContextManager_ExitNormal(PyObject *gen) { + // try: + // next(self.gen) if (!PyIter_Check(gen)) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not an iterator", Py_TYPE(gen)->tp_name); return false; } PyObject *value = Py_TYPE(gen)->tp_iternext(gen); + + // except StopIteration: + // return False if (value == NULL) { if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_StopIteration)) { PyErr_Clear(); @@ -246,28 +249,41 @@ bool CPyContextManager_ExitNormal(PyObject *gen) { } return false; } + + // else: + // try: + // raise RuntimeError("generator didn't stop") + // finally: + // self.gen.close() Py_DECREF(value); contextmanager_exit_error(gen, "generator didn't stop"); return false; } // Match contextlib._GeneratorContextManager.__exit__ for a failed with body. -// The borrowed exception arguments are already normalized by CPy_CatchError. -// Return 1 to suppress, 0 to propagate the original, or -1 with a new error pending. -// Preserve the caller's handled exception on every path. int CPyContextManager_ExitException(PyObject *gen, PyObject *type, - PyObject *value, PyObject *traceback) { + PyObject *value, PyObject *traceback) { + // Native generators can change the handled exception when throw() unwinds - // an except block. Preserve the caller's state before interpreting the result. + // an except block. Preserve the caller's state before interpreting the result PyObject *saved_type, *saved_value, *saved_traceback; PyErr_GetExcInfo(&saved_type, &saved_value, &saved_traceback); + + // try: + // self.gen.throw(typ, value, traceback) #if PY_VERSION_HEX >= 0x030C0000 PyObject *yielded = PyObject_CallMethod(gen, "throw", "(O)", value); #else PyObject *yielded = PyObject_CallMethod(gen, "throw", "(OOO)", type, value, traceback); #endif PyErr_SetExcInfo(saved_type, saved_value, saved_traceback); + + // ... if we didn't get an exception from throw, in particular, StopIteration ... if (yielded != NULL) { + // try: + // raise RuntimeError("generator didn't stop after throw()") + // finally: + // self.gen.close() Py_DECREF(yielded); contextmanager_exit_error(gen, "generator didn't stop after throw()"); return -1; @@ -277,11 +293,27 @@ int CPyContextManager_ExitException(PyObject *gen, PyObject *type, PyObject *error = CPy_GetExcValue(); int result = -1; + // # Note the `return False` in the code comments is how contextlib tells Python to + // # propagate the exception being handled by __exit__ + + // except StopIteration as exc: + // return exc is not value if (CPy_ExceptionMatches(PyExc_StopIteration)) { // A body StopIteration must not be mistaken for generator completion. result = error != value; goto done; } + // except RuntimeError as exc: + // if exc is value: + // exc.__traceback__ = traceback + // return False + // if ( + // isinstance(value, StopIteration) + // and exc.__cause__ is value + // ): + // value.__traceback__ = traceback + // return False + // raise if (CPy_ExceptionMatches(PyExc_RuntimeError)) { if (error == value) { goto original; @@ -302,7 +334,13 @@ int CPyContextManager_ExitException(PyObject *gen, PyObject *type, goto original; } } - } else if (error == value) { + } + // except BaseException as exc: + // if exc is not value: + // raise + // exc.__traceback__ = traceback + // return False + else if (error == value) { goto original; } CPy_Reraise(); @@ -310,8 +348,6 @@ int CPyContextManager_ExitException(PyObject *gen, PyObject *type, original: #if PY_VERSION_HEX >= 0x030B0000 - // contextlib removes throw() frames on these paths, but not on the - // same-StopIteration path above. Keep Python attribute semantics here. if (PyObject_SetAttrString(value, "__traceback__", traceback) < 0) { goto done; } From f90eeaead5bb8a12938628467c19c0ee7a107f17 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 20:38:14 -0700 Subject: [PATCH 3/7] test first --- mypyc/test-data/run-functions.test | 223 ++++++++++++++++++----------- 1 file changed, 136 insertions(+), 87 deletions(-) diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index 6c118ea65f88..fca52b1fe24a 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -1316,122 +1316,132 @@ import traceback @contextmanager def cm_no_yield() -> Iterator[None]: + print("enter cm_no_yield") if False: + print("UNREACHABLE") yield return +def test_no_yield() -> None: + try: + with cm_no_yield(): + print("UNREACHABLE") + print("UNREACHABLE") + except RuntimeError as e: + e_any = cast(Any, e) + print( + type(e).__name__, + str(e), + e_any.__cause__ is None, + e_any.__suppress_context__, + ) + @contextmanager def cm_no_stop() -> Iterator[str]: print("enter cm_no_stop") yield "first" - print("after first") + print("after first yield") yield "second" + print("UNREACHABLE") + +def test_no_stop() -> None: + try: + with cm_no_stop() as v: + print("entered body", v) + print("UNREACHABLE") + except RuntimeError as e: + print(type(e).__name__, str(e)) @contextmanager def cm_value_error_suppress() -> Iterator[None]: + print("enter cm_value_error_suppress") try: yield + print("UNREACHABLE") except ValueError as e: - print("suppress", str(e)) + print(type(e).__name__, str(e)) + +def test_suppress_value_error() -> None: + with cm_value_error_suppress(): + raise ValueError("boom") + print("after suppressing the ValueError") @contextmanager def cm_throw_twice() -> Iterator[None]: + print("enter cm_throw_twice") try: yield + print("UNREACHABLE") except ValueError: print("throw yield") yield + print("UNREACHABLE") + +def test_throw_twice() -> None: + try: + with cm_throw_twice(): + raise ValueError("oops") + print("UNREACHABLE") + except RuntimeError as e: + print(type(e).__name__, str(e)) @contextmanager def cm_stop_passthrough() -> Iterator[None]: + print("enter cm_stop_passthrough") yield + print("UNREACHABLE") + +def test_stop_iteration_passthrough() -> None: + try: + with cm_stop_passthrough(): + raise StopIteration("stop") + print("UNREACHABLE") + except Exception as e: + print("stop passthrough", type(e).__name__, str(e)) @contextmanager def cm_stop_suppress() -> Iterator[None]: + print("enter cm_stop_suppress") try: yield + print("UNREACHABLE") except StopIteration as e: - print("suppress stop", str(e)) + print(type(e).__name__, str(e)) + +def test_stop_iteration_suppress() -> None: + with cm_stop_suppress(): + raise StopIteration("manual stop") + print("after suppressing the StopIteration") @contextmanager def cm_value() -> Iterator[int]: print("enter cm_value") try: yield 1 + print("UNREACHABLE") finally: print("exit cm_value") -@contextmanager -def cm_gen() -> Iterator[int]: - print("cm enter") - try: - yield 1 - finally: - print("cm exit") - -@contextmanager -def cm_log(tag: str) -> Iterator[None]: - print(tag, "enter") - try: - yield - finally: - print(tag, "exit") - -def test_no_yield() -> None: - try: - with cm_no_yield(): - print("body no yield") - except RuntimeError as e: - e_any = cast(Any, e) - print( - "no_yield", - str(e), - e_any.__cause__ is None, - e_any.__suppress_context__, - ) - -def test_no_stop() -> None: - try: - with cm_no_stop() as v: - print("body", v) - except RuntimeError as e: - print("no_stop", str(e)) - -def test_suppress_value_error() -> None: - with cm_value_error_suppress(): - raise ValueError("boom") - print("after suppress value") - -def test_throw_twice() -> None: - try: - with cm_throw_twice(): - raise ValueError("oops") - except RuntimeError as e: - print("throw_twice", str(e)) - -def test_stop_iteration_passthrough() -> None: - try: - with cm_stop_passthrough(): - raise StopIteration("stop") - except Exception as e: - print("stop passthrough", type(e).__name__, str(e)) - -def test_stop_iteration_suppress() -> None: - with cm_stop_suppress(): - raise StopIteration("stop2") - print("after suppress stop") - def test_traceback() -> None: try: with cm_value(): raise ValueError("trace") + print("UNREACHABLE") except Exception as e: e_any = cast(Any, e) tb = e_any.__traceback__ assert tb is not None tb_mod = cast(Any, traceback) names = [frame.name for frame in tb_mod.extract_tb(tb)] - print("traceback_has_cm_value", "cm_value" in names) + print("traceback_has_cm_value", "test_traceback" in names, "cm_value" in names) + +@contextmanager +def cm_gen() -> Iterator[int]: + print("enter cm_gen") + try: + yield 1 + finally: + print("exit cm_gen") def test_generator_with_yield() -> None: def gen() -> Generator[int, None, None]: @@ -1443,14 +1453,24 @@ def test_generator_with_yield() -> None: print(next(g)) try: next(g) + print("UNREACHABLE") except StopIteration: print("gen done") +@contextmanager +def cm_log(tag: str) -> Iterator[None]: + print("enter cm_log", tag) + try: + yield + finally: + print("exit cm_log", tag) + def test_return_in_with() -> None: def inner() -> int: with cm_log("ret"): print("ret body") return 5 + print("UNREACHABLE") return 0 print("ret result", inner()) @@ -1468,39 +1488,68 @@ from native import ( test_return_in_with, ) -test_no_yield() -test_no_stop() -test_suppress_value_error() -test_throw_twice() -test_stop_iteration_passthrough() -test_stop_iteration_suppress() -test_traceback() -test_generator_with_yield() -test_return_in_with() +fns = ( + test_no_yield, + test_no_stop, + test_suppress_value_error, + test_throw_twice, + test_stop_iteration_passthrough, + test_stop_iteration_suppress, + test_traceback, + test_generator_with_yield, + test_return_in_with, +) +for i, fn in enumerate(fns): + if i: + print() + print(f"Running {fn.__name__}") + fn() [out] -no_yield generator didn't yield True True +Running test_no_yield +enter cm_no_yield +RuntimeError generator didn't yield True True + +Running test_no_stop enter cm_no_stop -body first -after first -no_stop generator didn't stop -suppress boom -after suppress value +entered body first +after first yield +RuntimeError generator didn't stop + +Running test_suppress_value_error +enter cm_value_error_suppress +ValueError boom +after suppressing the ValueError + +Running test_throw_twice +enter cm_throw_twice throw yield -throw_twice generator didn't stop after throw() +RuntimeError generator didn't stop after throw() + +Running test_stop_iteration_passthrough +enter cm_stop_passthrough stop passthrough StopIteration stop -suppress stop stop2 -after suppress stop + +Running test_stop_iteration_suppress +enter cm_stop_suppress +StopIteration manual stop +after suppressing the StopIteration + +Running test_traceback enter cm_value exit cm_value -traceback_has_cm_value False -cm enter +traceback_has_cm_value True False + +Running test_generator_with_yield +enter cm_gen 1 after yield in with -cm exit +exit cm_gen gen done -ret enter + +Running test_return_in_with +enter cm_log ret ret body -ret exit +exit cm_log ret ret result 5 [case testContextManagerRuntimePrimitives] From 32bb212331f9b5667785040fa53ed48775479ac9 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 20:52:37 -0700 Subject: [PATCH 4/7] test two --- mypyc/test-data/run-functions.test | 380 +++++++++++++---------------- 1 file changed, 165 insertions(+), 215 deletions(-) diff --git a/mypyc/test-data/run-functions.test b/mypyc/test-data/run-functions.test index fca52b1fe24a..3e3801b2e8d7 100644 --- a/mypyc/test-data/run-functions.test +++ b/mypyc/test-data/run-functions.test @@ -1488,7 +1488,7 @@ from native import ( test_return_in_with, ) -fns = ( +fns = [ test_no_yield, test_no_stop, test_suppress_value_error, @@ -1498,7 +1498,7 @@ fns = ( test_traceback, test_generator_with_yield, test_return_in_with, -) +] for i, fn in enumerate(fns): if i: print() @@ -1562,19 +1562,23 @@ import helper def implicit(values: list[int]) -> Iterator[int]: return iter(values) -@contextmanager -def noniterator() -> Iterator[None]: - return cast(Any, 42) - -def test_entry_protocol() -> None: +def test_empty_iterator() -> None: try: with implicit([]): - print('empty body') + print("UNREACHABLE") + print("UNREACHABLE") except RuntimeError as error: print(str(error), type(error.__context__).__name__) + +@contextmanager +def noniterator() -> Iterator[None]: + return cast(Any, 42) + +def test_noniterator() -> None: try: with noniterator(): - print('noniterator body') + print("UNREACHABLE") + print("UNREACHABLE") except TypeError as error: print(str(error)) @@ -1582,59 +1586,65 @@ def test_entry_protocol() -> None: @contextmanager def fail_entry(error: ValueError) -> Iterator[None]: raise error + print("UNREACHABLE") yield -@contextmanager -def fail_exit(error: ValueError) -> Iterator[None]: - yield - raise error - -def test_failure_identity() -> None: +def test_entry_failure_identity() -> None: original = ValueError('original') try: with fail_entry(original): - print('entry body') + print("UNREACHABLE") + print("UNREACHABLE") except ValueError as error: assert error is original print('entry error') + +@contextmanager +def fail_exit(error: ValueError) -> Iterator[None]: + yield + raise error + +def test_exit_failure_identity() -> None: + original = ValueError('original') try: with fail_exit(original): print('exit body') + print("UNREACHABLE") except ValueError as error: assert error is original print('exit error') # Protocol errors temporarily replace the handled exception. They must restore -# it and release the saved references, even after repeatedly closing generators. +# it and release the saved references, even after repeated failures. @contextmanager def missing() -> Iterator[None]: if False: + print("UNREACHABLE") yield -@contextmanager -def twice(closed: list[str]) -> Iterator[int]: - try: - yield 1 - yield 2 - finally: - closed.append('closed') - def failed_entry() -> None: try: with missing(): + print("UNREACHABLE") assert False + print("UNREACHABLE") except RuntimeError as error: assert isinstance(error.__context__, StopIteration) +@contextmanager +def twice() -> Iterator[int]: + yield 1 + yield 2 + print("UNREACHABLE") + def failed_exit() -> None: - closed: list[str] = [] try: - with twice(closed): + with twice(): pass except RuntimeError: - if helper.closes_generator: - assert closed == ['closed'] + pass else: + print("UNREACHABLE") assert False def test_outer_exception() -> None: @@ -1650,12 +1660,12 @@ def test_outer_exception() -> None: # Collect exception/traceback cycles before measuring retained references. helper.collect() before = helper.refcount(outer) - for i in range(20): + for i in range(9): failed_entry() helper.collect() assert helper.refcount(outer) == before assert helper.exc_info()[1] is outer - for i in range(20): + for i in range(9): failed_exit() helper.collect() assert helper.refcount(outer) == before @@ -1663,29 +1673,11 @@ def test_outer_exception() -> None: assert helper.exc_info()[1] is None print('outer exception restored; saved references released') -# A failure during close supersedes the protocol error but retains its context. -@contextmanager -def bad_close() -> Iterator[int]: - # Use a Python generator: native generator.close has different context chaining. - return helper.bad_close() - -def test_cleanup_error() -> None: - try: - with bad_close() as value: - print('body', value) - except ValueError as error: - current: BaseException | None = error - while current is not None: - print(f'{type(current).__name__}({current})') - current = current.__context__ - [file helper.py] -from typing import Any, Iterator +from typing import Any import gc import sys -closes_generator: bool = sys.version_info >= (3, 11) # type: ignore - def exc_info() -> Any: return sys.exc_info() # type: ignore @@ -1695,45 +1687,43 @@ def collect() -> None: def refcount(value: Any) -> int: return sys.getrefcount(value) # type: ignore -def bad_close() -> Iterator[int]: - try: - yield 1 - yield 2 - finally: - print('closing') - raise ValueError('close failed') - [file driver.py] -import helper from native import ( - test_entry_protocol, test_failure_identity, test_outer_exception, test_cleanup_error, + test_empty_iterator, + test_noniterator, + test_entry_failure_identity, + test_exit_failure_identity, + test_outer_exception, ) -test_entry_protocol() -test_failure_identity() -test_outer_exception() -# contextlib closes a generator after an extra yield starting in Python 3.11. -if helper.closes_generator: - test_cleanup_error() +fns = [ + test_empty_iterator, + test_noniterator, + test_entry_failure_identity, + test_exit_failure_identity, + test_outer_exception, +] +for i, fn in enumerate(fns): + if i: + print() + print(f"Running {fn.__name__}") + fn() [out] +Running test_empty_iterator generator didn't yield StopIteration + +Running test_noniterator 'int' object is not an iterator + +Running test_entry_failure_identity entry error + +Running test_exit_failure_identity exit body exit error + +Running test_outer_exception outer exception restored; saved references released -[out version>=3.11] -generator didn't yield StopIteration -'int' object is not an iterator -entry error -exit body -exit error -outer exception restored; saved references released -body 1 -closing -ValueError(close failed) -GeneratorExit() -RuntimeError(generator didn't stop) [case testContextManagerExceptionalExit] @@ -1746,19 +1736,10 @@ def scope(generator: Iterator[None]) -> Iterator[None]: return generator def invoke(generator: Iterator[None], error: BaseException, - captured: list[Any], reference: bool = False) -> None: - if reference: - # An ordinary call keeps contextlib as an oracle on the same generator type. - manager = scope(generator) - with manager: - helper.raise_body(error, captured) - else: - with scope(generator): - helper.raise_body(error, captured) - -# Native generators can propagate StopIteration directly instead of PEP 479 wrapping. -def native_passthrough() -> Iterator[None]: - yield + captured: list[Any]) -> None: + with scope(generator): + helper.raise_body(error, captured) + print("UNREACHABLE") # Unwinding this handler during throw() can change the native caller's exception state. def native_handler() -> Iterator[None]: @@ -1766,12 +1747,10 @@ def native_handler() -> Iterator[None]: raise TypeError('earlier') except BaseException: yield + print("UNREACHABLE") [file helper.py] from typing import Any, Iterator, cast -import sys - -closes_generator: bool = sys.version_info >= (3, 11) # type: ignore class Other(BaseException): pass @@ -1788,26 +1767,19 @@ def raise_body(error: BaseException, captured: list[Any]) -> None: def passthrough() -> Iterator[None]: yield + print("UNREACHABLE") def replace(error: BaseException) -> Iterator[None]: try: yield + print("UNREACHABLE") except BaseException: raise error -def extra_yield(close_error: bool) -> Iterator[None]: - try: - try: - yield - except ValueError: - yield - finally: - if close_error: - raise ValueError('close failed') - def suppress() -> Iterator[None]: try: yield + print("UNREACHABLE") except ValueError: pass @@ -1815,135 +1787,113 @@ def suppress() -> Iterator[None]: import gc import sys import helper -from native import invoke, native_passthrough, native_handler, scope +from native import invoke, native_handler -def observe(generator, original, reference=False): - captured = [] - try: - invoke(generator, original, captured, reference) - except BaseException as caught: - names = [] - has_original_tail = False - tb = caught.__traceback__ - while tb is not None: - has_original_tail |= tb is captured[0] - # The C primitive has no Python frame corresponding to __exit__. - if tb.tb_frame.f_code.co_name != '__exit__': - names.append(tb.tb_frame.f_code.co_name) - tb = tb.tb_next - chain = [] - error = caught - while error is not None: - chain.append(f'{type(error).__name__}({error})') - error = error.__context__ - closed = getattr(generator, 'gi_frame', None) is None - return caught is original, names, has_original_tail, chain, closed - raise AssertionError('unexpected suppression') - - -def compare(factory, error_type=ValueError): - direct = observe(factory(), error_type('body')) - reference = observe(factory(), error_type('body'), reference=True) - assert direct == reference, (direct, reference) - return direct - - -# These take different branches in contextlib; all must propagate the same object. -for error_type in [RuntimeError, helper.Other, helper.StopSubclass]: - result = compare(helper.passthrough, error_type) - assert result[0] and result[2] - print(error_type.__name__, 'identity and traceback match contextlib') -result = compare(native_passthrough, StopIteration) -assert result[0] -print('native StopIteration identity and traceback match contextlib') - -# Use an interpreted caller as the oracle: ordinary compiled with also relies on -# native handled-exception state. The interpreted generator isolates that state. -def propagate_from_handler(reference): +def test_exception_identity_and_traceback(): + # These take different branches in contextlib; all must propagate the same object. + for error_type in [RuntimeError, helper.Other, helper.StopSubclass]: + original = error_type('body') + captured = [] + try: + invoke(helper.passthrough(), original, captured) + except BaseException as caught: + assert caught is original + tb = caught.__traceback__ + while tb is not None and tb is not captured[0]: + tb = tb.tb_next + assert tb is captured[0] + print(error_type.__name__, 'identity and original traceback preserved') + else: + print("UNREACHABLE") + assert False, 'unexpected suppression' + +# The interpreted generator isolates the caller's handled-exception state. +def propagate_from_handler(): original = ValueError('body') try: - if reference: - with scope(native_handler()): - raise original - else: - invoke(native_handler(), original, []) + invoke(native_handler(), original, []) except BaseException as caught: yield caught is original else: + print("UNREACHABLE") yield False -def check_handler(): - for reference in [True, False]: - assert list(propagate_from_handler(reference)) == [True] +def test_native_handler_exception_state(): + assert list(propagate_from_handler()) == [True] + assert sys.exc_info()[1] is None + try: + raise KeyError('outer') + except KeyError as outer: + assert list(propagate_from_handler()) == [True] + assert sys.exc_info()[1] is outer + assert sys.exc_info()[1] is None + print('native handler preserves body exception with and without an outer handler') + +def test_replacement_exception(): + # A different exception from throw() must replace the body's exception. + for error_type in [RuntimeError, helper.Other]: + original = ValueError('body') + replacement = error_type('replacement') + try: + invoke(helper.replace(replacement), original, []) + except BaseException as caught: + assert caught is replacement + assert caught.__context__ is original + assert original.__context__ is None + print(f'{type(caught).__name__}({caught})') + print(f'{type(original).__name__}({original})') + else: + print("UNREACHABLE") + assert False, 'unexpected suppression' -check_handler() -assert sys.exc_info()[1] is None -try: - raise KeyError('outer') -except KeyError as outer: - check_handler() - assert sys.exc_info()[1] is outer -assert sys.exc_info()[1] is None -print('native handler preserves body exception with and without an outer handler') - -# A different exception from throw() must replace the body's exception. -for replacement in [RuntimeError, helper.Other]: - result = compare(lambda: helper.replace(replacement('replacement'))) - assert not result[0] - print(*result[3], sep='\n') - -# A second yield requires close(), including its effects and any cleanup failure. -if helper.closes_generator: - for close_error in [False, True]: - result = compare(lambda: helper.extra_yield(close_error)) - print('closed', result[4]) - print(*result[3], sep='\n') - -# Suppression must restore the outer handled exception without retaining its refs. -outer = ValueError('outer') -try: - raise outer -except ValueError: - gc.collect() - before = sys.getrefcount(outer) - for i in range(20): - invoke(helper.suppress(), ValueError('body'), []) - assert sys.exc_info()[1] is outer - gc.collect() - assert sys.getrefcount(outer) == before - print('suppression restores outer exception and releases saved references') -assert sys.exc_info()[1] is None + +def test_suppression_exception_state_and_references(): + # Suppression must restore the outer handled exception without retaining its refs. + outer = ValueError('outer') + try: + raise outer + except ValueError: + gc.collect() + before = sys.getrefcount(outer) + for i in range(9): + invoke(helper.suppress(), ValueError('body'), []) + assert sys.exc_info()[1] is outer + gc.collect() + assert sys.getrefcount(outer) == before + print('suppression restores outer exception and releases saved references') + assert sys.exc_info()[1] is None + + +fns = [ + test_exception_identity_and_traceback, + test_native_handler_exception_state, + test_replacement_exception, + test_suppression_exception_state_and_references, +] +for i, fn in enumerate(fns): + if i: + print() + print(f"Running {fn.__name__}") + fn() [out] -RuntimeError identity and traceback match contextlib -Other identity and traceback match contextlib -StopSubclass identity and traceback match contextlib -native StopIteration identity and traceback match contextlib -native handler preserves body exception with and without an outer handler -RuntimeError(replacement) -ValueError(body) -Other(replacement) -ValueError(body) -suppression restores outer exception and releases saved references -[out version>=3.11] -RuntimeError identity and traceback match contextlib -Other identity and traceback match contextlib -StopSubclass identity and traceback match contextlib -native StopIteration identity and traceback match contextlib +Running test_exception_identity_and_traceback +RuntimeError identity and original traceback preserved +Other identity and original traceback preserved +StopSubclass identity and original traceback preserved + +Running test_native_handler_exception_state native handler preserves body exception with and without an outer handler + +Running test_replacement_exception RuntimeError(replacement) ValueError(body) Other(replacement) ValueError(body) -closed True -RuntimeError(generator didn't stop after throw()) -ValueError(body) -closed True -ValueError(close failed) -GeneratorExit() -ValueError(body) + +Running test_suppression_exception_state_and_references suppression restores outer exception and releases saved references [case testContextManagerRuntimeMethodBinding] From 0b127a495c5fcaed1e504b3c0445b681aca2b9be Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:02:56 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/irbuild/statement.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 1762a88f44b3..576586733646 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1131,7 +1131,6 @@ def transform_with( line: int, ) -> None: - if isinstance(expr, mypy.nodes.CallExpr) and isinstance(expr.callee, mypy.nodes.RefExpr): callee = expr.callee ctm_def_node: mypy.nodes.SymbolNode | None = callee.node From d5f61ad94a60279581722ed41ff39ebd7a8b1456 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 20:52:37 -0700 Subject: [PATCH 6/7] test two --- mypyc/irbuild/statement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 576586733646..4ee3cdf8888d 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1137,7 +1137,7 @@ def transform_with( self_expr: Expression | None = None if isinstance(callee, mypy.nodes.MemberExpr) and not builder.is_module_member_expr(callee): ctm_def_node = None - # Keep the experimental method path to ordinary, non-suspending calls. + # Keep the method path to ordinary, non-suspending calls. if not find_walrus_targets(expr) and not expr_has_suspend(expr): self_type = get_proper_type(builder.types[callee.expr]) if isinstance(self_type, Instance): From a17a9316789cf4418b4fa415eba6280fa3320791 Mon Sep 17 00:00:00 2001 From: hauntsaninja Date: Wed, 21 Oct 2026 21:07:36 -0700 Subject: [PATCH 7/7] fix comment --- mypyc/irbuild/statement.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 4ee3cdf8888d..b20022c1ce8b 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -1137,7 +1137,8 @@ def transform_with( self_expr: Expression | None = None if isinstance(callee, mypy.nodes.MemberExpr) and not builder.is_module_member_expr(callee): ctm_def_node = None - # Keep the method path to ordinary, non-suspending calls. + # Since we construct the call to the method ourselves in _transform_with_contextmanager, + # we're a little paranoid about doing things in the wrong order in weird cases if not find_walrus_targets(expr) and not expr_has_suspend(expr): self_type = get_proper_type(builder.types[callee.expr]) if isinstance(self_type, Instance):