From 84a917d316e094d299a0646122a86aa0f1fc9211 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 21 Aug 2026 17:34:25 +0200 Subject: [PATCH 01/10] [mypyc] Synchronize native-to-native imports --- mypyc/codegen/emitmodule.py | 54 +++++++++- mypyc/common.py | 3 + mypyc/irbuild/builder.py | 51 +++++++++- mypyc/lib-rt/CPy.h | 26 +++++ mypyc/lib-rt/locks.c | 98 ++++++++++++++++++ mypyc/lib-rt/misc_ops.c | 192 +++++++++++++++++++++++++++-------- mypyc/primitives/misc_ops.py | 12 ++- 7 files changed, 387 insertions(+), 49 deletions(-) create mode 100644 mypyc/lib-rt/locks.c diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 9130007e3f6ef..64e53fcac90bd 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -47,7 +47,9 @@ from mypyc.codegen.literals import Literals from mypyc.common import ( EXT_SUFFIX, + IMPORT_STATE_PREFIX, IS_FREE_THREADED, + MODULE_LOCK_API_PREFIX, MODULE_PREFIX, PREFIX, RUNTIME_C_FILES, @@ -656,6 +658,13 @@ def __init__( # probably want to enable it always, but we'll wait until it's stable. self.multi_phase_init = IS_FREE_THREADED + def import_state_name(self, module_name: str) -> str: + return f"{IMPORT_STATE_PREFIX}{exported_name(module_name)}" + + def module_lock_api_name(self) -> str: + assert self.group_name is not None + return f"{MODULE_LOCK_API_PREFIX}{exported_name(self.group_name)}" + @property def group_suffix(self) -> str: return "_" + exported_name(self.group_name) if self.group_name else "" @@ -692,6 +701,9 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: base_emitter.emit_line(f'#include "__native_internal{self.short_group_suffix}.h"') emitter = base_emitter + if self.use_shared_lib: + self.declare_module_lock_api() + self.generate_literal_tables() for module_name, module in self.modules.items(): @@ -947,6 +959,16 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: "", ) + lock_api = self.module_lock_api_name() + emitter.emit_lines( + f"if ({lock_api} == NULL) {{", + f"{lock_api} = CPyModuleLockAPI_Alloc();", + f"if ({lock_api} == NULL) goto fail;", + "}", + "if (CPyGlobalsInit() < 0) goto fail;", + "", + ) + if self.compiler_options.separate: emitter.emit_lines( 'capsule = PyCapsule_New(&exports, "{}.exports", NULL);'.format( @@ -1265,8 +1287,9 @@ def emit_module_exec_func( exec_name = f"CPyExec_{exported_name(module_name)}" declaration = f"int {exec_name}(PyObject *module)" emitter.context.declarations[exec_name] = HeaderDeclaration(declaration + ";") + impl_name = f"{exec_name}__impl" module_static = self.module_internal_static_name(module_name, emitter) - emitter.emit_lines(declaration, "{") + emitter.emit_lines(f"static int {impl_name}(PyObject *module)", "{") emitter.emit_line("intern_strings();") if self.compiler_options.depends_on_librt_internal: emitter.emit_line("if (import_librt_internal() < 0) {") @@ -1333,7 +1356,10 @@ def emit_module_exec_func( name_prefix = cl.name_prefix(emitter.names) emitter.emit_line(f"CPyDef_{name_prefix}_trait_vtable_setup();") - emitter.emit_lines("if (CPyGlobalsInit() < 0)", " goto fail;") + if not self.use_shared_lib: + # With shared lib we initialize globals in its init function in case + # modules are executed concurrently. + emitter.emit_lines("if (CPyGlobalsInit() < 0)", " goto fail;") self.generate_top_level_call(module, emitter) @@ -1357,6 +1383,14 @@ def emit_module_exec_func( emitter.emit_line("return -1;") emitter.emit_line("}") + state = self.import_state_name(module_name) + emitter.emit_lines( + declaration, + "{", + f'return CPyImport_Exec(module, "{module_name}", {impl_name}, &{state});', + "}", + ) + def emit_init_only_func(self, emitter: Emitter, module_name: str, module_prefix: str) -> None: """Emit CPyInitOnly_* which creates the module object without executing the body. @@ -1386,10 +1420,12 @@ def emit_module_init_func( ) -> None: if not self.use_shared_lib: declaration = f"PyMODINIT_FUNC PyInit_{module_name}(void)" + impl_declaration = declaration else: n = f"CPyInit_{exported_name(module_name)}" declaration = f"PyObject *{n}(void)" emitter.context.declarations[n] = HeaderDeclaration(declaration + ";") + impl_declaration = declaration if self.multi_phase_init: emitter.emit_lines(declaration, "{") @@ -1404,7 +1440,7 @@ def emit_module_init_func( self.emit_init_only_func(emitter, module_name, module_prefix) # Emit CPyInit_* / PyInit_* which creates the module and executes the body. - emitter.emit_lines(declaration, "{") + emitter.emit_lines(impl_declaration, "{") module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_line("PyObject* modname = NULL;") @@ -1558,10 +1594,22 @@ def declare_module(self, module_name: str, emitter: Emitter) -> None: if module_name in self.modules: internal_static_name = self.module_internal_static_name(module_name, emitter) self.declare_global("CPyModule *", internal_static_name, initializer="NULL") + state_name = self.import_state_name(module_name) + if state_name not in self.context.declarations: + self.context.declarations[state_name] = HeaderDeclaration( + f"CPyImportState {state_name};", defn=[f"CPyImportState {state_name} = {{0}};"] + ) static_name = emitter.static_name(module_name, None, prefix=MODULE_PREFIX) self.declare_global("CPyModule *", static_name) self.simple_inits.append((static_name, "Py_None")) + def declare_module_lock_api(self) -> None: + name = self.module_lock_api_name() + if name not in self.context.declarations: + self.context.declarations[name] = HeaderDeclaration( + f"CPyModuleLockAPI *{name};", defn=[f"CPyModuleLockAPI *{name} = NULL;"] + ) + def declare_imports(self, imps: Iterable[str], emitter: Emitter) -> None: for imp in imps: self.declare_module(imp, emitter) diff --git a/mypyc/common.py b/mypyc/common.py index fa34647c5c729..61780f31bf055 100644 --- a/mypyc/common.py +++ b/mypyc/common.py @@ -14,6 +14,8 @@ STATIC_PREFIX: Final = "CPyStatic_" # Static variables (for literals etc.) TYPE_PREFIX: Final = "CPyType_" # Type object struct MODULE_PREFIX: Final = "CPyModule_" # Cached modules +IMPORT_STATE_PREFIX: Final = "CPyImportState_" # Native module initialization state +MODULE_LOCK_API_PREFIX: Final = "CPyModuleLockAPI_" # CPython module-lock API cache TYPE_VAR_PREFIX: Final = "CPyTypeVar_" # Type variables when using new-style Python 3.12 syntax ATTR_PREFIX: Final = "_" # Attributes FAST_PREFIX: Final = "__mypyc_fast_" # Optimized methods in non-extension classes @@ -92,6 +94,7 @@ "tuple_ops.c", "exc_ops.c", "misc_ops.c", + "locks.c", "generic_ops.c", "pythonsupport.c", "function_wrapper.c", diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index f4e3745836cf5..2f75b393acaa8 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -70,9 +70,11 @@ BITMAP_BITS, EXT_SUFFIX, GENERATOR_ATTRIBUTE_PREFIX, + IMPORT_STATE_PREFIX, IS_FREE_THREADED, KEEP_ALIVE_SHORT_LIVED, KEEP_ALIVE_WHOLE_EXPRESSION, + MODULE_LOCK_API_PREFIX, MODULE_PREFIX, SELF_NAME, TEMP_ATTR_NAME, @@ -168,6 +170,7 @@ check_unpack_count_op, get_module_dict_op, import_op, + native_import_is_initialized_op, native_import_op, ) from mypyc.primitives.registry import CFunctionDescription, function_ops @@ -523,10 +526,36 @@ def gen_import(self, module: str, line: int) -> None: self.imports[module] = None needs_import, out = BasicBlock(), BasicBlock() - self.check_if_module_loaded(module, line, needs_import, out) + is_native_module = self.is_native_module(module) + is_same_group_native = is_native_module and self.is_same_group_module(module) + import_state: Value | None = None + module_lock_api: Value | None = None + if is_same_group_native: + import_state = self.add( + LoadAddress(c_pointer_rprimitive, f"{IMPORT_STATE_PREFIX}{exported_name(module)}") + ) + group_name = self.mapper.group_map.get(self.module_name) + if group_name is not None: + module_lock_api = self.add( + LoadGlobal( + c_pointer_rprimitive, + f"{MODULE_LOCK_API_PREFIX}{exported_name(group_name)}", + ) + ) + else: + module_lock_api = Integer(0, c_pointer_rprimitive) + if is_native_module and not is_same_group_native: + # A sys.modules entry may still be executing in another thread. For + # native modules in another compilation group, use CPython's import + # path so that its per-module lock waits for initialization. + self.goto(needs_import) + else: + self.check_if_module_loaded(module, line, needs_import, out, import_state) self.activate_block(needs_import) - if self.is_native_module(module) and self.is_same_group_module(module): + if is_same_group_native: + assert import_state is not None + assert module_lock_api is not None # Use custom import machinery for native-to-native imports in the same group init_only_func = self.add( LoadGlobal(c_pointer_rprimitive, f"CPyInitOnly_{exported_name(module)}") @@ -559,6 +588,8 @@ def gen_import(self, module: str, line: int) -> None: init_only_func, exec_func, module_static, + import_state, + module_lock_api, shared_lib_file, ext_suffix, Integer(1 if is_pkg else 0, c_pyssize_t_rprimitive), @@ -572,7 +603,12 @@ def gen_import(self, module: str, line: int) -> None: self.goto_and_activate(out) def check_if_module_loaded( - self, id: str, line: int, needs_import: BasicBlock, out: BasicBlock + self, + id: str, + line: int, + needs_import: BasicBlock, + out: BasicBlock, + import_state: Value | None = None, ) -> None: """Generate code that checks if the module `id` has been loaded yet. @@ -583,7 +619,14 @@ def check_if_module_loaded( out: the BasicBlock that is run if the module has already been loaded""" first_load = self.load_module(id) comparison = self.translate_is_op(first_load, self.none_object(line), "is not", line) - self.add_bool_branch(comparison, out, needs_import) + if import_state is None: + self.add_bool_branch(comparison, out, needs_import) + else: + check_initialized = BasicBlock() + self.add_bool_branch(comparison, check_initialized, needs_import) + self.activate_block(check_initialized) + initialized = self.call_c(native_import_is_initialized_op, [import_state], line) + self.add_bool_branch(initialized, out, needs_import) def get_module(self, module: str, line: int) -> Value: # Python 3.7 has a nice 'PyImport_GetModule' function that we can't use :( diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index e2b4ff0e8c750..186cdce817349 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -22,6 +22,29 @@ extern "C" { #define CPYTHON_LARGE_INT_ERRMSG "Python int too large to convert to C ssize_t" +// Native module import synchronization + +typedef struct CPyModuleLockAPI CPyModuleLockAPI; + +typedef struct { + int32_t initialized; +} CPyImportState; + +enum { + CPY_LOCK_ERROR = -1, + CPY_LOCK_ACQUIRED = 0, + CPY_LOCK_DEADLOCK = 1, +}; + +CPyModuleLockAPI *CPyModuleLockAPI_Alloc(void); +void CPyModuleLockAPI_Free(CPyModuleLockAPI *api); +int CPyImport_AcquireLock(CPyModuleLockAPI *api, PyObject *module_name, + PyObject **module_lock); +int CPyImport_ReleaseLock(PyObject *module_lock); +bool CPyImport_IsInitialized(const CPyImportState *state); +void CPyImport_SetInitialized(CPyImportState *state, bool initialized); + + // Naming conventions: // // Tagged: tagged int @@ -1043,8 +1066,11 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, PyObject *(*init_only_fn)(void), int (*exec_fn)(PyObject *), CPyModule **module_static, + CPyImportState *state, CPyModuleLockAPI *lock_api, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); +int CPyImport_Exec(PyObject *module, const char *module_name, + int (*exec_fn)(PyObject *), CPyImportState *state); int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); diff --git a/mypyc/lib-rt/locks.c b/mypyc/lib-rt/locks.c new file mode 100644 index 0000000000000..16ded3aa983fc --- /dev/null +++ b/mypyc/lib-rt/locks.c @@ -0,0 +1,98 @@ +#include "CPy.h" + +#ifdef _WIN32 +#include +#endif + +struct CPyModuleLockAPI { + PyObject *get_module_lock; + PyObject *deadlock_error; +}; + +CPyModuleLockAPI *CPyModuleLockAPI_Alloc(void) { + CPyModuleLockAPI *api = PyMem_Calloc(1, sizeof(CPyModuleLockAPI)); + if (api == NULL) { + PyErr_NoMemory(); + return NULL; + } + + PyObject *bootstrap = PyImport_ImportModule("importlib._bootstrap"); + if (bootstrap == NULL) { + PyMem_Free(api); + return NULL; + } + api->get_module_lock = PyObject_GetAttrString(bootstrap, "_get_module_lock"); + api->deadlock_error = PyObject_GetAttrString(bootstrap, "_DeadlockError"); + Py_DECREF(bootstrap); + if (api->get_module_lock == NULL || api->deadlock_error == NULL) { + CPyModuleLockAPI_Free(api); + return NULL; + } + return api; +} + +void CPyModuleLockAPI_Free(CPyModuleLockAPI *api) { + if (api == NULL) { + return; + } + Py_XDECREF(api->get_module_lock); + Py_XDECREF(api->deadlock_error); + PyMem_Free(api); +} + +int CPyImport_AcquireLock(CPyModuleLockAPI *api, PyObject *module_name, + PyObject **acquired_lock) { + *acquired_lock = NULL; + if (api == NULL) { + return CPY_LOCK_ACQUIRED; + } + + PyObject *module_lock = PyObject_CallOneArg(api->get_module_lock, module_name); + if (module_lock == NULL) { + return CPY_LOCK_ERROR; + } + PyObject *result = PyObject_CallMethod(module_lock, "acquire", NULL); + if (result == NULL) { + if (PyErr_ExceptionMatches(api->deadlock_error)) { + PyErr_Clear(); + Py_DECREF(module_lock); + return CPY_LOCK_DEADLOCK; + } + Py_DECREF(module_lock); + return CPY_LOCK_ERROR; + } + Py_DECREF(result); + *acquired_lock = module_lock; + return CPY_LOCK_ACQUIRED; +} + +int CPyImport_ReleaseLock(PyObject *module_lock) { + if (module_lock == NULL) { + return 0; + } + + PyObject *result = PyObject_CallMethod(module_lock, "release", NULL); + Py_DECREF(module_lock); + if (result == NULL) { + return -1; + } + Py_DECREF(result); + return 0; +} + +bool CPyImport_IsInitialized(const CPyImportState *state) { +#ifdef _WIN32 + return InterlockedCompareExchange( + (volatile LONG *)&state->initialized, 0, 0) != 0; +#else + return __atomic_load_n(&state->initialized, __ATOMIC_ACQUIRE) != 0; +#endif +} + +void CPyImport_SetInitialized(CPyImportState *state, bool initialized) { +#ifdef _WIN32 + InterlockedExchange((volatile LONG *)&state->initialized, initialized); +#else + __atomic_store_n(&state->initialized, initialized, __ATOMIC_RELEASE); +#endif +} diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 392dba0deca4c..f1880b141f64b 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1506,46 +1506,143 @@ static int CPyImport_SetModuleSpec(PyObject *modobj, PyObject *module_name, return 0; } +// Set module.__spec__._initializing for CPython's import machinery. +static int CPyImport_SetInitializing(PyObject *module, bool initializing) { + PyObject *spec = PyObject_GetAttrString(module, "__spec__"); + if (spec == NULL) { + return -1; + } + int result = PyObject_SetAttrString(spec, "_initializing", + initializing ? Py_True : Py_False); + Py_DECREF(spec); + return result; +} + +// Import module_name's parent and return owned parent and child-name references. +// Both outputs are NULL for a top-level module. +static int CPyImport_ImportParent(PyObject *module_name, PyObject **parent_module, + PyObject **child_name) { + *parent_module = NULL; + *child_name = NULL; + Py_ssize_t name_len = PyUnicode_GetLength(module_name); + if (name_len < 0) { + return -1; + } + Py_ssize_t dot = PyUnicode_FindChar(module_name, '.', 0, name_len, -1); + if (dot < 0) { + return 0; + } + PyObject *parent_name = PyUnicode_Substring(module_name, 0, dot); + *child_name = PyUnicode_Substring(module_name, dot + 1, name_len); + if (parent_name == NULL || *child_name == NULL) { + Py_XDECREF(parent_name); + Py_CLEAR(*child_name); + return -1; + } + *parent_module = PyImport_Import(parent_name); + Py_DECREF(parent_name); + if (*parent_module == NULL) { + Py_CLEAR(*child_name); + return -1; + } + return 0; +} + +// Import module_name's parent and bind the module under its child name. +static int CPyImport_SetParentAttr(PyObject *module, PyObject *module_name) { + PyObject *parent_module; + PyObject *child_name; + if (CPyImport_ImportParent(module_name, &parent_module, &child_name) < 0) { + return -1; + } + if (parent_module == NULL) { + return 0; + } + int result = PyObject_SetAttr(parent_module, child_name, module); + Py_DECREF(parent_module); + Py_DECREF(child_name); + return result; +} + +static int CPyImport_ReleaseLockPreservingException(PyObject *module_lock) { + PyObject *exc_type, *exc_val, *exc_tb; + PyErr_Fetch(&exc_type, &exc_val, &exc_tb); + int result = CPyImport_ReleaseLock(module_lock); + if (result < 0) { + PyErr_Clear(); + } + PyErr_Restore(exc_type, exc_val, exc_tb); + return result; +} + +// Execute a module once and publish completion; caller holds the module lock. +int CPyImport_Exec(PyObject *module, const char *module_name, + int (*exec_fn)(PyObject *), CPyImportState *state) { + if (CPyImport_IsInitialized(state)) { + return 0; + } + PyObject *name = PyUnicode_FromString(module_name); + if (name == NULL) { + return -1; + } + + int result = exec_fn(module); + if (result == 0) { + // Match CPython import semantics: publish parent.child only after the + // child module finished executing successfully. + result = CPyImport_SetParentAttr(module, name); + } + if (result == 0) { + CPyImport_SetInitialized(state, true); + } + Py_DECREF(name); + return result; +} + PyObject *CPyImport_ImportNative(PyObject *module_name, PyObject *(*init_only_fn)(void), int (*exec_fn)(PyObject *), CPyModule **module_static, + CPyImportState *state, CPyModuleLockAPI *lock_api, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package) { PyObject *parent_module = NULL; PyObject *child_name = NULL; PyObject *exc_type, *exc_val, *exc_tb; - Py_ssize_t name_len = PyUnicode_GetLength(module_name); - if (name_len < 0) { + // Import the parent package first to preserve import ordering semantics. + if (CPyImport_ImportParent(module_name, &parent_module, &child_name) < 0) { return NULL; } - Py_ssize_t dot = PyUnicode_FindChar(module_name, '.', 0, name_len, -1); - if (dot >= 0) { - // Import the parent package first to preserve import ordering semantics. - PyObject *parent_name = PyUnicode_Substring(module_name, 0, dot); - if (parent_name == NULL) { - CPyError_OutOfMemory(); - } - child_name = PyUnicode_Substring(module_name, dot + 1, name_len); - if (child_name == NULL) { - CPyError_OutOfMemory(); - } - parent_module = PyImport_Import(parent_name); - Py_DECREF(parent_name); - if (parent_module == NULL) { - Py_DECREF(child_name); - return NULL; - } - } + Py_XDECREF(parent_module); + Py_XDECREF(child_name); // Create the module object without executing the module body. // CPyInitOnly_* uses an internal static to cache the module object. // We then check sys.modules to determine whether the module body // has already been executed (or is being executed in a circular import). + PyObject *module_lock; + int lock_result = CPyImport_AcquireLock(lock_api, module_name, &module_lock); + if (lock_result == CPY_LOCK_DEADLOCK) { + PyObject *partial = PyDict_GetItemWithError(PyImport_GetModuleDict(), module_name); + if (partial != NULL && + (*module_static == NULL || partial == (PyObject *)*module_static)) { + Py_INCREF(partial); + return partial; + } + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_ImportError, + "import deadlock for native module '%U' without a partial module", + module_name); + } + return NULL; + } + if (lock_result == CPY_LOCK_ERROR) { + return NULL; + } + PyObject *module_dict = PyImport_GetModuleDict(); if (module_dict == NULL) { - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } @@ -1554,33 +1651,36 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, if (*module_static != NULL) { if (existing == (PyObject *)*module_static) { Py_INCREF(existing); - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + if (CPyImport_ReleaseLock(module_lock) < 0) { + Py_DECREF(existing); + existing = NULL; + } return existing; } PyErr_Format(PyExc_ImportError, "native module '%U' in sys.modules was replaced after initialization", module_name); - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } } if (PyErr_Occurred()) { - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } - PyObject *modobj = init_only_fn(); - if (modobj == NULL) { - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + if (CPyImport_IsInitialized(state)) { + PyErr_Format(PyExc_ImportError, + "initialized native module '%U' is missing from sys.modules", + module_name); + CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } - if (PyObject_SetItem(module_dict, module_name, modobj) < 0) { - goto fail; + PyObject *modobj = init_only_fn(); + if (modobj == NULL) { + CPyImport_ReleaseLockPreservingException(module_lock); + return NULL; } if (*module_static != (CPyModule *)modobj) { @@ -1594,31 +1694,41 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, goto fail; } + if (CPyImport_SetInitializing(modobj, true) < 0) { + goto fail; + } + + if (PyObject_SetItem(module_dict, module_name, modobj) < 0) { + goto fail; + } + // Now execute the module body, with __file__ and __package__ already set. if (exec_fn(modobj) != 0) { goto fail; } - // Match CPython import semantics: publish parent.child only after the - // child module finished executing successfully. - if (parent_module != NULL && PyObject_SetAttr(parent_module, child_name, modobj) < 0) { + if (CPyImport_SetInitializing(modobj, false) < 0) { goto fail; } - Py_XDECREF(parent_module); - Py_XDECREF(child_name); + if (CPyImport_ReleaseLock(module_lock) < 0) { + Py_DECREF(modobj); + return NULL; + } return modobj; fail: // Clean up on failure so that a subsequent import attempt will retry // initialization. PyErr_Fetch(&exc_type, &exc_val, &exc_tb); + CPyImport_SetInitializing(modobj, false); + PyErr_Clear(); PyObject_DelItem(module_dict, module_name); PyErr_Clear(); PyErr_Restore(exc_type, exc_val, exc_tb); - Py_XDECREF(parent_module); - Py_XDECREF(child_name); Py_CLEAR(*module_static); + CPyImport_SetInitialized(state, false); + CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 7b78b61f50e26..52da7ae512e7b 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -141,12 +141,15 @@ # Import a native same-group module directly via C-level init/exec functions. native_import_op = custom_op( # (module name, init-only function, exec function, module static, - # shared lib __file__, ext suffix, is_package) + # import state, compilation-unit lock, shared lib __file__, ext suffix, + # is_package) arg_types=[ str_rprimitive, c_pointer_rprimitive, c_pointer_rprimitive, object_pointer_rprimitive, + c_pointer_rprimitive, + c_pointer_rprimitive, object_rprimitive, str_rprimitive, c_pyssize_t_rprimitive, @@ -156,6 +159,13 @@ error_kind=ERR_MAGIC, ) +native_import_is_initialized_op = custom_op( + arg_types=[c_pointer_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyImport_IsInitialized", + error_kind=ERR_NEVER, +) + # Table-driven import op. import_many_op = custom_op( arg_types=[ From 730337405bee8dacf26563760f66d190e139cedc Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 21 Aug 2026 17:34:42 +0200 Subject: [PATCH 02/10] Add tests for concurrent imports --- mypyc/test-data/run-multimodule.test | 478 +++++++++++++++++++++++++++ 1 file changed, 478 insertions(+) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index fed4d3606ed90..a6a7219383551 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2048,3 +2048,481 @@ from mypy_extensions import mypyc_attr class CompiledBase: def value(self) -> int: raise NotImplementedError + +[case testCallFunctionInLazilyImportedModule] +import my_lib + +def get_version() -> str: + return my_lib.version() + +[file my_lib.py] +def version() -> str: + return "1.0.0" + +[file other.py] +def lazy_import() -> str: + import native + + return native.get_version() + +[file driver.py] +from threading import Barrier, Thread + +import other + +def test_lazy_import() -> None: + barrier = Barrier(2) + + def run_once() -> None: + barrier.wait() + assert other.lazy_import() == "1.0.0" + + threads = [] + for _ in range(2): + t = Thread(target = run_once) + threads.append(t) + t.start() + + for t in threads: + t.join() + +test_lazy_import() + +[case testConcurrentLazyNativeImportForms] +# separate: [(["native.py", "other_dependency.py", "other_importer.py"], "testgroup")] +from time import sleep + +sleep(0.05) +import other_dependency + +def get_value() -> int: + return other_dependency.value + +[file other_dependency.py] +value = 42 + +[file other_importer.py] +def by_module() -> int: + import native + return native.get_value() + +def by_from() -> int: + from native import get_value + return get_value() + +def by_from_alias() -> int: + from native import get_value as load_value + return load_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import other_importer + +functions = [ + other_importer.by_module, + other_importer.by_from, + other_importer.by_from_alias, +] +barrier = Barrier(len(functions)) + +def run(function): + barrier.wait() + return function() + +with ThreadPoolExecutor(max_workers=len(functions)) as executor: + futures = [executor.submit(run, function) for function in functions] + assert [future.result() for future in futures] == [42] * len(functions) + +[case testConcurrentLazyNativePackageImportForms] +# separate: [(["other_pkg/__init__.py", "other_pkg/other_target.py", "other_pkg/other_importer.py"], "testgroup")] +pass + +[file other_pkg/__init__.py] + +[file other_pkg/other_target.py] +from time import sleep + +sleep(0.05) +import py_dependency + +def get_value() -> int: + return py_dependency.value + +[file other_pkg/other_importer.py] +def by_dotted_module() -> int: + import other_pkg.other_target + return other_pkg.other_target.get_value() + +def by_dotted_alias() -> int: + import other_pkg.other_target as target + return target.get_value() + +def by_from_package() -> int: + from other_pkg import other_target + return other_target.get_value() + +def by_from_module() -> int: + from other_pkg.other_target import get_value + return get_value() + +def by_relative_package() -> int: + from . import other_target + return other_target.get_value() + +def by_relative_module() -> int: + from .other_target import get_value + return get_value() + +[file py_dependency.py] +value = 42 + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +from other_pkg import other_importer + +functions = [ + other_importer.by_dotted_module, + other_importer.by_dotted_alias, + other_importer.by_from_package, + other_importer.by_from_module, + other_importer.by_relative_package, + other_importer.by_relative_module, +] +barrier = Barrier(len(functions)) + +def run(function): + barrier.wait() + return function() + +with ThreadPoolExecutor(max_workers=len(functions)) as executor: + futures = [executor.submit(run, function) for function in functions] + assert [future.result() for future in futures] == [42] * len(functions) + +[case testConcurrentLazyNativeImportFromInterpretedModules] +from time import sleep + +sleep(0.05) +import py_dependency + +def get_value() -> int: + return py_dependency.value + +[file py_dependency.py] +value = 42 + +[file py_importer.py] +def by_module(): + import native + return native.get_value() + +def by_from(): + from native import get_value + return get_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import py_importer + +functions = [py_importer.by_module, py_importer.by_from] +barrier = Barrier(len(functions)) + +def run(function): + barrier.wait() + return function() + +with ThreadPoolExecutor(max_workers=len(functions)) as executor: + futures = [executor.submit(run, function) for function in functions] + assert [future.result() for future in futures] == [42] * len(functions) + +[case testConcurrentLazyNativeImportFromNativeAndInterpretedModules] +# separate: [(["native.py", "other_importer.py"], "testgroup")] +from time import sleep + +sleep(0.05) +import py_dependency + +def get_value() -> int: + return py_dependency.value + +[file other_importer.py] +def load() -> int: + import native + return native.get_value() + +[file py_dependency.py] +value = 42 + +[file py_importer.py] +def load(): + from native import get_value + return get_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import other_importer +import py_importer + +functions = [other_importer.load, py_importer.load] +barrier = Barrier(len(functions)) + +def run(function): + barrier.wait() + return function() + +with ThreadPoolExecutor(max_workers=len(functions)) as executor: + futures = [executor.submit(run, function) for function in functions] + assert [future.result() for future in futures] == [42] * len(functions) + +[case testConcurrentLazyNativeImportAcrossCompilationGroups] +# separate: [(["native.py", "other_import_a.py"], "target"), (["other_import_b.py"], "importer_b")] +from time import sleep + +sleep(0.05) + +def get_value() -> int: + return 42 + +[file other_import_a.py] +def load() -> int: + import native + return native.get_value() + +[file other_import_b.py] +def load() -> int: + from native import get_value + return get_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import other_import_a +import other_import_b + +functions = [other_import_a.load, other_import_b.load] +barrier = Barrier(len(functions)) + +def run(function): + barrier.wait() + return function() + +with ThreadPoolExecutor(max_workers=len(functions)) as executor: + futures = [executor.submit(run, function) for function in functions] + assert [future.result() for future in futures] == [42] * len(functions) + +[case testRegularImportWaitsForDirectNativeImport] +# separate: [(["native.py", "other_importer.py"], "testgroup")] +from time import sleep + +import import_sync + +import_sync.direct_started.set() +assert import_sync.regular_started.wait(timeout=5) +sleep(0.05) + +def get_value() -> int: + return 42 + +[file import_sync.py] +from threading import Event + +direct_started = Event() +regular_started = Event() + +[file other_importer.py] +def load() -> int: + import native + return native.get_value() + +[file py_importer.py] +import import_sync + +def load(): + import_sync.regular_started.set() + import native + return native.get_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor + +import import_sync +import other_importer +import py_importer + +with ThreadPoolExecutor(max_workers=2) as executor: + direct_future = executor.submit(other_importer.load) + assert import_sync.direct_started.wait(timeout=5) + regular_future = executor.submit(py_importer.load) + assert direct_future.result(timeout=10) == 42 + assert regular_future.result(timeout=10) == 42 + +[case testConcurrentFailedLazyNativeImportRetries] +# separate: [(["native.py", "other_importer.py"], "testgroup")] +from time import sleep + +import failure_state + +failure_state.attempts += 1 +sleep(0.05) +if failure_state.attempts == 1: + raise RuntimeError("first initialization failed") + +def get_value() -> int: + return 42 + +[file failure_state.py] +attempts = 0 + +[file other_importer.py] +def load() -> int: + from native import get_value + return get_value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier + +import failure_state +import other_importer + +barrier = Barrier(2) + +def run(): + barrier.wait() + return other_importer.load() + +with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(run) for _ in range(2)] + +values = [] +errors = [] +for future in futures: + try: + values.append(future.result()) + except RuntimeError as error: + errors.append(str(error)) + +assert values == [42] +assert errors == ["first initialization failed"] +assert failure_state.attempts == 2 +assert other_importer.load() == 42 + +[case testConcurrentCircularNativeImports] +# separate: [(["other_a.py", "other_b.py"], "testgroup")] +pass + +[file import_sync.py] +from threading import Barrier + +barrier = Barrier(2) + +[file other_a.py] +from import_sync import barrier + +def value() -> str: + return "a" + +barrier.wait(timeout=5) +import other_b +other_value = other_b.value() + +[file other_b.py] +from import_sync import barrier + +def value() -> str: + return "b" + +barrier.wait(timeout=5) +import other_a +other_value = other_a.value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +import importlib + +with ThreadPoolExecutor(max_workers=2) as executor: + future_a = executor.submit(importlib.import_module, "other_a") + future_b = executor.submit(importlib.import_module, "other_b") + other_a = future_a.result(timeout=10) + other_b = future_b.result(timeout=10) + +assert other_a.other_value == "b" +assert other_b.other_value == "a" + +[case testTopLevelThreadImportsNativeModuleFromSameGroup] +# separate: [(["native.py", "other_target.py"], "testgroup")] +from threading import Thread + +results: list[int] = [] + +def load_target() -> None: + import other_target + results.append(other_target.value) + +thread = Thread(target=load_target) +thread.start() +thread.join(timeout=5) + +assert not thread.is_alive(), "native import blocked on its compilation unit" +assert results == [42] + +[file other_target.py] +value = 42 + +[file driver.py] +import native + +assert native.results == [42] + +[case testCallFunctionInLazilyImportedModuleThroughAny] +[file my_lib.py] +def version() -> str: + return "1.0.0" + +[file other_pkg/__init__.py] + +[file other_pkg/other_callmylib.py] +import my_lib + +def get_version() -> str: + return my_lib.version() + +[file other_pkg/other_import.py] +from typing import Any + +def lazy_import() -> str: + import other_pkg.other_callmylib + + pkg: Any = other_pkg + return pkg.other_callmylib.get_version() + +[file driver.py] +from threading import Barrier, Thread + +import other_pkg.other_import + +def test_lazy_import() -> None: + barrier = Barrier(2) + + def run_once() -> None: + barrier.wait() + assert other_pkg.other_import.lazy_import() == "1.0.0" + + threads = [] + for _ in range(2): + t = Thread(target = run_once) + threads.append(t) + t.start() + + for t in threads: + t.join() + +test_lazy_import() From e1d2e4a0ca67cb93d07bb4cb5ab00cf8629a0e0f Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Fri, 21 Aug 2026 17:35:47 +0200 Subject: [PATCH 03/10] Remove workaround --- mypy/nativeparse.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index d7e660b470ffb..3a8796cf03d8c 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -19,7 +19,6 @@ from __future__ import annotations import os -import time from typing import Final, cast import ast_serialize @@ -280,13 +279,6 @@ def parse_to_binary_ast( source: str | bytes | None = None, skip_function_bodies: bool = False, ) -> tuple[bytes, list[ParseError], TypeIgnores, bytes, bool, bool, str, list[tuple[int, str]]]: - # This is a horrible hack to work around a mypyc bug where imported - # module may be not ready in a thread sometimes. - t0 = time.time() - while ast_serialize is None: - time.sleep(0.0001) # type: ignore[unreachable] - if time.time() - t0 > 10.0: - raise ImportError("Cannot import ast_serialize") ast_bytes, errors, ignores, import_bytes, ast_data = ast_serialize.parse( filename, source, From 1b5650fa00bc63189fd8ac363628041a770d6d2c Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 24 Aug 2026 16:14:01 +0200 Subject: [PATCH 04/10] Set _initializing in module init function --- mypyc/codegen/emitmodule.py | 16 +++++++++++++--- mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/misc_ops.c | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 64e53fcac90bd..2442624f5c430 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1489,15 +1489,19 @@ def emit_module_init_func( emitter.emit_line("Py_DECREF(shared_lib_file);") emitter.emit_line("if (rv < 0) goto fail;") - # Register in sys.modules early so that circular imports via - # CPyImport_ImportNative can detect that this module is already - # being initialized and avoid re-executing the module body. + # Mark the module as initializing before publishing it so that CPython's + # import fast path waits on the module lock. Publishing early also lets + # CPyImport_ImportNative detect circular imports. + emitter.emit_line(f"if (CPyImport_SetInitializing({module_static}, 1) < 0)") + emitter.emit_line(" goto fail;") emitter.emit_line( f"if (PyObject_SetItem(PyImport_GetModuleDict(), modname, {module_static}) < 0)" ) emitter.emit_line(" goto fail;") emitter.emit_line("Py_CLEAR(modname);") emitter.emit_lines(f"if ({exec_func}({module_static}) != 0)", " goto fail;") + emitter.emit_line(f"if (CPyImport_SetInitializing({module_static}, 0) < 0)") + emitter.emit_line(" goto fail;") emitter.emit_line(f"return {module_static};") emitter.emit_lines("fail:") # Clean up on failure: remove from sys.modules and clear the static @@ -1505,6 +1509,12 @@ def emit_module_init_func( emitter.emit_line("{") emitter.emit_line(" PyObject *exc_type, *exc_val, *exc_tb;") emitter.emit_line(" PyErr_Fetch(&exc_type, &exc_val, &exc_tb);") + emitter.emit_line(f" if ({module_static} != NULL) {{") + emitter.emit_line(f" CPyImport_SetInitializing({module_static}, 0);") + emitter.emit_line(" PyErr_Clear();") + emitter.emit_line(" }") + state = self.import_state_name(module_name) + emitter.emit_line(f" CPyImport_SetInitialized(&{state}, 0);") emitter.emit_line(" if (modname == NULL) {") emitter.emit_line(f' modname = PyUnicode_FromString("{module_name}");') emitter.emit_line(" if (modname == NULL) CPyError_OutOfMemory();") diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 186cdce817349..73603a6c2420d 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -1071,6 +1071,7 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, Py_ssize_t is_package); int CPyImport_Exec(PyObject *module, const char *module_name, int (*exec_fn)(PyObject *), CPyImportState *state); +int CPyImport_SetInitializing(PyObject *module, bool initializing); int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index f1880b141f64b..03a8906bd3534 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1507,7 +1507,7 @@ static int CPyImport_SetModuleSpec(PyObject *modobj, PyObject *module_name, } // Set module.__spec__._initializing for CPython's import machinery. -static int CPyImport_SetInitializing(PyObject *module, bool initializing) { +int CPyImport_SetInitializing(PyObject *module, bool initializing) { PyObject *spec = PyObject_GetAttrString(module, "__spec__"); if (spec == NULL) { return -1; From 1bc3c28662d0639c91bf087cd3434663db81fc87 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 26 Aug 2026 16:23:16 +0200 Subject: [PATCH 05/10] Intern strings in shared lib init --- mypyc/codegen/emitmodule.py | 4 +++- mypyc/lib-rt/strings/librt_strings.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 2442624f5c430..688b90df66f4e 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -965,6 +965,7 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None: f"{lock_api} = CPyModuleLockAPI_Alloc();", f"if ({lock_api} == NULL) goto fail;", "}", + "if (intern_strings() < 0) goto fail;", "if (CPyGlobalsInit() < 0) goto fail;", "", ) @@ -1290,7 +1291,8 @@ def emit_module_exec_func( impl_name = f"{exec_name}__impl" module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_lines(f"static int {impl_name}(PyObject *module)", "{") - emitter.emit_line("intern_strings();") + if not self.use_shared_lib: + emitter.emit_lines("if (intern_strings() < 0)", " return -1;") if self.compiler_options.depends_on_librt_internal: emitter.emit_line("if (import_librt_internal() < 0) {") emitter.emit_line("return -1;") diff --git a/mypyc/lib-rt/strings/librt_strings.c b/mypyc/lib-rt/strings/librt_strings.c index b79ffcd9f7c89..153ef54454f36 100644 --- a/mypyc/lib-rt/strings/librt_strings.c +++ b/mypyc/lib-rt/strings/librt_strings.c @@ -1358,6 +1358,8 @@ static PyModuleDef librt_strings_module = { PyMODINIT_FUNC PyInit_strings(void) { - intern_strings(); + if (intern_strings() < 0) { + return NULL; + } return PyModuleDef_Init(&librt_strings_module); } From c6755e6f5c6f86477caefa7504d2df5ef91650f1 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 26 Aug 2026 16:43:02 +0200 Subject: [PATCH 06/10] Publish module cache using CAS --- mypyc/irbuild/builder.py | 18 +++++++----- mypyc/irbuild/statement.py | 6 ++-- mypyc/lib-rt/CPy.h | 2 ++ mypyc/lib-rt/locks.c | 25 +++++++++++++++++ mypyc/lib-rt/misc_ops.c | 5 ++-- mypyc/primitives/misc_ops.py | 15 ++++++++++ mypyc/test-data/run-multimodule.test | 42 ++++++++++++++++++++++++++++ 7 files changed, 102 insertions(+), 11 deletions(-) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 2f75b393acaa8..6bfa58b843292 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -85,7 +85,6 @@ from mypyc.ir.class_ir import ClassIR, NonExtClassInfo from mypyc.ir.func_ir import INVALID_FUNC_DEF, FuncDecl, FuncIR, FuncSignature, RuntimeArg from mypyc.ir.ops import ( - NAMESPACE_MODULE, NAMESPACE_TYPE_VAR, NO_TRACEBACK_LINE_NO, Assign, @@ -169,6 +168,8 @@ from mypyc.primitives.misc_ops import ( check_unpack_count_op, get_module_dict_op, + import_cache_get_op, + import_cache_set_op, import_op, native_import_is_initialized_op, native_import_op, @@ -526,6 +527,9 @@ def gen_import(self, module: str, line: int) -> None: self.imports[module] = None needs_import, out = BasicBlock(), BasicBlock() + module_cache = self.add( + LoadAddress(object_pointer_rprimitive, f"{MODULE_PREFIX}{exported_name(module)}") + ) is_native_module = self.is_native_module(module) is_same_group_native = is_native_module and self.is_same_group_module(module) import_state: Value | None = None @@ -550,7 +554,7 @@ def gen_import(self, module: str, line: int) -> None: # path so that its per-module lock waits for initialization. self.goto(needs_import) else: - self.check_if_module_loaded(module, line, needs_import, out, import_state) + self.check_if_module_loaded(module_cache, line, needs_import, out, import_state) self.activate_block(needs_import) if is_same_group_native: @@ -599,25 +603,25 @@ def gen_import(self, module: str, line: int) -> None: else: # Import using generic Python C API value = self.call_c(import_op, [self.load_str(module, line)], line) - self.add(InitStatic(value, module, namespace=NAMESPACE_MODULE)) + self.call_c(import_cache_set_op, [module_cache, value], line) self.goto_and_activate(out) def check_if_module_loaded( self, - id: str, + module_cache: Value, line: int, needs_import: BasicBlock, out: BasicBlock, import_state: Value | None = None, ) -> None: - """Generate code that checks if the module `id` has been loaded yet. + """Generate code that checks if a module cache has been populated. Arguments: - id: name of module to check if imported + module_cache: address of the module cache to check line: line number that the import occurs on needs_import: the BasicBlock that is run if the module has not been loaded yet out: the BasicBlock that is run if the module has already been loaded""" - first_load = self.load_module(id) + first_load = self.call_c(import_cache_get_op, [module_cache], line) comparison = self.translate_is_op(first_load, self.none_object(line), "is not", line) if import_state is None: self.add_bool_branch(comparison, out, needs_import) diff --git a/mypyc/irbuild/statement.py b/mypyc/irbuild/statement.py index 21f47b190c302..921ed460eda05 100644 --- a/mypyc/irbuild/statement.py +++ b/mypyc/irbuild/statement.py @@ -58,7 +58,6 @@ BasicBlock, Branch, Call, - InitStatic, Integer, LoadAddress, LoadErrorValue, @@ -127,6 +126,7 @@ check_stop_op, coro_op, get_native_attrs_op, + import_cache_set_op, import_from_many_op, import_many_op, import_op, @@ -597,7 +597,9 @@ def transform_import_from_buckets( line, ) if module is not None: - builder.add(InitStatic(module, module_id, namespace=NAMESPACE_MODULE)) + mod_static = LoadStatic(object_rprimitive, module_id, namespace=NAMESPACE_MODULE) + module_cache = builder.add(LoadAddress(object_pointer_rprimitive, mod_static)) + builder.call_c(import_cache_set_op, [module_cache, module], line) def transform_import_all(builder: IRBuilder, node: ImportAll) -> None: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 73603a6c2420d..bd4b6203f24bc 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -43,6 +43,8 @@ int CPyImport_AcquireLock(CPyModuleLockAPI *api, PyObject *module_name, int CPyImport_ReleaseLock(PyObject *module_lock); bool CPyImport_IsInitialized(const CPyImportState *state); void CPyImport_SetInitialized(CPyImportState *state, bool initialized); +PyObject *CPyImport_GetModuleCache(CPyModule **cache); +void CPyImport_SetModuleCache(CPyModule **cache, PyObject *module); // Naming conventions: diff --git a/mypyc/lib-rt/locks.c b/mypyc/lib-rt/locks.c index 16ded3aa983fc..3350439f563ff 100644 --- a/mypyc/lib-rt/locks.c +++ b/mypyc/lib-rt/locks.c @@ -96,3 +96,28 @@ void CPyImport_SetInitialized(CPyImportState *state, bool initialized) { __atomic_store_n(&state->initialized, initialized, __ATOMIC_RELEASE); #endif } + +PyObject *CPyImport_GetModuleCache(CPyModule **cache) { +#ifdef _WIN32 + return InterlockedCompareExchangePointer((PVOID volatile *)cache, NULL, NULL); +#else + return (PyObject *)__atomic_load_n(cache, __ATOMIC_ACQUIRE); +#endif +} + +void CPyImport_SetModuleCache(CPyModule **cache, PyObject *module) { + Py_INCREF(module); +#ifdef _WIN32 + PVOID previous = InterlockedCompareExchangePointer( + (PVOID volatile *)cache, module, Py_None); + if (previous != Py_None) { + Py_DECREF(module); + } +#else + CPyModule *expected = (CPyModule *)Py_None; + if (!__atomic_compare_exchange_n(cache, &expected, (CPyModule *)module, false, + __ATOMIC_RELEASE, __ATOMIC_ACQUIRE)) { + Py_DECREF(module); + } +#endif +} diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 03a8906bd3534..28d2c334c7b46 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -772,12 +772,13 @@ CPy_Super(PyObject *builtins, PyObject *self) { static bool import_single(PyObject *mod_id, PyObject **mod_static, PyObject *globals_id, PyObject *globals_name, PyObject *globals) { - if (Py_IsNone(*mod_static)) { + if (Py_IsNone(CPyImport_GetModuleCache((CPyModule **)mod_static))) { CPyModule *mod = PyImport_Import(mod_id); if (mod == NULL) { return false; } - *mod_static = mod; + CPyImport_SetModuleCache((CPyModule **)mod_static, mod); + Py_DECREF(mod); } PyObject *mod_dict = PyImport_GetModuleDict(); diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 52da7ae512e7b..913164bad100d 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -166,6 +166,21 @@ error_kind=ERR_NEVER, ) +import_cache_get_op = custom_op( + arg_types=[object_pointer_rprimitive], + return_type=object_rprimitive, + c_function_name="CPyImport_GetModuleCache", + error_kind=ERR_NEVER, + is_borrowed=True, +) + +import_cache_set_op = custom_op( + arg_types=[object_pointer_rprimitive, object_rprimitive], + return_type=void_rtype, + c_function_name="CPyImport_SetModuleCache", + error_kind=ERR_NEVER, +) + # Table-driven import op. import_many_op = custom_op( arg_types=[ diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index a6a7219383551..9b865628cd5f1 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2526,3 +2526,45 @@ def test_lazy_import() -> None: t.join() test_lazy_import() + +[case testConcurrentNativeImportDoesNotLeakCacheReference] +# separate: [(["native.py", "other_target.py", "other_importer.py"], "testgroup")] +from time import sleep + +sleep(0.05) + +def value() -> int: + return 42 + +[file other_target.py] +def value() -> int: + return 42 + +[file other_importer.py] +def load_native() -> int: + import native + return native.value() + +def load_target() -> int: + import other_target + return other_target.value() + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +from sys import getrefcount, modules +from threading import Barrier + +import other_importer + +barrier = Barrier(2) + +def load_native() -> int: + barrier.wait() + return other_importer.load_native() + +with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(load_native) for _ in range(2)] + assert [future.result() for future in futures] == [42, 42] + +assert other_importer.load_target() == 42 +assert getrefcount(modules["native"]) == getrefcount(modules["other_target"]) From ab789259d27db5c5804c7675c269e306bbdba76e Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 26 Aug 2026 17:01:46 +0200 Subject: [PATCH 07/10] Use same spec object to mark _initializing = False --- mypyc/codegen/emitmodule.py | 12 +++-- mypyc/lib-rt/CPy.h | 3 +- mypyc/lib-rt/misc_ops.c | 33 +++++++++---- mypyc/test-data/run-multimodule.test | 70 ++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 14 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 688b90df66f4e..b39a637114a48 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1446,6 +1446,7 @@ def emit_module_init_func( module_static = self.module_internal_static_name(module_name, emitter) emitter.emit_line("PyObject* modname = NULL;") + emitter.emit_line("PyObject *initializing_spec = NULL;") emitter.emit_lines( f"if ({module_static}) {{", f"Py_INCREF({module_static});", @@ -1494,7 +1495,8 @@ def emit_module_init_func( # Mark the module as initializing before publishing it so that CPython's # import fast path waits on the module lock. Publishing early also lets # CPyImport_ImportNative detect circular imports. - emitter.emit_line(f"if (CPyImport_SetInitializing({module_static}, 1) < 0)") + emitter.emit_line(f"initializing_spec = CPyImport_BeginInitializing({module_static});") + emitter.emit_line("if (initializing_spec == NULL)") emitter.emit_line(" goto fail;") emitter.emit_line( f"if (PyObject_SetItem(PyImport_GetModuleDict(), modname, {module_static}) < 0)" @@ -1502,7 +1504,9 @@ def emit_module_init_func( emitter.emit_line(" goto fail;") emitter.emit_line("Py_CLEAR(modname);") emitter.emit_lines(f"if ({exec_func}({module_static}) != 0)", " goto fail;") - emitter.emit_line(f"if (CPyImport_SetInitializing({module_static}, 0) < 0)") + emitter.emit_line("rv = CPyImport_EndInitializing(initializing_spec);") + emitter.emit_line("initializing_spec = NULL;") + emitter.emit_line("if (rv < 0)") emitter.emit_line(" goto fail;") emitter.emit_line(f"return {module_static};") emitter.emit_lines("fail:") @@ -1511,8 +1515,8 @@ def emit_module_init_func( emitter.emit_line("{") emitter.emit_line(" PyObject *exc_type, *exc_val, *exc_tb;") emitter.emit_line(" PyErr_Fetch(&exc_type, &exc_val, &exc_tb);") - emitter.emit_line(f" if ({module_static} != NULL) {{") - emitter.emit_line(f" CPyImport_SetInitializing({module_static}, 0);") + emitter.emit_line(" if (initializing_spec != NULL) {") + emitter.emit_line(" CPyImport_EndInitializing(initializing_spec);") emitter.emit_line(" PyErr_Clear();") emitter.emit_line(" }") state = self.import_state_name(module_name) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index bd4b6203f24bc..1bc7d6e782747 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -1073,7 +1073,8 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, Py_ssize_t is_package); int CPyImport_Exec(PyObject *module, const char *module_name, int (*exec_fn)(PyObject *), CPyImportState *state); -int CPyImport_SetInitializing(PyObject *module, bool initializing); +PyObject *CPyImport_BeginInitializing(PyObject *module); +int CPyImport_EndInitializing(PyObject *spec); int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 28d2c334c7b46..ac7055fdaafd5 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1507,14 +1507,22 @@ static int CPyImport_SetModuleSpec(PyObject *modobj, PyObject *module_name, return 0; } -// Set module.__spec__._initializing for CPython's import machinery. -int CPyImport_SetInitializing(PyObject *module, bool initializing) { +// Mark module's current spec as initializing and return an owned reference. +PyObject *CPyImport_BeginInitializing(PyObject *module) { PyObject *spec = PyObject_GetAttrString(module, "__spec__"); if (spec == NULL) { - return -1; + return NULL; + } + if (PyObject_SetAttrString(spec, "_initializing", Py_True) < 0) { + Py_DECREF(spec); + return NULL; } - int result = PyObject_SetAttrString(spec, "_initializing", - initializing ? Py_True : Py_False); + return spec; +} + +// Clear initializing on spec and consume its reference. +int CPyImport_EndInitializing(PyObject *spec) { + int result = PyObject_SetAttrString(spec, "_initializing", Py_False); Py_DECREF(spec); return result; } @@ -1609,7 +1617,9 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, Py_ssize_t is_package) { PyObject *parent_module = NULL; PyObject *child_name = NULL; + PyObject *initializing_spec = NULL; PyObject *exc_type, *exc_val, *exc_tb; + int end_result; // Import the parent package first to preserve import ordering semantics. if (CPyImport_ImportParent(module_name, &parent_module, &child_name) < 0) { return NULL; @@ -1695,7 +1705,8 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, goto fail; } - if (CPyImport_SetInitializing(modobj, true) < 0) { + initializing_spec = CPyImport_BeginInitializing(modobj); + if (initializing_spec == NULL) { goto fail; } @@ -1708,7 +1719,9 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, goto fail; } - if (CPyImport_SetInitializing(modobj, false) < 0) { + end_result = CPyImport_EndInitializing(initializing_spec); + initializing_spec = NULL; + if (end_result < 0) { goto fail; } @@ -1722,8 +1735,10 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, // Clean up on failure so that a subsequent import attempt will retry // initialization. PyErr_Fetch(&exc_type, &exc_val, &exc_tb); - CPyImport_SetInitializing(modobj, false); - PyErr_Clear(); + if (initializing_spec != NULL) { + CPyImport_EndInitializing(initializing_spec); + PyErr_Clear(); + } PyObject_DelItem(module_dict, module_name); PyErr_Clear(); PyErr_Restore(exc_type, exc_val, exc_tb); diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 9b865628cd5f1..4a710bf8cbcf1 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2568,3 +2568,73 @@ with ThreadPoolExecutor(max_workers=2) as executor: assert other_importer.load_target() == 42 assert getrefcount(modules["native"]) == getrefcount(modules["other_target"]) + +[case testNativeModuleCanReplaceSpecDuringRegularImport] +assert __spec__ is not None +body_spec = __spec__ +__spec__ = None +value = 42 + +[file driver.py] +import native + +assert native.value == 42 +assert native.body_spec._initializing is False + +[case testNativeModuleCanReplaceSpecDuringDirectImport] +# separate: [(["native.py", "other_importer.py"], "testgroup")] +assert __spec__ is not None +body_spec = __spec__ +__spec__ = None +value = 42 + +[file other_importer.py] +def load() -> int: + import native + return native.value + +[file driver.py] +import other_importer + +assert other_importer.load() == 42 +assert __import__("native").__spec__ is None +assert __import__("native").body_spec._initializing is False + +[case testNativeModuleReplacingSpecClearsOriginalSpecOnFailure] +# separate: [(["native.py", "other_importer.py"], "testgroup")] +import spec_state + +assert __spec__ is not None +spec_state.specs.append(__spec__) +__spec__ = None +raise RuntimeError("initialization failed") + +[file spec_state.py] +from typing import Any + +specs: list[Any] = [] + +[file other_importer.py] +def load() -> None: + import native + +[file driver.py] +import sys + +import other_importer +import spec_state + +from testutil import assertRaises + +with assertRaises(RuntimeError, "initialization failed"): + other_importer.load() + +assert spec_state.specs[0]._initializing is False +assert "native" not in sys.modules + +with assertRaises(RuntimeError, "initialization failed"): + import native + +assert len(spec_state.specs) == 2 +assert spec_state.specs[1]._initializing is False +assert "native" not in sys.modules From ed113898851a22f49d0dd7b0c5f802bf72a1f5c8 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Wed, 26 Aug 2026 17:12:49 +0200 Subject: [PATCH 08/10] Publish child module only in native-to-native imports --- mypyc/codegen/emitmodule.py | 5 +- mypyc/lib-rt/CPy.h | 3 +- mypyc/lib-rt/misc_ops.c | 21 ++--- mypyc/test-data/run-multimodule.test | 110 +++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 19 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index b39a637114a48..70f3cc351bc09 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1387,10 +1387,7 @@ def emit_module_exec_func( state = self.import_state_name(module_name) emitter.emit_lines( - declaration, - "{", - f'return CPyImport_Exec(module, "{module_name}", {impl_name}, &{state});', - "}", + declaration, "{", f"return CPyImport_Exec(module, {impl_name}, &{state});", "}" ) def emit_init_only_func(self, emitter: Emitter, module_name: str, module_prefix: str) -> None: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 1bc7d6e782747..c98d9ac7d9440 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -1071,8 +1071,7 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, CPyImportState *state, CPyModuleLockAPI *lock_api, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); -int CPyImport_Exec(PyObject *module, const char *module_name, - int (*exec_fn)(PyObject *), CPyImportState *state); +int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state); PyObject *CPyImport_BeginInitializing(PyObject *module); int CPyImport_EndInitializing(PyObject *spec); int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index ac7055fdaafd5..4a7c30f4b3102 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1584,27 +1584,16 @@ static int CPyImport_ReleaseLockPreservingException(PyObject *module_lock) { return result; } -// Execute a module once and publish completion; caller holds the module lock. -int CPyImport_Exec(PyObject *module, const char *module_name, - int (*exec_fn)(PyObject *), CPyImportState *state) { +// Execute a module once; caller holds the module lock. +int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state) { if (CPyImport_IsInitialized(state)) { return 0; } - PyObject *name = PyUnicode_FromString(module_name); - if (name == NULL) { - return -1; - } int result = exec_fn(module); - if (result == 0) { - // Match CPython import semantics: publish parent.child only after the - // child module finished executing successfully. - result = CPyImport_SetParentAttr(module, name); - } if (result == 0) { CPyImport_SetInitialized(state, true); } - Py_DECREF(name); return result; } @@ -1725,6 +1714,12 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, goto fail; } + // Direct imports must publish parent.child themselves; normal extension + // loading leaves this to importlib. + if (CPyImport_SetParentAttr(modobj, module_name) < 0) { + goto fail; + } + if (CPyImport_ReleaseLock(module_lock) < 0) { Py_DECREF(modobj); return NULL; diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 4a710bf8cbcf1..574e7f847af52 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2638,3 +2638,113 @@ with assertRaises(RuntimeError, "initialization failed"): assert len(spec_state.specs) == 2 assert spec_state.specs[1]._initializing is False assert "native" not in sys.modules + +[case testNormalNativeImportPublishesParentAttributeOnce] +pass + +[file pkg/__init__.py] +from sys import modules +from types import ModuleType +from typing import Any + +child_setattr_calls = 0 + +class TrackingModule(ModuleType): + def __setattr__(self, name: str, value: Any) -> None: + if name == "other_child": + self.child_setattr_calls += 1 + ModuleType.__setattr__(self, name, value) + +modules[__name__].__class__ = TrackingModule + +[file pkg/other_child.py] +value = 42 + +[file driver.py] +import importlib +import pkg + +child = importlib.import_module("pkg.other_child") + +assert child.value == 42 +assert pkg.other_child is child +assert pkg.child_setattr_calls == 1 + +[file sys/__init__.pyi] +modules: dict[str, object] + +[case testNormalNativeImportWarnsWhenParentRejectsAttribute] +pass + +[file pkg/__init__.py] +from sys import modules +from types import ModuleType +from typing import Any + +class RejectingModule(ModuleType): + def __setattr__(self, name: str, value: Any) -> None: + if name == "other_child": + raise AttributeError("child rejected") + ModuleType.__setattr__(self, name, value) + +modules[__name__].__class__ = RejectingModule + +[file pkg/other_child.py] +value = 42 + +[file driver.py] +import importlib +import sys +import warnings + +import pkg + +with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + child = importlib.import_module("pkg.other_child") + +assert child.value == 42 +assert sys.modules["pkg.other_child"] is child +assert not hasattr(pkg, "other_child") +assert len(caught) == 1 +assert caught[0].category is ImportWarning + +[file sys/__init__.pyi] +modules: dict[str, object] + +[case testDirectNativeImportPublishesParentAttributeOnce] +# separate: [(["native.py", "pkg/other_child.py"], "testgroup")] +def load() -> int: + import pkg.other_child + return pkg.other_child.value + +[file pkg/__init__.py] +from sys import modules +from types import ModuleType +from typing import Any + +child_setattr_calls = 0 + +class TrackingModule(ModuleType): + def __setattr__(self, name: str, value: Any) -> None: + if name == "other_child": + self.child_setattr_calls += 1 + ModuleType.__setattr__(self, name, value) + +modules[__name__].__class__ = TrackingModule + +[file pkg/other_child.py] +value = 42 + +[file driver.py] +import sys + +import native +import pkg + +assert native.load() == 42 +assert pkg.other_child is sys.modules["pkg.other_child"] +assert pkg.child_setattr_calls == 1 + +[file sys/__init__.pyi] +modules: dict[str, object] From dafc3f6b967d67e1a0feac1f189d84c9ba35a811 Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Thu, 27 Aug 2026 15:07:53 +0200 Subject: [PATCH 09/10] Update IR build tests --- mypyc/test-data/irbuild-basic.test | 925 ++++++++++++++------------- mypyc/test-data/irbuild-classes.test | 829 ++++++++++++------------ 2 files changed, 904 insertions(+), 850 deletions(-) diff --git a/mypyc/test-data/irbuild-basic.test b/mypyc/test-data/irbuild-basic.test index c6c231f0386be..c0e3bd4cd6deb 100644 --- a/mypyc/test-data/irbuild-basic.test +++ b/mypyc/test-data/irbuild-basic.test @@ -516,78 +516,81 @@ def hello() -> None: [out] def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4 :: object - r5, r6, r7, r8 :: object_ptr - r9 :: object_ptr[4] - r10 :: c_ptr - r11 :: native_int[4] - r12 :: c_ptr - r13 :: object - r14 :: dict - r15, r16 :: str - r17 :: bit - r18 :: str - r19 :: dict - r20 :: str - r21 :: i32 - r22 :: bit - r23 :: object_ptr - r24 :: object_ptr[1] - r25 :: c_ptr - r26 :: native_int[1] - r27 :: c_ptr - r28 :: object - r29 :: dict - r30, r31 :: str - r32 :: bit - r33 :: object - r34 :: str - r35, r36 :: object -L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5 :: object + r6, r7, r8, r9 :: object_ptr + r10 :: object_ptr[4] + r11 :: c_ptr + r12 :: native_int[4] + r13 :: c_ptr + r14 :: object + r15 :: dict + r16, r17 :: str + r18 :: bit + r19 :: str + r20 :: dict + r21 :: str + r22 :: i32 + r23 :: bit + r24 :: object_ptr + r25 :: object_ptr[1] + r26 :: c_ptr + r27 :: native_int[1] + r28 :: c_ptr + r29 :: object + r30 :: dict + r31, r32 :: str + r33 :: bit + r34 :: object + r35 :: str + r36, r37 :: object +L0: + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = load_address sys :: module - r6 = load_address enum :: module - r7 = load_address collections.abc :: module + r6 = load_address sys :: module + r7 = load_address enum :: module r8 = load_address collections.abc :: module - r9 = [r5, r6, r7, r8] - r10 = load_address r9 - r11 = [1, 2, 3, 4] - r12 = load_address r11 - r13 = (('sys', 'sys', 'sys'), ('enum', 'enum', 'enum2'), ('collections.abc', 'collections', 'collections'), ('collections.abc', 'collections.abc', 'abc2')) - r14 = __main__.globals :: static - r15 = 'main' - r16 = '' - r17 = CPyImport_ImportMany(r13, r10, r14, r15, r16, r12) - r18 = 'filler' - r19 = __main__.globals :: static - r20 = '_' - r21 = CPyDict_SetItem(r19, r20, r18) - r22 = r21 >= 0 :: signed - r23 = load_address single :: module - r24 = [r23] - r25 = load_address r24 - r26 = [6] - r27 = load_address r26 - r28 = (('single', 'single', 'single'),) - r29 = __main__.globals :: static - r30 = 'main' - r31 = '' - r32 = CPyImport_ImportMany(r28, r25, r29, r30, r31, r27) - r33 = single :: module - r34 = 'hello' - r35 = CPyObject_GetAttr(r33, r34) - r36 = PyObject_Vectorcall(r35, 0, 0, 0) + r9 = load_address collections.abc :: module + r10 = [r6, r7, r8, r9] + r11 = load_address r10 + r12 = [1, 2, 3, 4] + r13 = load_address r12 + r14 = (('sys', 'sys', 'sys'), ('enum', 'enum', 'enum2'), ('collections.abc', 'collections', 'collections'), ('collections.abc', 'collections.abc', 'abc2')) + r15 = __main__.globals :: static + r16 = 'main' + r17 = '' + r18 = CPyImport_ImportMany(r14, r11, r15, r16, r17, r13) + r19 = 'filler' + keep_alive r0 + r20 = __main__.globals :: static + r21 = '_' + r22 = CPyDict_SetItem(r20, r21, r19) + r23 = r22 >= 0 :: signed + r24 = load_address single :: module + r25 = [r24] + r26 = load_address r25 + r27 = [6] + r28 = load_address r27 + r29 = (('single', 'single', 'single'),) + r30 = __main__.globals :: static + r31 = 'main' + r32 = '' + r33 = CPyImport_ImportMany(r29, r26, r30, r31, r32, r28) + r34 = single :: module + r35 = 'hello' + r36 = CPyObject_GetAttr(r34, r35) + r37 = PyObject_Vectorcall(r36, 0, 0, 0) return 1 [case testFromImport_toplevel] @@ -643,37 +646,44 @@ L0: r19 = CPyTagged_Add(r13, r18) return r19 def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8, r9, r10 :: object - r11 :: str - r12 :: dict - r13 :: object + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11, r12 :: object + r13 :: str + r14 :: dict + r15 :: object + r16 :: object_ptr L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('g', 'h') - r6 = 'testmodule' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - testmodule = r8 :: module - r9 = ('h',) - r10 = ('two',) - r11 = 'testmodule' - r12 = __main__.globals :: static - r13 = CPyImport_ImportFromMany(r11, r9, r10, r12) - testmodule = r13 :: module + r6 = ('g', 'h') + r7 = 'testmodule' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address testmodule :: module + CPyImport_SetModuleCache(r10, r9) + r11 = ('h',) + r12 = ('two',) + r13 = 'testmodule' + r14 = __main__.globals :: static + r15 = CPyImport_ImportFromMany(r13, r11, r12, r14) + r16 = load_address testmodule :: module + CPyImport_SetModuleCache(r16, r15) return 1 [case testPrintFullname] @@ -1448,52 +1458,55 @@ L0: keep_alive r7 return 1 def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4 :: object - r5 :: dict - r6 :: str - r7 :: object - r8 :: i32 - r9 :: bit - r10 :: dict - r11 :: str - r12 :: object - r13 :: int - r14 :: object - r15 :: str - r16, r17 :: object - r18 :: object[1] - r19 :: object_ptr - r20 :: object + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5 :: object + r6 :: dict + r7 :: str + r8 :: object + r9 :: i32 + r10 :: bit + r11 :: dict + r12 :: str + r13 :: object + r14 :: int + r15 :: object + r16 :: str + r17, r18 :: object + r19 :: object[1] + r20 :: object_ptr + r21 :: object L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = __main__.globals :: static - r6 = 'x' - r7 = object 1 - r8 = CPyDict_SetItem(r5, r6, r7) - r9 = r8 >= 0 :: signed - r10 = __main__.globals :: static - r11 = 'x' - r12 = CPyDict_GetItem(r10, r11) - r13 = unbox(int, r12) - r14 = builtins :: module - r15 = 'print' - r16 = CPyObject_GetAttr(r14, r15) - r17 = box(int, r13) - r18 = [r17] - r19 = load_address r18 - r20 = PyObject_Vectorcall(r16, r19, 1, 0) - keep_alive r17 + keep_alive r0 + r6 = __main__.globals :: static + r7 = 'x' + r8 = object 1 + r9 = CPyDict_SetItem(r6, r7, r8) + r10 = r9 >= 0 :: signed + r11 = __main__.globals :: static + r12 = 'x' + r13 = CPyDict_GetItem(r11, r12) + r14 = unbox(int, r13) + r15 = builtins :: module + r16 = 'print' + r17 = CPyObject_GetAttr(r15, r16) + r18 = box(int, r14) + r19 = [r18] + r20 = load_address r19 + r21 = PyObject_Vectorcall(r17, r20, 1, 0) + keep_alive r18 return 1 [case testCallOverloaded] @@ -2150,161 +2163,166 @@ Bar = NewType('Bar', Foo) y = Bar([1,2,3]) [out] def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8 :: object - r9, r10 :: str - r11 :: object - r12 :: tuple[str, object] + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11, r12 :: str r13 :: object - r14 :: str + r14 :: tuple[str, object] r15 :: object - r16 :: tuple[str, object] + r16 :: str r17 :: object - r18 :: tuple[object, object] + r18 :: tuple[str, object] r19 :: object - r20 :: dict - r21 :: str - r22 :: object - r23 :: object[2] - r24 :: object_ptr - r25 :: object - r26 :: dict - r27 :: str - r28 :: i32 - r29 :: bit - r30 :: str - r31 :: dict + r20 :: tuple[object, object] + r21 :: object + r22 :: dict + r23 :: str + r24 :: object + r25 :: object[2] + r26 :: object_ptr + r27 :: object + r28 :: dict + r29 :: str + r30 :: i32 + r31 :: bit r32 :: str - r33, r34 :: object - r35 :: object[2] - r36 :: object_ptr - r37 :: object - r38 :: tuple - r39 :: dict - r40 :: str - r41 :: i32 - r42 :: bit - r43 :: dict - r44 :: str - r45, r46, r47 :: object - r48 :: dict - r49 :: str - r50 :: i32 - r51 :: bit - r52 :: str - r53 :: dict + r33 :: dict + r34 :: str + r35, r36 :: object + r37 :: object[2] + r38 :: object_ptr + r39 :: object + r40 :: tuple + r41 :: dict + r42 :: str + r43 :: i32 + r44 :: bit + r45 :: dict + r46 :: str + r47, r48, r49 :: object + r50 :: dict + r51 :: str + r52 :: i32 + r53 :: bit r54 :: str - r55 :: object - r56 :: dict - r57 :: str - r58 :: object - r59 :: object[2] - r60 :: object_ptr - r61 :: object - r62 :: dict - r63 :: str - r64 :: i32 - r65 :: bit - r66 :: list - r67, r68, r69 :: object - r70 :: ptr - r71 :: dict - r72 :: str - r73 :: i32 - r74 :: bit -L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r55 :: dict + r56 :: str + r57 :: object + r58 :: dict + r59 :: str + r60 :: object + r61 :: object[2] + r62 :: object_ptr + r63 :: object + r64 :: dict + r65 :: str + r66 :: i32 + r67 :: bit + r68 :: list + r69, r70, r71 :: object + r72 :: ptr + r73 :: dict + r74 :: str + r75 :: i32 + r76 :: bit +L0: + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('List', 'NewType', 'NamedTuple') - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module - r9 = 'Lol' - r10 = 'a' - r11 = load_address PyLong_Type - r12 = (r10, r11) - r13 = box(tuple[str, object], r12) - r14 = 'b' - r15 = load_address PyUnicode_Type - r16 = (r14, r15) - r17 = box(tuple[str, object], r16) - r18 = (r13, r17) - r19 = box(tuple[object, object], r18) - r20 = __main__.globals :: static - r21 = 'NamedTuple' - r22 = CPyDict_GetItem(r20, r21) - r23 = [r9, r19] - r24 = load_address r23 - r25 = PyObject_Vectorcall(r22, r24, 2, 0) - keep_alive r9, r19 - r26 = __main__.globals :: static - r27 = 'Lol' - r28 = CPyDict_SetItem(r26, r27, r25) - r29 = r28 >= 0 :: signed - r30 = '' - r31 = __main__.globals :: static - r32 = 'Lol' - r33 = CPyDict_GetItem(r31, r32) - r34 = object 1 - r35 = [r34, r30] - r36 = load_address r35 - r37 = PyObject_Vectorcall(r33, r36, 2, 0) - keep_alive r34, r30 - r38 = cast(tuple, r37) - r39 = __main__.globals :: static - r40 = 'x' - r41 = CPyDict_SetItem(r39, r40, r38) - r42 = r41 >= 0 :: signed - r43 = __main__.globals :: static - r44 = 'List' - r45 = CPyDict_GetItem(r43, r44) - r46 = load_address PyLong_Type - r47 = PyObject_GetItem(r45, r46) - r48 = __main__.globals :: static - r49 = 'Foo' - r50 = CPyDict_SetItem(r48, r49, r47) - r51 = r50 >= 0 :: signed - r52 = 'Bar' - r53 = __main__.globals :: static - r54 = 'Foo' - r55 = CPyDict_GetItem(r53, r54) - r56 = __main__.globals :: static - r57 = 'NewType' - r58 = CPyDict_GetItem(r56, r57) - r59 = [r52, r55] - r60 = load_address r59 - r61 = PyObject_Vectorcall(r58, r60, 2, 0) - keep_alive r52, r55 - r62 = __main__.globals :: static - r63 = 'Bar' - r64 = CPyDict_SetItem(r62, r63, r61) - r65 = r64 >= 0 :: signed - r66 = PyList_New(3) - r67 = object 1 - r68 = object 2 - r69 = object 3 - r70 = list_items r66 - buf_init_item r70, 0, r67 - buf_init_item r70, 1, r68 - buf_init_item r70, 2, r69 - keep_alive r66 - r71 = __main__.globals :: static - r72 = 'y' - r73 = CPyDict_SetItem(r71, r72, r66) - r74 = r73 >= 0 :: signed + r6 = ('List', 'NewType', 'NamedTuple') + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) + r11 = 'Lol' + keep_alive r0 + r12 = 'a' + r13 = load_address PyLong_Type + r14 = (r12, r13) + r15 = box(tuple[str, object], r14) + r16 = 'b' + r17 = load_address PyUnicode_Type + r18 = (r16, r17) + r19 = box(tuple[str, object], r18) + r20 = (r15, r19) + r21 = box(tuple[object, object], r20) + r22 = __main__.globals :: static + r23 = 'NamedTuple' + r24 = CPyDict_GetItem(r22, r23) + r25 = [r11, r21] + r26 = load_address r25 + r27 = PyObject_Vectorcall(r24, r26, 2, 0) + keep_alive r11, r21 + r28 = __main__.globals :: static + r29 = 'Lol' + r30 = CPyDict_SetItem(r28, r29, r27) + r31 = r30 >= 0 :: signed + r32 = '' + r33 = __main__.globals :: static + r34 = 'Lol' + r35 = CPyDict_GetItem(r33, r34) + r36 = object 1 + r37 = [r36, r32] + r38 = load_address r37 + r39 = PyObject_Vectorcall(r35, r38, 2, 0) + keep_alive r36, r32 + r40 = cast(tuple, r39) + r41 = __main__.globals :: static + r42 = 'x' + r43 = CPyDict_SetItem(r41, r42, r40) + r44 = r43 >= 0 :: signed + r45 = __main__.globals :: static + r46 = 'List' + r47 = CPyDict_GetItem(r45, r46) + r48 = load_address PyLong_Type + r49 = PyObject_GetItem(r47, r48) + r50 = __main__.globals :: static + r51 = 'Foo' + r52 = CPyDict_SetItem(r50, r51, r49) + r53 = r52 >= 0 :: signed + r54 = 'Bar' + r55 = __main__.globals :: static + r56 = 'Foo' + r57 = CPyDict_GetItem(r55, r56) + r58 = __main__.globals :: static + r59 = 'NewType' + r60 = CPyDict_GetItem(r58, r59) + r61 = [r54, r57] + r62 = load_address r61 + r63 = PyObject_Vectorcall(r60, r62, 2, 0) + keep_alive r54, r57 + r64 = __main__.globals :: static + r65 = 'Bar' + r66 = CPyDict_SetItem(r64, r65, r63) + r67 = r66 >= 0 :: signed + r68 = PyList_New(3) + r69 = object 1 + r70 = object 2 + r71 = object 3 + r72 = list_items r68 + buf_init_item r72, 0, r69 + buf_init_item r72, 1, r70 + buf_init_item r72, 2, r71 + keep_alive r68 + r73 = __main__.globals :: static + r74 = 'y' + r75 = CPyDict_SetItem(r73, r74, r68) + r76 = r75 >= 0 :: signed return 1 [case testChainedConditional] @@ -2644,68 +2662,72 @@ L0: r26 = PyObject_Vectorcall(d, 0, 0, 0) return 1 def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8 :: object - r9 :: dict - r10 :: str - r11 :: object - r12 :: dict - r13 :: str - r14 :: object - r15 :: object[1] - r16 :: object_ptr - r17 :: object - r18 :: dict - r19 :: str - r20 :: object - r21 :: object[1] - r22 :: object_ptr - r23 :: object - r24 :: dict - r25 :: str - r26 :: i32 - r27 :: bit + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11 :: dict + r12 :: str + r13 :: object + r14 :: dict + r15 :: str + r16 :: object + r17 :: object[1] + r18 :: object_ptr + r19 :: object + r20 :: dict + r21 :: str + r22 :: object + r23 :: object[1] + r24 :: object_ptr + r25 :: object + r26 :: dict + r27 :: str + r28 :: i32 + r29 :: bit L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('Callable',) - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module - r9 = __main__.globals :: static - r10 = 'c' - r11 = CPyDict_GetItem(r9, r10) - r12 = __main__.globals :: static - r13 = 'b' - r14 = CPyDict_GetItem(r12, r13) - r15 = [r11] - r16 = load_address r15 - r17 = PyObject_Vectorcall(r14, r16, 1, 0) - keep_alive r11 - r18 = __main__.globals :: static - r19 = 'a' - r20 = CPyDict_GetItem(r18, r19) - r21 = [r17] - r22 = load_address r21 - r23 = PyObject_Vectorcall(r20, r22, 1, 0) - keep_alive r17 - r24 = __main__.globals :: static - r25 = 'c' - r26 = PyDict_SetItem(r24, r25, r23) - r27 = r26 >= 0 :: signed + r6 = ('Callable',) + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) + r11 = __main__.globals :: static + r12 = 'c' + r13 = CPyDict_GetItem(r11, r12) + r14 = __main__.globals :: static + r15 = 'b' + r16 = CPyDict_GetItem(r14, r15) + r17 = [r13] + r18 = load_address r17 + r19 = PyObject_Vectorcall(r16, r18, 1, 0) + keep_alive r13 + r20 = __main__.globals :: static + r21 = 'a' + r22 = CPyDict_GetItem(r20, r21) + r23 = [r19] + r24 = load_address r23 + r25 = PyObject_Vectorcall(r22, r24, 1, 0) + keep_alive r19 + r26 = __main__.globals :: static + r27 = 'c' + r28 = PyDict_SetItem(r26, r27, r25) + r29 = r28 >= 0 :: signed return 1 [case testDecoratorsSimple_toplevel] @@ -2785,28 +2807,32 @@ L0: g = r2 return g def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8 :: object + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('Callable',) - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module + r6 = ('Callable',) + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) return 1 [case testAnyAllG] @@ -3291,103 +3317,110 @@ x = 1 [out] def root(): r0 :: dict - r1, r2 :: object - r3 :: bit - r4 :: str - r5 :: object - r6 :: str - r7 :: dict - r8 :: str - r9 :: object - r10 :: i32 - r11 :: bit - r12 :: dict - r13, r14 :: object - r15 :: bit - r16 :: str - r17 :: object + r1 :: object_ptr + r2, r3 :: object + r4 :: bit + r5 :: str + r6 :: object + r7 :: str + r8 :: dict + r9 :: str + r10 :: object + r11 :: i32 + r12 :: bit + r13 :: dict + r14 :: object_ptr + r15, r16 :: object + r17 :: bit r18 :: str - r19 :: dict + r19 :: object r20 :: str - r21 :: object - r22 :: i32 - r23 :: bit + r21 :: dict + r22 :: str + r23 :: object + r24 :: i32 + r25 :: bit L0: r0 = __main__.globals :: static - r1 = dataclasses :: module - r2 = load_address _Py_NoneStruct - r3 = r1 != r2 - if r3 goto L2 else goto L1 :: bool + r1 = load_address CPyModule_dataclasses + r2 = CPyImport_GetModuleCache(r1) + r3 = load_address _Py_NoneStruct + r4 = r2 != r3 + if r4 goto L2 else goto L1 :: bool L1: - r4 = 'dataclasses' - r5 = PyImport_Import(r4) - dataclasses = r5 :: module + r5 = 'dataclasses' + r6 = PyImport_Import(r5) + CPyImport_SetModuleCache(r1, r6) L2: - r6 = 'dataclasses' - r7 = PyImport_GetModuleDict() - r8 = 'dataclasses' - r9 = CPyDict_GetItem(r7, r8) - r10 = CPyDict_SetItem(r0, r6, r9) - r11 = r10 >= 0 :: signed - r12 = __main__.globals :: static - r13 = enum :: module - r14 = load_address _Py_NoneStruct - r15 = r13 != r14 - if r15 goto L4 else goto L3 :: bool + r7 = 'dataclasses' + r8 = PyImport_GetModuleDict() + r9 = 'dataclasses' + r10 = CPyDict_GetItem(r8, r9) + r11 = CPyDict_SetItem(r0, r7, r10) + r12 = r11 >= 0 :: signed + r13 = __main__.globals :: static + r14 = load_address CPyModule_enum + r15 = CPyImport_GetModuleCache(r14) + r16 = load_address _Py_NoneStruct + r17 = r15 != r16 + if r17 goto L4 else goto L3 :: bool L3: - r16 = 'enum' - r17 = PyImport_Import(r16) - enum = r17 :: module -L4: r18 = 'enum' - r19 = PyImport_GetModuleDict() + r19 = PyImport_Import(r18) + CPyImport_SetModuleCache(r14, r19) +L4: r20 = 'enum' - r21 = CPyDict_GetItem(r19, r20) - r22 = CPyDict_SetItem(r12, r18, r21) - r23 = r22 >= 0 :: signed + r21 = PyImport_GetModuleDict() + r22 = 'enum' + r23 = CPyDict_GetItem(r21, r22) + r24 = CPyDict_SetItem(r13, r20, r23) + r25 = r24 >= 0 :: signed return 1 def submodule(): r0 :: dict - r1, r2 :: object - r3 :: bit - r4 :: str - r5 :: object - r6 :: str - r7 :: dict - r8 :: str - r9 :: object - r10 :: i32 - r11 :: bit - r12 :: dict - r13 :: str - r14 :: object - r15 :: str - r16 :: object - r17 :: int + r1 :: object_ptr + r2, r3 :: object + r4 :: bit + r5 :: str + r6 :: object + r7 :: str + r8 :: dict + r9 :: str + r10 :: object + r11 :: i32 + r12 :: bit + r13 :: dict + r14 :: str + r15 :: object + r16 :: str + r17 :: object + r18 :: int L0: r0 = __main__.globals :: static - r1 = p.m :: module - r2 = load_address _Py_NoneStruct - r3 = r1 != r2 - if r3 goto L2 else goto L1 :: bool + r1 = load_address CPyModule_p___m + r2 = CPyImport_GetModuleCache(r1) + r3 = load_address _Py_NoneStruct + r4 = r2 != r3 + if r4 goto L2 else goto L1 :: bool L1: - r4 = 'p.m' - r5 = PyImport_Import(r4) - p.m = r5 :: module + r5 = 'p.m' + r6 = PyImport_Import(r5) + CPyImport_SetModuleCache(r1, r6) L2: - r6 = 'p' - r7 = PyImport_GetModuleDict() - r8 = 'p' - r9 = CPyDict_GetItem(r7, r8) - r10 = CPyDict_SetItem(r0, r6, r9) - r11 = r10 >= 0 :: signed - r12 = PyImport_GetModuleDict() - r13 = 'p' - r14 = CPyDict_GetItem(r12, r13) - r15 = 'x' - r16 = CPyObject_GetAttr(r14, r15) - r17 = unbox(int, r16) - return r17 + r7 = 'p' + r8 = PyImport_GetModuleDict() + r9 = 'p' + r10 = CPyDict_GetItem(r8, r9) + r11 = CPyDict_SetItem(r0, r7, r10) + r12 = r11 >= 0 :: signed + r13 = PyImport_GetModuleDict() + r14 = 'p' + r15 = CPyDict_GetItem(r13, r14) + keep_alive r1 + r16 = 'x' + r17 = CPyObject_GetAttr(r15, r16) + r18 = unbox(int, r17) + return r18 [case testIsinstanceBool] def f(x: object) -> bool: diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 8eaa63e4583b5..f7d7c092bb08d 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -196,160 +196,168 @@ class D(C, S, Generic[T]): [out] def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8, r9 :: object - r10 :: str - r11 :: dict - r12 :: object - r13 :: str - r14 :: dict - r15 :: str - r16 :: object - r17 :: object[1] - r18 :: object_ptr + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11 :: object + r12 :: str + r13 :: dict + r14 :: object + r15 :: object_ptr + r16 :: str + r17 :: dict + r18 :: str r19 :: object - r20 :: dict - r21 :: str - r22 :: i32 - r23 :: bit - r24 :: object - r25 :: str - r26, r27 :: object - r28, r29 :: bool - r30 :: str - r31 :: tuple - r32 :: i32 - r33 :: bit - r34 :: dict - r35 :: str - r36 :: i32 - r37 :: bit - r38 :: bool - r39 :: object - r40 :: str - r41, r42 :: object + r20 :: object[1] + r21 :: object_ptr + r22 :: object + r23 :: dict + r24 :: str + r25 :: i32 + r26 :: bit + r27 :: object + r28 :: str + r29, r30 :: object + r31, r32 :: bool + r33 :: str + r34 :: tuple + r35 :: i32 + r36 :: bit + r37 :: dict + r38 :: str + r39 :: i32 + r40 :: bit + r41 :: bool + r42 :: object r43 :: str - r44 :: tuple - r45 :: i32 - r46 :: bit - r47 :: dict - r48 :: str - r49 :: i32 - r50 :: bit - r51 :: bool - r52, r53 :: object - r54 :: dict - r55 :: str - r56 :: object + r44, r45 :: object + r46 :: str + r47 :: tuple + r48 :: i32 + r49 :: bit + r50 :: dict + r51 :: str + r52 :: i32 + r53 :: bit + r54 :: bool + r55, r56 :: object r57 :: dict r58 :: str - r59, r60 :: object - r61 :: tuple - r62 :: str - r63, r64 :: object - r65, r66 :: bool - r67, r68 :: str - r69 :: tuple - r70 :: i32 - r71 :: bit - r72 :: dict - r73 :: str - r74 :: i32 - r75 :: bit - r76 :: bool -L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r59 :: object + r60 :: dict + r61 :: str + r62, r63 :: object + r64 :: tuple + r65 :: str + r66, r67 :: object + r68, r69 :: bool + r70, r71 :: str + r72 :: tuple + r73 :: i32 + r74 :: bit + r75 :: dict + r76 :: str + r77 :: i32 + r78 :: bit + r79 :: bool +L0: + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('TypeVar', 'Generic') - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module - r9 = ('trait',) - r10 = 'mypy_extensions' - r11 = __main__.globals :: static - r12 = CPyImport_ImportFromMany(r10, r9, r9, r11) - mypy_extensions = r12 :: module - r13 = 'T' - r14 = __main__.globals :: static - r15 = 'TypeVar' - r16 = CPyDict_GetItem(r14, r15) - r17 = [r13] - r18 = load_address r17 - r19 = PyObject_Vectorcall(r16, r18, 1, 0) - keep_alive r13 - r20 = __main__.globals :: static - r21 = 'T' - r22 = CPyDict_SetItem(r20, r21, r19) - r23 = r22 >= 0 :: signed - r24 = :: object - r25 = '__main__' - r26 = __main__.C_template :: type - r27 = CPyType_FromTemplate(r26, r24, r25) - r28 = C_trait_vtable_setup() - r29 = C_coroutine_setup(r27) - r30 = '__mypyc_attrs__' - r31 = CPyTuple_LoadEmptyTupleConstant() - r32 = PyObject_SetAttr(r27, r30, r31) - r33 = r32 >= 0 :: signed - __main__.C = r27 :: type - r34 = __main__.globals :: static - r35 = 'C' - r36 = PyDict_SetItem(r34, r35, r27) - r37 = r36 >= 0 :: signed - r38 = CPy_InitSubclass(r27) - r39 = :: object - r40 = '__main__' - r41 = __main__.S_template :: type - r42 = CPyType_FromTemplate(r41, r39, r40) - r43 = '__mypyc_attrs__' - r44 = CPyTuple_LoadEmptyTupleConstant() - r45 = PyObject_SetAttr(r42, r43, r44) - r46 = r45 >= 0 :: signed - __main__.S = r42 :: type - r47 = __main__.globals :: static - r48 = 'S' - r49 = PyDict_SetItem(r47, r48, r42) - r50 = r49 >= 0 :: signed - r51 = CPy_InitSubclass(r42) - r52 = __main__.C :: type - r53 = __main__.S :: type - r54 = __main__.globals :: static - r55 = 'Generic' - r56 = CPyDict_GetItem(r54, r55) + r6 = ('TypeVar', 'Generic') + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) + r11 = ('trait',) + r12 = 'mypy_extensions' + r13 = __main__.globals :: static + r14 = CPyImport_ImportFromMany(r12, r11, r11, r13) + r15 = load_address mypy_extensions :: module + CPyImport_SetModuleCache(r15, r14) + r16 = 'T' + keep_alive r0 + r17 = __main__.globals :: static + r18 = 'TypeVar' + r19 = CPyDict_GetItem(r17, r18) + r20 = [r16] + r21 = load_address r20 + r22 = PyObject_Vectorcall(r19, r21, 1, 0) + keep_alive r16 + r23 = __main__.globals :: static + r24 = 'T' + r25 = CPyDict_SetItem(r23, r24, r22) + r26 = r25 >= 0 :: signed + r27 = :: object + r28 = '__main__' + r29 = __main__.C_template :: type + r30 = CPyType_FromTemplate(r29, r27, r28) + r31 = C_trait_vtable_setup() + r32 = C_coroutine_setup(r30) + r33 = '__mypyc_attrs__' + r34 = CPyTuple_LoadEmptyTupleConstant() + r35 = PyObject_SetAttr(r30, r33, r34) + r36 = r35 >= 0 :: signed + __main__.C = r30 :: type + r37 = __main__.globals :: static + r38 = 'C' + r39 = PyDict_SetItem(r37, r38, r30) + r40 = r39 >= 0 :: signed + r41 = CPy_InitSubclass(r30) + r42 = :: object + r43 = '__main__' + r44 = __main__.S_template :: type + r45 = CPyType_FromTemplate(r44, r42, r43) + r46 = '__mypyc_attrs__' + r47 = CPyTuple_LoadEmptyTupleConstant() + r48 = PyObject_SetAttr(r45, r46, r47) + r49 = r48 >= 0 :: signed + __main__.S = r45 :: type + r50 = __main__.globals :: static + r51 = 'S' + r52 = PyDict_SetItem(r50, r51, r45) + r53 = r52 >= 0 :: signed + r54 = CPy_InitSubclass(r45) + r55 = __main__.C :: type + r56 = __main__.S :: type r57 = __main__.globals :: static - r58 = 'T' + r58 = 'Generic' r59 = CPyDict_GetItem(r57, r58) - r60 = PyObject_GetItem(r56, r59) - r61 = PyTuple_Pack(3, r52, r53, r60) - r62 = '__main__' - r63 = __main__.D_template :: type - r64 = CPyType_FromTemplate(r63, r61, r62) - r65 = D_trait_vtable_setup() - r66 = D_coroutine_setup(r64) - r67 = '__mypyc_attrs__' - r68 = '__dict__' - r69 = PyTuple_Pack(1, r68) - r70 = PyObject_SetAttr(r64, r67, r69) - r71 = r70 >= 0 :: signed - __main__.D = r64 :: type - r72 = __main__.globals :: static - r73 = 'D' - r74 = PyDict_SetItem(r72, r73, r64) - r75 = r74 >= 0 :: signed - r76 = CPy_InitSubclass(r64) + r60 = __main__.globals :: static + r61 = 'T' + r62 = CPyDict_GetItem(r60, r61) + r63 = PyObject_GetItem(r59, r62) + r64 = PyTuple_Pack(3, r55, r56, r63) + r65 = '__main__' + r66 = __main__.D_template :: type + r67 = CPyType_FromTemplate(r66, r64, r65) + r68 = D_trait_vtable_setup() + r69 = D_coroutine_setup(r67) + r70 = '__mypyc_attrs__' + r71 = '__dict__' + r72 = PyTuple_Pack(1, r71) + r73 = PyObject_SetAttr(r67, r70, r72) + r74 = r73 >= 0 :: signed + __main__.D = r67 :: type + r75 = __main__.globals :: static + r76 = 'D' + r77 = PyDict_SetItem(r75, r76, r67) + r78 = r77 >= 0 :: signed + r79 = CPy_InitSubclass(r67) return 1 [case testIsInstance] @@ -3014,103 +3022,109 @@ class Ext: B: ClassVar[Set[int]] = A | {3} [out] def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8, r9 :: object - r10 :: str - r11, r12 :: object - r13, r14 :: bool - r15 :: str - r16 :: tuple - r17 :: i32 - r18 :: bit - r19 :: dict - r20 :: str - r21 :: i32 - r22 :: bit - r23 :: object - r24 :: set + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11 :: object + r12 :: str + r13, r14 :: object + r15, r16 :: bool + r17 :: str + r18 :: tuple + r19 :: i32 + r20 :: bit + r21 :: dict + r22 :: str + r23 :: i32 + r24 :: bit r25 :: object - r26 :: i32 - r27 :: bit - r28 :: object - r29 :: i32 - r30 :: bit - r31 :: str - r32 :: i32 - r33 :: bit - r34 :: object - r35 :: str + r26 :: set + r27 :: object + r28 :: i32 + r29 :: bit + r30 :: object + r31 :: i32 + r32 :: bit + r33 :: str + r34 :: i32 + r35 :: bit r36 :: object - r37, r38 :: set - r39 :: object - r40 :: i32 - r41 :: bit - r42 :: object - r43 :: set - r44 :: str - r45 :: i32 - r46 :: bit - r47 :: bool -L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r37 :: str + r38 :: object + r39, r40 :: set + r41 :: object + r42 :: i32 + r43 :: bit + r44 :: object + r45 :: set + r46 :: str + r47 :: i32 + r48 :: bit + r49 :: bool +L0: + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('ClassVar', 'Set') - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module - r9 = :: object - r10 = '__main__' - r11 = __main__.Ext_template :: type - r12 = CPyType_FromTemplate(r11, r9, r10) - r13 = Ext_trait_vtable_setup() - r14 = Ext_coroutine_setup(r12) - r15 = '__mypyc_attrs__' - r16 = CPyTuple_LoadEmptyTupleConstant() - r17 = PyObject_SetAttr(r12, r15, r16) - r18 = r17 >= 0 :: signed - __main__.Ext = r12 :: type - r19 = __main__.globals :: static - r20 = 'Ext' - r21 = PyDict_SetItem(r19, r20, r12) - r22 = r21 >= 0 :: signed - r23 = __main__.Ext :: type - r24 = PySet_New(0) - r25 = object 1 - r26 = PySet_Add(r24, r25) - r27 = r26 >= 0 :: signed - r28 = object 2 - r29 = PySet_Add(r24, r28) - r30 = r29 >= 0 :: signed - r31 = 'A' - r32 = PyObject_SetAttr(r23, r31, r24) - r33 = r32 >= 0 :: signed - r34 = __main__.Ext :: type - r35 = 'A' - r36 = CPyObject_GetAttr(r12, r35) - r37 = cast(set, r36) - r38 = PySet_New(0) - r39 = object 3 - r40 = PySet_Add(r38, r39) - r41 = r40 >= 0 :: signed - r42 = PyNumber_Or(r37, r38) - r43 = cast(set, r42) - r44 = 'B' - r45 = PyObject_SetAttr(r34, r44, r43) - r46 = r45 >= 0 :: signed - r47 = CPy_InitSubclass(r12) + r6 = ('ClassVar', 'Set') + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) + r11 = :: object + r12 = '__main__' + r13 = __main__.Ext_template :: type + r14 = CPyType_FromTemplate(r13, r11, r12) + r15 = Ext_trait_vtable_setup() + r16 = Ext_coroutine_setup(r14) + r17 = '__mypyc_attrs__' + r18 = CPyTuple_LoadEmptyTupleConstant() + r19 = PyObject_SetAttr(r14, r17, r18) + r20 = r19 >= 0 :: signed + __main__.Ext = r14 :: type + r21 = __main__.globals :: static + r22 = 'Ext' + r23 = PyDict_SetItem(r21, r22, r14) + r24 = r23 >= 0 :: signed + r25 = __main__.Ext :: type + keep_alive r0 + r26 = PySet_New(0) + r27 = object 1 + r28 = PySet_Add(r26, r27) + r29 = r28 >= 0 :: signed + r30 = object 2 + r31 = PySet_Add(r26, r30) + r32 = r31 >= 0 :: signed + r33 = 'A' + r34 = PyObject_SetAttr(r25, r33, r26) + r35 = r34 >= 0 :: signed + r36 = __main__.Ext :: type + r37 = 'A' + r38 = CPyObject_GetAttr(r14, r37) + r39 = cast(set, r38) + r40 = PySet_New(0) + r41 = object 3 + r42 = PySet_Add(r40, r41) + r43 = r42 >= 0 :: signed + r44 = PyNumber_Or(r39, r40) + r45 = cast(set, r44) + r46 = 'B' + r47 = PyObject_SetAttr(r36, r46, r45) + r48 = r47 >= 0 :: signed + r49 = CPy_InitSubclass(r14) return 1 [case testClassVarSelfReferenceNonExt_withgil_toplevel] @@ -3123,182 +3137,189 @@ class NonExt: B: ClassVar[Set[str]] = A | {"b"} [out] def __top_level__(): - r0, r1 :: object - r2 :: bit - r3 :: str - r4, r5 :: object - r6 :: str - r7 :: dict - r8, r9 :: object - r10 :: str - r11 :: dict - r12 :: object - r13 :: tuple - r14, r15 :: object - r16 :: str - r17 :: bool - r18, r19 :: str - r20 :: object - r21 :: object[2] - r22 :: object_ptr + r0 :: object_ptr + r1, r2 :: object + r3 :: bit + r4 :: str + r5, r6 :: object + r7 :: str + r8 :: dict + r9 :: object + r10 :: object_ptr + r11 :: object + r12 :: str + r13 :: dict + r14 :: object + r15 :: object_ptr + r16 :: tuple + r17, r18 :: object + r19 :: str + r20 :: bool + r21, r22 :: str r23 :: object - r24, r25, r26, r27 :: dict - r28 :: object - r29 :: str - r30 :: i32 - r31 :: bit + r24 :: object[2] + r25 :: object_ptr + r26 :: object + r27, r28, r29, r30 :: dict + r31 :: object r32 :: str - r33 :: set - r34 :: i32 - r35 :: bit - r36 :: str + r33 :: i32 + r34 :: bit + r35 :: str + r36 :: set r37 :: i32 r38 :: bit - r39 :: object - r40 :: str - r41 :: i32 - r42 :: bit + r39 :: str + r40 :: i32 + r41 :: bit + r42 :: object r43 :: str - r44 :: object - r45 :: set + r44 :: i32 + r45 :: bit r46 :: str - r47 :: set - r48 :: i32 - r49 :: bit - r50 :: object - r51 :: set - r52 :: str - r53 :: i32 - r54 :: bit - r55, r56 :: str - r57 :: i32 - r58 :: bit - r59, r60 :: str - r61 :: i32 - r62 :: bit - r63, r64 :: str - r65 :: i32 - r66 :: bit - r67 :: object[3] - r68 :: object_ptr - r69 :: object - r70 :: dict - r71 :: str - r72, r73 :: object - r74 :: object[1] - r75 :: object_ptr - r76, r77 :: object - r78 :: object[1] - r79 :: object_ptr - r80 :: object - r81 :: dict - r82 :: str - r83 :: i32 - r84 :: bit - r85 :: object -L0: - r0 = builtins :: module - r1 = load_address _Py_NoneStruct - r2 = r0 != r1 - if r2 goto L2 else goto L1 :: bool + r47 :: object + r48 :: set + r49 :: str + r50 :: set + r51 :: i32 + r52 :: bit + r53 :: object + r54 :: set + r55 :: str + r56 :: i32 + r57 :: bit + r58, r59 :: str + r60 :: i32 + r61 :: bit + r62, r63 :: str + r64 :: i32 + r65 :: bit + r66, r67 :: str + r68 :: i32 + r69 :: bit + r70 :: object[3] + r71 :: object_ptr + r72 :: object + r73 :: dict + r74 :: str + r75, r76 :: object + r77 :: object[1] + r78 :: object_ptr + r79, r80 :: object + r81 :: object[1] + r82 :: object_ptr + r83 :: object + r84 :: dict + r85 :: str + r86 :: i32 + r87 :: bit + r88 :: object +L0: + r0 = load_address CPyModule_builtins + r1 = CPyImport_GetModuleCache(r0) + r2 = load_address _Py_NoneStruct + r3 = r1 != r2 + if r3 goto L2 else goto L1 :: bool L1: - r3 = 'builtins' - r4 = PyImport_Import(r3) - builtins = r4 :: module + r4 = 'builtins' + r5 = PyImport_Import(r4) + CPyImport_SetModuleCache(r0, r5) L2: - r5 = ('ClassVar', 'Set') - r6 = 'typing' - r7 = __main__.globals :: static - r8 = CPyImport_ImportFromMany(r6, r5, r5, r7) - typing = r8 :: module - r9 = ('mypyc_attr',) - r10 = 'mypy_extensions' - r11 = __main__.globals :: static - r12 = CPyImport_ImportFromMany(r10, r9, r9, r11) - mypy_extensions = r12 :: module - r13 = CPyTuple_LoadEmptyTupleConstant() - r14 = load_address PyType_Type - r15 = CPy_CalculateMetaclass(r14, r13) - r16 = '__prepare__' - r17 = PyObject_HasAttr(r15, r16) - if r17 goto L3 else goto L4 :: bool -L3: - r18 = 'NonExt' + r6 = ('ClassVar', 'Set') + r7 = 'typing' + r8 = __main__.globals :: static + r9 = CPyImport_ImportFromMany(r7, r6, r6, r8) + r10 = load_address typing :: module + CPyImport_SetModuleCache(r10, r9) + r11 = ('mypyc_attr',) + r12 = 'mypy_extensions' + r13 = __main__.globals :: static + r14 = CPyImport_ImportFromMany(r12, r11, r11, r13) + r15 = load_address mypy_extensions :: module + CPyImport_SetModuleCache(r15, r14) + r16 = CPyTuple_LoadEmptyTupleConstant() + r17 = load_address PyType_Type + r18 = CPy_CalculateMetaclass(r17, r16) r19 = '__prepare__' - r20 = CPyObject_GetAttr(r15, r19) - r21 = [r18, r13] - r22 = load_address r21 - r23 = PyObject_Vectorcall(r20, r22, 2, 0) - keep_alive r18, r13 - r24 = cast(dict, r23) - r25 = r24 + r20 = PyObject_HasAttr(r18, r19) + if r20 goto L3 else goto L4 :: bool +L3: + r21 = 'NonExt' + r22 = '__prepare__' + r23 = CPyObject_GetAttr(r18, r22) + r24 = [r21, r16] + r25 = load_address r24 + r26 = PyObject_Vectorcall(r23, r25, 2, 0) + keep_alive r21, r16 + r27 = cast(dict, r26) + r28 = r27 goto L5 L4: - r26 = PyDict_New() - r25 = r26 + r29 = PyDict_New() + r28 = r29 L5: - r27 = PyDict_New() - r28 = load_address PySet_Type - r29 = 'A' - r30 = PyDict_SetItem(r27, r29, r28) - r31 = r30 >= 0 :: signed - r32 = 'a' - keep_alive r14, r13 - r33 = PySet_New(0) - r34 = PySet_Add(r33, r32) - r35 = r34 >= 0 :: signed - r36 = 'A' - r37 = CPyDict_SetItem(r25, r36, r33) + r30 = PyDict_New() + r31 = load_address PySet_Type + r32 = 'A' + r33 = PyDict_SetItem(r30, r32, r31) + r34 = r33 >= 0 :: signed + r35 = 'a' + keep_alive r0, r17, r16 + r36 = PySet_New(0) + r37 = PySet_Add(r36, r35) r38 = r37 >= 0 :: signed - r39 = load_address PySet_Type - r40 = 'B' - r41 = PyDict_SetItem(r27, r40, r39) - r42 = r41 >= 0 :: signed - r43 = 'A' - r44 = CPyDict_GetItem(r25, r43) - r45 = cast(set, r44) - r46 = 'b' - r47 = PySet_New(0) - r48 = PySet_Add(r47, r46) - r49 = r48 >= 0 :: signed - r50 = PyNumber_Or(r45, r47) - r51 = cast(set, r50) - r52 = 'B' - r53 = CPyDict_SetItem(r25, r52, r51) - r54 = r53 >= 0 :: signed - r55 = 'NonExt' - r56 = '__annotations__' - r57 = CPyDict_SetItem(r25, r56, r27) - r58 = r57 >= 0 :: signed - r59 = 'mypyc filler docstring' - r60 = '__doc__' - r61 = CPyDict_SetItem(r25, r60, r59) - r62 = r61 >= 0 :: signed - r63 = '__main__' - r64 = '__module__' - r65 = CPyDict_SetItem(r25, r64, r63) - r66 = r65 >= 0 :: signed - r67 = [r55, r13, r25] - r68 = load_address r67 - r69 = PyObject_Vectorcall(r15, r68, 3, 0) - keep_alive r55, r13, r25 - r70 = __main__.globals :: static - r71 = 'mypyc_attr' - r72 = CPyDict_GetItem(r70, r71) - r73 = box(bool, 0) - r74 = [r73] - r75 = load_address r74 - r76 = ('native_class',) - r77 = PyObject_Vectorcall(r72, r75, 0, r76) - keep_alive r73 - r78 = [r69] - r79 = load_address r78 - r80 = PyObject_Vectorcall(r77, r79, 1, 0) - keep_alive r69 - __main__.NonExt = r80 :: type - r81 = __main__.globals :: static - r82 = 'NonExt' - r83 = PyDict_SetItem(r81, r82, r80) - r84 = r83 >= 0 :: signed - r85 = __main__.NonExt :: type + r39 = 'A' + r40 = CPyDict_SetItem(r28, r39, r36) + r41 = r40 >= 0 :: signed + r42 = load_address PySet_Type + r43 = 'B' + r44 = PyDict_SetItem(r30, r43, r42) + r45 = r44 >= 0 :: signed + r46 = 'A' + r47 = CPyDict_GetItem(r28, r46) + r48 = cast(set, r47) + r49 = 'b' + r50 = PySet_New(0) + r51 = PySet_Add(r50, r49) + r52 = r51 >= 0 :: signed + r53 = PyNumber_Or(r48, r50) + r54 = cast(set, r53) + r55 = 'B' + r56 = CPyDict_SetItem(r28, r55, r54) + r57 = r56 >= 0 :: signed + r58 = 'NonExt' + r59 = '__annotations__' + r60 = CPyDict_SetItem(r28, r59, r30) + r61 = r60 >= 0 :: signed + r62 = 'mypyc filler docstring' + r63 = '__doc__' + r64 = CPyDict_SetItem(r28, r63, r62) + r65 = r64 >= 0 :: signed + r66 = '__main__' + r67 = '__module__' + r68 = CPyDict_SetItem(r28, r67, r66) + r69 = r68 >= 0 :: signed + r70 = [r58, r16, r28] + r71 = load_address r70 + r72 = PyObject_Vectorcall(r18, r71, 3, 0) + keep_alive r58, r16, r28 + r73 = __main__.globals :: static + r74 = 'mypyc_attr' + r75 = CPyDict_GetItem(r73, r74) + r76 = box(bool, 0) + r77 = [r76] + r78 = load_address r77 + r79 = ('native_class',) + r80 = PyObject_Vectorcall(r75, r78, 0, r79) + keep_alive r76 + r81 = [r72] + r82 = load_address r81 + r83 = PyObject_Vectorcall(r80, r82, 1, 0) + keep_alive r72 + __main__.NonExt = r83 :: type + r84 = __main__.globals :: static + r85 = 'NonExt' + r86 = PyDict_SetItem(r84, r85, r83) + r87 = r86 >= 0 :: signed + r88 = __main__.NonExt :: type return 1 From 9cf64c05815ee773021eff2d78896358a145194d Mon Sep 17 00:00:00 2001 From: Piotr Sawicki Date: Mon, 31 Aug 2026 18:46:54 +0200 Subject: [PATCH 10/10] Synchronize module cache; report error when deleting and reimporting --- mypyc/codegen/emitmodule.py | 8 +- mypyc/irbuild/builder.py | 4 +- mypyc/lib-rt/CPy.h | 6 +- mypyc/lib-rt/locks.c | 35 ++++++++ mypyc/lib-rt/misc_ops.c | 21 ++++- mypyc/primitives/misc_ops.py | 4 +- mypyc/test-data/run-multimodule.test | 114 +++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 9 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 70f3cc351bc09..a413b31408ba5 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -1290,6 +1290,8 @@ def emit_module_exec_func( emitter.context.declarations[exec_name] = HeaderDeclaration(declaration + ";") impl_name = f"{exec_name}__impl" module_static = self.module_internal_static_name(module_name, emitter) + state = self.import_state_name(module_name) + module_cache = emitter.static_name(module_name, None, prefix=MODULE_PREFIX) emitter.emit_lines(f"static int {impl_name}(PyObject *module)", "{") if not self.use_shared_lib: emitter.emit_lines("if (intern_strings() < 0)", " return -1;") @@ -1385,9 +1387,11 @@ def emit_module_exec_func( emitter.emit_line("return -1;") emitter.emit_line("}") - state = self.import_state_name(module_name) emitter.emit_lines( - declaration, "{", f"return CPyImport_Exec(module, {impl_name}, &{state});", "}" + declaration, + "{", + f"return CPyImport_Exec(module, {impl_name}, &{state}, &{module_cache});", + "}", ) def emit_init_only_func(self, emitter: Emitter, module_name: str, module_prefix: str) -> None: diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 6bfa58b843292..22f8b521e52c5 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -629,7 +629,9 @@ def check_if_module_loaded( check_initialized = BasicBlock() self.add_bool_branch(comparison, check_initialized, needs_import) self.activate_block(check_initialized) - initialized = self.call_c(native_import_is_initialized_op, [import_state], line) + initialized = self.call_c( + native_import_is_initialized_op, [import_state, first_load, module_cache], line + ) self.add_bool_branch(initialized, out, needs_import) def get_module(self, module: str, line: int) -> Value: diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index c98d9ac7d9440..adc42623bd44e 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -42,9 +42,12 @@ int CPyImport_AcquireLock(CPyModuleLockAPI *api, PyObject *module_name, PyObject **module_lock); int CPyImport_ReleaseLock(PyObject *module_lock); bool CPyImport_IsInitialized(const CPyImportState *state); +bool CPyImport_IsInitializedForModule(const CPyImportState *state, PyObject *module, + CPyModule **module_cache); void CPyImport_SetInitialized(CPyImportState *state, bool initialized); PyObject *CPyImport_GetModuleCache(CPyModule **cache); void CPyImport_SetModuleCache(CPyModule **cache, PyObject *module); +void CPyImport_ReplaceModuleCache(CPyModule **cache, PyObject *module); // Naming conventions: @@ -1071,7 +1074,8 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, CPyImportState *state, CPyModuleLockAPI *lock_api, PyObject *shared_lib_file, PyObject *ext_suffix, Py_ssize_t is_package); -int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state); +int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state, + CPyModule **module_cache); PyObject *CPyImport_BeginInitializing(PyObject *module); int CPyImport_EndInitializing(PyObject *spec); int CPyImport_SetDunderAttrs(PyObject *module, PyObject *module_name, PyObject *shared_lib_file, diff --git a/mypyc/lib-rt/locks.c b/mypyc/lib-rt/locks.c index 3350439f563ff..ccaf6568264c0 100644 --- a/mypyc/lib-rt/locks.c +++ b/mypyc/lib-rt/locks.c @@ -89,6 +89,14 @@ bool CPyImport_IsInitialized(const CPyImportState *state) { #endif } +bool CPyImport_IsInitializedForModule(const CPyImportState *state, PyObject *module, + CPyModule **module_cache) { + // Read initialized before re-reading the cache. If a retry completed after + // the caller's first cache load, this load rejects its stale pointer. + return CPyImport_IsInitialized(state) + && CPyImport_GetModuleCache(module_cache) == module; +} + void CPyImport_SetInitialized(CPyImportState *state, bool initialized) { #ifdef _WIN32 InterlockedExchange((volatile LONG *)&state->initialized, initialized); @@ -97,6 +105,19 @@ void CPyImport_SetInitialized(CPyImportState *state, bool initialized) { #endif } +static void CPyImport_DecRefOld(PyObject *previous) { + if (previous == NULL) { + return; + } +#ifdef Py_GIL_DISABLED + // Atomic loads return borrowed references, so defer releasing the old + // reference until concurrent readers have passed a quiescent point. + CPy_DecRefAttrOld(previous); +#else + Py_DECREF(previous); +#endif +} + PyObject *CPyImport_GetModuleCache(CPyModule **cache) { #ifdef _WIN32 return InterlockedCompareExchangePointer((PVOID volatile *)cache, NULL, NULL); @@ -121,3 +142,17 @@ void CPyImport_SetModuleCache(CPyModule **cache, PyObject *module) { } #endif } + +void CPyImport_ReplaceModuleCache(CPyModule **cache, PyObject *module) { + Py_INCREF(module); + CPyModule *previous; +#ifdef _WIN32 + previous = InterlockedExchangePointer((PVOID volatile *)cache, module); +#else + previous = __atomic_exchange_n(cache, (CPyModule *)module, __ATOMIC_ACQ_REL); +#endif + if (previous == NULL || previous == (CPyModule *)Py_None) { + return; + } + CPyImport_DecRefOld((PyObject *)previous); +} diff --git a/mypyc/lib-rt/misc_ops.c b/mypyc/lib-rt/misc_ops.c index 4a7c30f4b3102..9eb9eef2fca51 100644 --- a/mypyc/lib-rt/misc_ops.c +++ b/mypyc/lib-rt/misc_ops.c @@ -1585,13 +1585,28 @@ static int CPyImport_ReleaseLockPreservingException(PyObject *module_lock) { } // Execute a module once; caller holds the module lock. -int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state) { +int CPyImport_Exec(PyObject *module, int (*exec_fn)(PyObject *), CPyImportState *state, + CPyModule **module_cache) { if (CPyImport_IsInitialized(state)) { - return 0; + const char *module_name = PyModule_GetName(module); + if (module_name != NULL) { + PyErr_Format(PyExc_ImportError, + "native module '%s' does not support reinitialization", + module_name); + } + return -1; } int result = exec_fn(module); if (result == 0) { + // Keep the cache lazy. A normal shim import should not populate it, since + // the first compiled native import must still validate sys.modules. If a + // compiled import already populated the cache (including with a partial + // module during a circular import), refresh it to this instance. + PyObject *cached_module = CPyImport_GetModuleCache(module_cache); + if (cached_module != NULL && cached_module != Py_None) { + CPyImport_ReplaceModuleCache(module_cache, module); + } CPyImport_SetInitialized(state, true); } return result; @@ -1737,8 +1752,8 @@ PyObject *CPyImport_ImportNative(PyObject *module_name, PyObject_DelItem(module_dict, module_name); PyErr_Clear(); PyErr_Restore(exc_type, exc_val, exc_tb); - Py_CLEAR(*module_static); CPyImport_SetInitialized(state, false); + Py_CLEAR(*module_static); CPyImport_ReleaseLockPreservingException(module_lock); return NULL; } diff --git a/mypyc/primitives/misc_ops.py b/mypyc/primitives/misc_ops.py index 913164bad100d..80c9eadabd44f 100644 --- a/mypyc/primitives/misc_ops.py +++ b/mypyc/primitives/misc_ops.py @@ -160,9 +160,9 @@ ) native_import_is_initialized_op = custom_op( - arg_types=[c_pointer_rprimitive], + arg_types=[c_pointer_rprimitive, object_rprimitive, object_pointer_rprimitive], return_type=bit_rprimitive, - c_function_name="CPyImport_IsInitialized", + c_function_name="CPyImport_IsInitializedForModule", error_kind=ERR_NEVER, ) diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index 574e7f847af52..0eeabe14b30fd 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -2415,6 +2415,93 @@ assert errors == ["first initialization failed"] assert failure_state.attempts == 2 assert other_importer.load() == 42 +[case testFailedConcurrentCircularNativeImportRefreshesCache] +# separate: [(["native.py", "other_a.py", "other_b.py"], "testgroup")] +pass + +[file import_sync.py] +from threading import Event + +a_started = Event() +b_started = Event() +a_importing_b = Event() +b_waiting_for_driver = Event() +b_may_import_a = Event() + +[file failure_state.py] +attempts = 0 + +[file other_a.py] +import failure_state + +failure_state.attempts += 1 +if failure_state.attempts == 1: + import import_sync + + import_sync.a_started.set() + assert import_sync.b_started.wait(timeout=5) + import_sync.a_importing_b.set() + import other_b + raise RuntimeError("first initialization failed") + +value = 42 + +[file other_b.py] +import import_sync + +import_sync.b_started.set() +assert import_sync.a_started.wait(timeout=5) +assert import_sync.a_importing_b.wait(timeout=5) +import_sync.b_waiting_for_driver.set() +assert import_sync.b_may_import_a.wait(timeout=5) +import other_a + +saw_partial_a = not hasattr(other_a, "value") + +def get_a_value() -> int: + return other_a.value + +[file driver.py] +from concurrent.futures import ThreadPoolExecutor +import importlib +from time import monotonic, sleep + +from testutil import assertRaises + +import failure_state +import import_sync +import native # Preload the compilation group's shared library. + +with ThreadPoolExecutor(max_workers=2) as executor: + future_a = executor.submit(importlib.import_module, "other_a") + assert import_sync.a_started.wait(timeout=5) + future_b = executor.submit(importlib.import_module, "other_b") + assert import_sync.a_importing_b.wait(timeout=5) + assert import_sync.b_waiting_for_driver.wait(timeout=5) + + # Wait until other_a is blocked on other_b's module lock. Letting other_b + # import other_a then deterministically closes the lock cycle and returns + # other_a's partial module. + bootstrap = importlib.import_module("importlib._bootstrap") + lock = bootstrap._get_module_lock("other_b") + deadline = monotonic() + 5 + while not lock.waiters and monotonic() < deadline: + sleep(0.001) + has_waiter = bool(lock.waiters) + import_sync.b_may_import_a.set() + assert has_waiter + + with assertRaises(RuntimeError, "first initialization failed"): + future_a.result(timeout=10) + other_b = future_b.result(timeout=10) + +assert failure_state.attempts == 1 +assert other_b.saw_partial_a +other_a = importlib.import_module("other_a") +assert failure_state.attempts == 2 +assert other_a.value == 42 +assert other_b.get_a_value() == 42 + [case testConcurrentCircularNativeImports] # separate: [(["other_a.py", "other_b.py"], "testgroup")] pass @@ -2457,6 +2544,33 @@ with ThreadPoolExecutor(max_workers=2) as executor: assert other_a.other_value == "b" assert other_b.other_value == "a" +[case testNativeModuleReimportBehavior] +value = 42 + +def get_value() -> int: + return value + +[file driver.py] +import importlib +import sys + +from testutil import assertRaises + +import native + +first = native +assert first.get_value() == 42 +del sys.modules["native"] +if hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled(): + with assertRaises(ImportError, "native module 'native' does not support reinitialization"): + importlib.import_module("native") + assert "native" not in sys.modules +else: + # Legacy single-phase initialization returns the original module object. + second = importlib.import_module("native") + assert second is first +assert first.get_value() == 42 + [case testTopLevelThreadImportsNativeModuleFromSameGroup] # separate: [(["native.py", "other_target.py"], "testgroup")] from threading import Thread