From f7a8bc96304aece03ac3c982bf3ee37cece2d5d2 Mon Sep 17 00:00:00 2001 From: Ryan Heard Date: Tue, 6 Oct 2026 22:40:49 -0400 Subject: [PATCH 1/2] [mypyc] Fix class decorators that don't return a class --- mypyc/ir/class_ir.py | 6 ++ mypyc/irbuild/builder.py | 9 +++ mypyc/irbuild/classdef.py | 41 +++++++++-- mypyc/irbuild/prepare.py | 2 + mypyc/irbuild/util.py | 33 +++++---- mypyc/test-data/irbuild-classes.test | 45 ++++++++++++ mypyc/test-data/run-classes.test | 100 ++++++++++++++++++++++++++ test-data/unit/lib-stub/functools.pyi | 5 ++ 8 files changed, 223 insertions(+), 18 deletions(-) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index ab5df1ac31ff6..3400d3d0cdb40 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -105,6 +105,10 @@ def __init__( # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False + # Does this (non-extension) class have class decorators other than the ones native + # classes support? These can replace the class with an arbitrary object, such as a + # functools.cache wrapper, so the name of the class may not refer to the class. + self.is_decorated = False # Does this inherit from a Python class? self.inherits_python = False # Do instances of this class have __dict__? @@ -480,6 +484,7 @@ def serialize(self) -> JsonDict: "is_abstract": self.is_abstract, "is_generated": self.is_generated, "is_augmented": self.is_augmented, + "is_decorated": self.is_decorated, "is_final_class": self.is_final_class, "inherits_python": self.inherits_python, "has_dict": self.has_dict, @@ -550,6 +555,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.is_abstract = data["is_abstract"] ir.is_ext_class = data["is_ext_class"] ir.is_augmented = data["is_augmented"] + ir.is_decorated = data["is_decorated"] ir.is_final_class = data["is_final_class"] ir.inherits_python = data["inherits_python"] ir.has_dict = data["has_dict"] diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 957cc27044410..1b61cc0cd7444 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -1250,6 +1250,14 @@ def is_synthetic_type(self, typ: TypeInfo) -> bool: """Is a type something other than just a class we've created?""" return typ.is_named_tuple or typ.is_newtype or typ.typeddict_type is not None + def is_decorated_class(self, typ: TypeInfo) -> bool: + """Might a class decorator have replaced the class with another object? + + The name of the class then refers to the decorated object, not to the class. + """ + ir = self.mapper.type_to_ir.get(typ) + return ir is not None and ir.is_decorated + def get_final_ref(self, expr: MemberExpr) -> tuple[str, Var, bool] | None: """Check if `expr` is a final class, module or instance attribute. @@ -1755,6 +1763,7 @@ def load_global(self, expr: NameExpr) -> Value: self.is_native_module_ref_expr(expr) and isinstance(expr.node, TypeInfo) and not self.is_synthetic_type(expr.node) + and not self.is_decorated_class(expr.node) ): assert expr.fullname return self.load_native_type_object(expr.fullname) diff --git a/mypyc/irbuild/classdef.py b/mypyc/irbuild/classdef.py index 3eb617cf86550..05d196fb9dd64 100644 --- a/mypyc/irbuild/classdef.py +++ b/mypyc/irbuild/classdef.py @@ -76,6 +76,7 @@ from mypyc.primitives.misc_ops import ( dataclass_sleight_of_hand, import_op, + isinstance_type, not_implemented_op, py_calc_meta_op, py_init_subclass_op, @@ -287,23 +288,30 @@ def add_attr(self, lvalue: NameExpr, stmt: AssignmentStmt) -> None: def finalize(self, ir: ClassIR) -> None: # Dynamically create the class via the type constructor non_ext_class = load_non_ext_class(self.builder, ir, self.non_ext, self.cdef.line) - non_ext_class = load_decorated_class(self.builder, self.cdef, non_ext_class) + decorated_class = load_decorated_class(self.builder, self.cdef, non_ext_class) + type_obj = decorated_class + if ir.is_decorated: + type_obj = load_decorated_class_type( + self.builder, non_ext_class, decorated_class, self.cdef.line + ) # Try to avoid contention when using free threading. - self.builder.set_immortal_if_free_threaded(non_ext_class, self.cdef.line) + self.builder.set_immortal_if_free_threaded(type_obj, self.cdef.line) + if type_obj is not decorated_class: + self.builder.set_immortal_if_free_threaded(decorated_class, self.cdef.line) - # Save the decorated class + # Save the class object self.builder.add( - InitStatic(non_ext_class, self.cdef.name, self.builder.module_name, NAMESPACE_TYPE) + InitStatic(type_obj, self.cdef.name, self.builder.module_name, NAMESPACE_TYPE) ) - # Add the non-extension class to the dict + # Add the decorated class to the dict self.builder.call_c( exact_dict_set_item_op, [ self.builder.load_globals_dict(), self.builder.load_str(self.cdef.name), - non_ext_class, + decorated_class, ], self.cdef.line, ) @@ -933,6 +941,27 @@ def load_decorated_class(builder: IRBuilder, cdef: ClassDef, type_obj: Value) -> return dec_class +def load_decorated_class_type( + builder: IRBuilder, undecorated: Value, decorated: Value, line: int +) -> Value: + """Find the class object to use in type checks of a decorated non-extension class. + + A class decorator can replace the class with an object that isn't a class, such as + a functools.cache wrapper. Instances still have the undecorated class as their type + then, so use it. If the decorator returned a class (perhaps a new one, as + six.add_metaclass does), use that. + """ + result = Register(object_rprimitive) + builder.assign(result, decorated, line) + is_class = builder.primitive_op(isinstance_type, [decorated], line) + not_class, done = BasicBlock(), BasicBlock() + builder.add(Branch(is_class, done, not_class, Branch.BOOL)) + builder.activate_block(not_class) + builder.assign(result, undecorated, line) + builder.goto_and_activate(done) + return result + + def cache_class_attrs( builder: IRBuilder, attrs_to_cache: list[tuple[Lvalue, RType]], cdef: ClassDef ) -> None: diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index bf9b81532d54e..f2e8141b3f41d 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -78,6 +78,7 @@ get_mypyc_attrs, is_class_body_final, is_dataclass, + is_decorated_class, is_extension_class, is_trait, ) @@ -110,6 +111,7 @@ def build_type_map( is_final_class=cdef.info.is_final, ) class_ir.is_ext_class = is_extension_class(module.path, cdef, errors) + class_ir.is_decorated = is_decorated_class(cdef) if class_ir.is_ext_class: class_ir.deletable = cdef.info.deletable_attributes.copy() # If global optimizations are disabled, turn of tracking of class children diff --git a/mypyc/irbuild/util.py b/mypyc/irbuild/util.py index a2378fa55bcc1..85e4d499f3b0e 100644 --- a/mypyc/irbuild/util.py +++ b/mypyc/irbuild/util.py @@ -304,21 +304,30 @@ def get_explicit_native_class(path: str, cdef: ClassDef, errors: Errors) -> bool return None +def is_decorated_class(cdef: ClassDef) -> bool: + """Does a class have decorators other than the ones native classes support? + + Only non-extension classes can have them. Unlike the supported decorators, which + return a class, these can replace the class with an arbitrary object. + """ + return any( + not is_trait_decorator(d) + and not is_dataclass_decorator(d) + and not get_mypyc_attr_call(d) + and not is_final_decorator(d) + for d in cdef.decorators + ) + + def is_implicit_extension_class(cdef: ClassDef) -> tuple[bool, str]: """Check if class can be extension class and return a user-friendly reason it can't be one.""" - for d in cdef.decorators: - if ( - not is_trait_decorator(d) - and not is_dataclass_decorator(d) - and not get_mypyc_attr_call(d) - and not is_final_decorator(d) - ): - return ( - False, - "Classes that have decorators other than supported decorators" - " can't be native classes.", - ) + if is_decorated_class(cdef): + return ( + False, + "Classes that have decorators other than supported decorators" + " can't be native classes.", + ) if cdef.info.typeddict_type: return False, "TypedDict classes can't be native classes." diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index a08bfb61646e9..e86164a5007e8 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1498,6 +1498,51 @@ class M(type): # E: Inheriting from most builtin types is unimplemented \ class A(metaclass=M): # E: Class is marked as native_class=True but it can't be a native class. Classes with a metaclass other than ABCMeta, TypingMeta or GenericMeta can't be native classes. pass +[case testDecoratedNonExtClassReference] +from typing import Any + +def decorator(cls: Any) -> Any: + return cls + +@decorator +class C: + pass + +def f() -> C: + return C() + +def g(x: object) -> bool: + return isinstance(x, C) +[out] +def decorator(cls): + cls :: object +L0: + return cls +def f(): + r0 :: dict + r1 :: str + r2, r3 :: object + r4 :: __main__.C +L0: + r0 = __main__.globals :: static + r1 = 'C' + r2 = CPyDict_GetItem(r0, r1) + r3 = PyObject_Vectorcall(r2, 0, 0, 0) + r4 = cast(__main__.C, r3) + return r4 +def g(x): + x, r0 :: object + r1 :: ptr + r2 :: object + r3 :: bit +L0: + r0 = __main__.C :: type + r1 = get_element_ptr x ob_type :: PyObject + r2 = borrow load_mem r1 :: builtins.object* + keep_alive x + r3 = r2 == r0 + return r3 + [case testReservedName] from typing import Any, overload diff --git a/mypyc/test-data/run-classes.test b/mypyc/test-data/run-classes.test index 77f029b6b80b5..44b710770c5e6 100644 --- a/mypyc/test-data/run-classes.test +++ b/mypyc/test-data/run-classes.test @@ -314,6 +314,106 @@ assert o.get(20) == 20 assert get_class_var() == 'xy' +[case testNonExtClassDecoratorReturningNonClass] +from functools import cache +from typing import Any, Callable, TypeVar, cast + +from mypy_extensions import mypyc_attr + +T = TypeVar("T") + +@cache +class Group: + size = 10 + + def __init__(self, d: int) -> None: + self.d = d + + def order(self) -> int: + return self.d * 2 + + def same(self, other: Group) -> bool: + return self is other + +def make(d: int) -> Group: + return Group(d) + +def order(g: Group) -> int: + return g.order() + +@mypyc_attr(native_class=False) +class Base: + def __init__(self, x: int) -> None: + self.x = x + +@cache +class Derived(Base): + def __init__(self, x: int) -> None: + super().__init__(x) + self.y = x + 1 + +def singleton(cls: Callable[[], T]) -> Callable[[], T]: + instances: list[T] = [] + + def get() -> T: + if not instances: + instances.append(cls()) + return instances[0] + + return get + +@singleton +class Config: + def __init__(self) -> None: + self.value = 5 + + def get_value(self) -> int: + return self.value + +def replace_class(cls: Any) -> Any: + # Return a new class with the same namespace, like six.add_metaclass does + ns = {k: v for k, v in cls.__dict__.items() if k not in ("__dict__", "__weakref__")} + return type(cls)(cls.__name__, cls.__bases__, ns) + +@replace_class +class Replaced: + def __init__(self, v: int) -> None: + self.v = v + + def get(self) -> int: + return self.v + +def make_replaced(v: int) -> Replaced: + return Replaced(v) + +def test_cache() -> None: + g = make(3) + assert g.order() == 6 + assert order(g) == 6 + assert g is make(3) + assert g is Group(3) + assert g is not make(4) + assert g.same(Group(3)) + assert Group.size == 10 + assert type(Group).__name__ == "_lru_cache_wrapper" + assert cast(Any, Group).cache_info().currsize == 2 + +def test_super() -> None: + d = Derived(4) + assert d.x == 4 + assert d.y == 5 + assert d is Derived(4) + +def test_singleton() -> None: + c = Config() + assert c.get_value() == 5 + assert c is Config() + +def test_decorator_returning_class() -> None: + r = make_replaced(5) + assert r.get() == 5 + assert type(r) is Replaced + [case testEnum] from enum import Enum diff --git a/test-data/unit/lib-stub/functools.pyi b/test-data/unit/lib-stub/functools.pyi index b8a15fe60b742..8fc68813062a0 100644 --- a/test-data/unit/lib-stub/functools.pyi +++ b/test-data/unit/lib-stub/functools.pyi @@ -23,6 +23,11 @@ def singledispatch(func: Callable[..., _T]) -> _SingleDispatchCallable[_T]: ... def total_ordering(cls: type[_T]) -> type[_T]: ... +class _lru_cache_wrapper(Generic[_T]): + def __call__(self, *args: Any, **kwargs: Any) -> _T: ... + +def cache(user_function: Callable[..., _T], /) -> _lru_cache_wrapper[_T]: ... + class cached_property(Generic[_T]): func: Callable[[Any], _T] attrname: str | None From 8eb85d271f017b119e885d8b888e7bc40e168b26 Mon Sep 17 00:00:00 2001 From: Ryan Heard Date: Wed, 7 Oct 2026 13:59:22 -0400 Subject: [PATCH 2/2] [mypyc] Recompile dependents when a class decorator changes --- mypyc/codegen/emitmodule.py | 29 ++++++-- mypyc/test-data/run-multimodule.test | 105 +++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1b21b51709db6..38632b4004585 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -23,7 +23,7 @@ ) from mypy.errors import CompileError from mypy.fscache import FileSystemCache -from mypy.nodes import MypyFile, TypeInfo +from mypy.nodes import ClassDef, MypyFile, TypeInfo from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps @@ -82,6 +82,7 @@ from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map +from mypyc.irbuild.util import is_decorated_class from mypyc.namegen import NameGenerator, exported_name from mypyc.options import CompilerOptions from mypyc.transform.borrow_generator_attrs import borrow_generator_attrs @@ -139,6 +140,10 @@ class MypycPlugin(Plugin): * If the IR metadata is missing or stale or any of the generated C source files associated missing or stale, then we need to recompile the module so we mark it as stale. + * Mypy ignores class decorators, but a decorator that can replace + a class changes how other modules refer to the class (see + ClassIR.is_decorated), so we make such decorated classes part + of the module's interface. """ def __init__( @@ -154,8 +159,11 @@ def __init__( self.compiler_options = compiler_options self.metastore = create_metastore(options, parallel_worker=False) - def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list[str]] | None: - # The config data we report is the group map entry for the module. + def report_config_data( + self, ctx: ReportConfigContext + ) -> tuple[str | None, list[str], list[str]] | None: + # The config data we report is the group map entry for the module, + # followed by the names of its decorated classes. # If the data is being used to check validity, we do additional checks # that the IR cache exists and matches the metadata cache and all # output source files exist and are up to date. @@ -167,7 +175,8 @@ def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list # If we aren't doing validity checks, just return the cache data if not is_check: - return self.group_map[id] + assert self._modules is not None + return (*self.group_map[id], decorated_class_names(self._modules[id])) # Load the metadata and IR cache meta_path, _, _ = get_cache_names(id, path, self.options) @@ -201,7 +210,10 @@ def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list if hash != real_hash: return None - return self.group_map[id] + # The module hasn't been parsed yet, so take the decorated classes + # from the cached IR. It was built from the source the cache is for. + decorated = sorted(c["name"] for c in ir_data["ir"]["classes"] if c["is_decorated"]) + return (*self.group_map[id], decorated) def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]: # Report dependency on modules in the module's group @@ -230,6 +242,13 @@ def get_additional_indirect_deps(self, file: MypyFile) -> set[str]: return mods +def decorated_class_names(tree: MypyFile) -> list[str]: + """Names of the classes that build_type_map sets ClassIR.is_decorated for.""" + return sorted( + node.name for node in tree.defs if isinstance(node, ClassDef) and is_decorated_class(node) + ) + + def parse_and_typecheck( sources: list[BuildSource], options: Options, diff --git a/mypyc/test-data/run-multimodule.test b/mypyc/test-data/run-multimodule.test index baab2f52ce08f..ee726db69eb97 100644 --- a/mypyc/test-data/run-multimodule.test +++ b/mypyc/test-data/run-multimodule.test @@ -1442,6 +1442,111 @@ import native [out2] 5 +[case testIncrementalCompilationAddingNonClassDecorator] +# Mypy ignores class decorators, but adding one that replaces the class must +# recompile native.py, since C then no longer refers to the class itself. +from typing import Any, cast +from other import C + +def same_instance() -> bool: + cls = cast(Any, C) + return cls() is cls() + +[file other.py] +from functools import cache +from mypy_extensions import mypyc_attr + +@mypyc_attr(native_class=False) +class C: + pass + +[file other.py.2] +from functools import cache +from mypy_extensions import mypyc_attr + +@cache +@mypyc_attr(native_class=False) +class C: + pass + +[file other.py.3] +from functools import cache +from mypy_extensions import mypyc_attr + +@mypyc_attr(native_class=False) +class C: + pass + +[file driver.py] +from native import same_instance +from other import C + +print(same_instance(), C() is C()) +[out] +False False +[out2] +True True +[out3] +False False + +[rechecked native, other] +[rechecked2 native, other] + +[case testIncrementalCompilationAddingNonClassDecoratorToNativeClass] +# The decorator also makes C a non-native class, which changes how native.py +# constructs it. In step 3 other.py is unchanged, so it must stay fresh. +from other import C + +def get() -> int: + return C(3).get() + +def same_instance() -> bool: + return C(1) is C(1) + +[file native.py.3] +from other import C + +def get() -> int: + return C(4).get() + +def same_instance() -> bool: + return C(1) is C(1) + +[file other.py] +from functools import cache + +class C: + def __init__(self, x: int) -> None: + self.x = x + + def get(self) -> int: + return self.x + +[file other.py.2] +from functools import cache + +@cache +class C: + def __init__(self, x: int) -> None: + self.x = x + + def get(self) -> int: + return self.x + +[file driver.py] +from native import get, same_instance + +print(get(), same_instance()) +[out] +3 False +[out2] +3 True +[out3] +4 True + +[rechecked native, other] +[rechecked2 native] + [case testIncrementalCompilationWithNonClassTypeDef] import other_a [file other_a.py]