gh-92041: Avoid repeated sys.modules scans in inspect.getmodule

Include/internal/pycore_interp_structs.h

@@ -319,6 +319,8 @@ struct _import_runtime_state {
 struct _import_state {
     /* cached sys.modules dictionary */
     PyObject *modules;
+    /* Incremented when sys.modules or module lookup metadata changes. */
+    uint64_t module_registry_version;
     /* This is the list of module objects for all legacy (single-phase init)
        extension modules ever loaded in this process (i.e. imported
        in this interpreter or in any other).  Py_None stands in for

Lib/inspect.py

@@ -143,6 +143,7 @@
 
 
 import abc
+import _imp
 from annotationlib import Format, ForwardRef
 from annotationlib import get_annotations  # re-exported
 import ast
@@ -925,39 +926,287 @@ def getabsfile(object, _filename=None):
 
 modulesbyfile = {}
 _filesbymodname = {}
+# (registry version, filename map, per-module metadata, volatile metadata)
+_modulesbyfile_cache = (None, {}, {}, ())
+_module_cache_missing = object()
+
+
+def _get_volatile_module_metadata(module):
+    """Return unwatchable module metadata used to validate the cache."""
+    try:
+        module_file = getattr(module, '__file__')
+    except AttributeError:
+        return _module_cache_missing, _module_cache_missing
+    if module_file is None:
+        return None, _module_cache_missing
+    return module_file, getattr(module, '__name__')
+
+
+def _volatile_modules_unchanged(modules, volatile_modules):
+    for modname, module_file, module_name in volatile_modules:
+        module = modules.get(modname, _module_cache_missing)
+        if not ismodule(module):
+            return False
+        current_file, current_name = _get_volatile_module_metadata(module)
+        if current_file != module_file or current_name != module_name:
+            return False
+    return True
+
+
+def _module_cache_is_current(modules, version, cache):
+    if (modules is not sys.modules or
+            version is None or version != cache[0]):
+        return False
+    volatile_modules = cache[3]
+    if not volatile_modules:
+        return True
+    if not _volatile_modules_unchanged(modules, volatile_modules):
+        return False
+    # Dynamic metadata may execute arbitrary Python code.
+    return (modules is sys.modules and
+            version == _imp._get_module_registry_version(modules))
+
+
+def _publish_module_cache(version, modules_by_file, files_by_module,
+                          volatile_modules):
+    """Publish a consistent filename cache and update its legacy mirrors."""
+    global _modulesbyfile_cache
+
+    # Keep these longstanding, externally visible dictionaries alive, but do
+    # not use them as the authoritative cache: updating two dictionaries and a
+    # version cannot be made atomic for concurrent getmodule() calls.
+    modulesbyfile.clear()
+    modulesbyfile.update(modules_by_file)
+    _filesbymodname.clear()
+    _filesbymodname.update(
+        (modname, metadata[0])
+        for modname, metadata in files_by_module.items()
+    )
+    # This single assignment is the authoritative publication point.
+    _modulesbyfile_cache = (
+        version, modules_by_file, files_by_module, volatile_modules)
+
+
+def _getframemodule(frame, _filename=None):
+    """Return a module owned by a frame's globals, or None if not found."""
+    frame_globals = frame.f_globals
+    module_name = frame_globals.get('__name__')
+    if not isinstance(module_name, str):
+        return None
+    module = sys.modules.get(module_name)
+    if not (ismodule(module) and module.__dict__ is frame_globals):
+        return None
+    try:
+        if type(module) is types.ModuleType:
+            module_file = module.__file__
+        else:
+            # A module subclass can supply dynamic metadata through
+            # descriptors. getattr() avoids a module-attribute specialization
+            # that may read the subclass's underlying module dict.
+            module_file = getattr(module, '__file__')
+    except AttributeError:
+        return None
+    if module_file is None:
+        return None
+    frame_filename = (
+        frame.f_code.co_filename if _filename is None else _filename
+    )
+    try:
+        file = getabsfile(frame, frame_filename)
+    except (TypeError, FileNotFoundError):
+        return None
+    if frame_filename == module_file:
+        return module
+    try:
+        module_file = getabsfile(module, module_file)
+    except (TypeError, FileNotFoundError):
+        return None
+    if file == module_file or file == os.path.realpath(module_file):
+        return module
+
 
 def getmodule(object, _filename=None):
     """Return the module an object was defined in, or None if not found."""
     if ismodule(object):
         return object
     if hasattr(object, '__module__'):
         return sys.modules.get(object.__module__)
+    if istraceback(object):
+        object = object.tb_frame
+    if isframe(object):
+        # Frame globals identify the execution namespace directly. Preserve
+        # the private filename override when it names a different file.
+        if _filename is None or _filename == object.f_code.co_filename:
+            module = _getframemodule(object, _filename)
+            if module is not None:
+                return module
+
+    modules = sys.modules
+    version = _imp._get_module_registry_version(modules)
+    cache = _modulesbyfile_cache
+    cache_is_current = _module_cache_is_current(modules, version, cache)
+    cached_modulesbyfile = cache[1]
+
+    # Try the filename to modulename cache. Recheck the version after reading
+    # the module so a concurrent registry mutation cannot make the result
+    # stale.
+    if (cache_is_current and _filename is not None and
+            _filename in cached_modulesbyfile):
+        module = modules.get(cached_modulesbyfile[_filename])
+        if _module_cache_is_current(
+                modules, _imp._get_module_registry_version(modules), cache):
+            return module
+        cache_is_current = False
 
-    # Try the filename to modulename cache
-    if _filename is not None and _filename in modulesbyfile:
-        return sys.modules.get(modulesbyfile[_filename])
     # Try the cache again with the absolute file name
     try:
         file = getabsfile(object, _filename)
     except (TypeError, FileNotFoundError):
         return None
-    if file in modulesbyfile:
-        return sys.modules.get(modulesbyfile[file])
-    # Update the filename to module name cache and check yet again
-    # Copy sys.modules in order to cope with changes while iterating
-    for modname, module in sys.modules.copy().items():
-        if ismodule(module) and hasattr(module, '__file__'):
-            f = module.__file__
-            if f == _filesbymodname.get(modname, None):
-                # Have already mapped this module, so skip it
-                continue
-            _filesbymodname[modname] = f
-            f = getabsfile(module)
-            # Always map to the name the module knows itself by
-            modulesbyfile[f] = modulesbyfile[
-                os.path.realpath(f)] = module.__name__
-    if file in modulesbyfile:
-        return sys.modules.get(modulesbyfile[file])
+    if modules is not sys.modules:
+        # getabsfile() may execute user code through dynamic module metadata.
+        modules = sys.modules
+        cache_is_current = False
+    if not cache_is_current:
+        # getabsfile() can recurse through getsourcefile() and populate this
+        # cache. Avoid rebuilding it a second time in the outer call.
+        version = _imp._get_module_registry_version(modules)
+        cache = _modulesbyfile_cache
+        cache_is_current = _module_cache_is_current(modules, version, cache)
+        cached_modulesbyfile = cache[1]
+    if cache_is_current and file in cached_modulesbyfile:
+        module = modules.get(cached_modulesbyfile[file])
+        if _module_cache_is_current(
+                modules, _imp._get_module_registry_version(modules), cache):
+            return module
+        cache_is_current = False
+
+    if cache_is_current:
+        # A stable cache miss is a negative-cache hit. Validate it after the
+        # lookup just as for a positive hit.
+        cache_is_current = _module_cache_is_current(
+            modules, _imp._get_module_registry_version(modules), cache)
+
+    # Update the filename to module name cache when the canonical module
+    # registry or relevant module metadata has changed. Replacement
+    # sys.modules mappings cannot be watched, so scan those on every miss.
+    if not cache_is_current:
+        # Retry once if a mutation overlaps the scan. This handles imports
+        # triggered by the first scan without spinning under sustained churn.
+        for _ in range(2):
+            # Copy sys.modules in order to cope with changes while iterating.
+            modules = sys.modules
+            version = _imp._get_module_registry_version(modules)
+            modules_copy = modules.copy()
+            new_modulesbyfile = {}
+            new_filesbymodname = {}
+            volatile_modules = []
+            cached_filesbymodname = _modulesbyfile_cache[2]
+            for modname, module in modules_copy.items():
+                if ismodule(module):
+                    if type(module) is types.ModuleType:
+                        module_dict = module.__dict__
+                        try:
+                            f = module.__file__
+                        except AttributeError:
+                            if '__getattr__' in module_dict:
+                                volatile_modules.append((
+                                    modname,
+                                    _module_cache_missing,
+                                    _module_cache_missing,
+                                ))
+                            continue
+                        volatile = (
+                            module_dict.get(
+                                '__file__', _module_cache_missing) is not f
+                        )
+                    else:
+                        # Module subclasses may resolve __file__ or __name__
+                        # dynamically. Validate just these entries on cache
+                        # hits instead of making every fallback rescan all of
+                        # sys.modules.
+                        module_dict = None
+                        volatile = True
+                        try:
+                            f = getattr(module, '__file__')
+                        except AttributeError:
+                            volatile_modules.append((
+                                modname,
+                                _module_cache_missing,
+                                _module_cache_missing,
+                            ))
+                            continue
+                    if f is None:
+                        if volatile:
+                            volatile_modules.append((
+                                modname, None, _module_cache_missing))
+                        continue
+                    if module_dict is None:
+                        module_name = getattr(module, '__name__')
+                    else:
+                        module_name = module.__name__
+                    if (module_dict is not None and
+                            module_dict.get(
+                                '__name__', _module_cache_missing)
+                            is not module_name):
+                        volatile = True
+                    if volatile:
+                        volatile_modules.append(
+                            (modname, f, module_name))
+                    cached = cached_filesbymodname.get(modname)
+                    if (isinstance(cached, tuple) and len(cached) == 4 and
+                            f == cached[0] and module_name == cached[1]):
+                        _, _, absfile, realpath = cached
+                    else:
+                        # Canonicalize the same metadata value that was
+                        # validated above.  Re-reading __file__ here can
+                        # observe a different dynamic value.
+                        absfile = getabsfile(module, f)
+                        realpath = os.path.realpath(absfile)
+                    new_filesbymodname[modname] = (
+                        f, module_name, absfile, realpath)
+                    # Always map to the name the module knows itself by.
+                    new_modulesbyfile[absfile] = new_modulesbyfile[
+                        realpath] = module_name
+            module = None
+            found = file in new_modulesbyfile
+            if found:
+                module = modules.get(new_modulesbyfile[file])
+            # Watcher callbacks run before their mutations. Validate both the
+            # watched generation and the small set of dynamic metadata before
+            # retaining the rebuilt caches.
+            current_version = _imp._get_module_registry_version(modules)
+            if version is not None:
+                if (modules is not sys.modules or
+                        version != current_version):
+                    continue
+                volatile_current = _volatile_modules_unchanged(
+                    modules, volatile_modules)
+                current_version = _imp._get_module_registry_version(modules)
+                if (modules is not sys.modules or
+                        version != current_version or
+                        not volatile_current):
+                    continue
+                _publish_module_cache(
+                    version,
+                    new_modulesbyfile,
+                    new_filesbymodname,
+                    tuple(volatile_modules),
+                )
+                if found:
+                    return module
+                break
+            if modules is sys.modules:
+                # A replacement sys.modules mapping cannot be watched, but its
+                # just-scanned result is still useful for this call.
+                _publish_module_cache(
+                    None, new_modulesbyfile, new_filesbymodname,
+                    tuple(volatile_modules))
+                if found:
+                    return module
+                break
+        # If both attempts overlap mutations, fall through rather than using
+        # a snapshot that could not be validated.
     # Check the main module
     main = sys.modules['__main__']
     if not hasattr(object, '__name__'):

Lib/test/libregrtest/utils.py

@@ -296,6 +296,7 @@ def clear_caches():
         inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear()
         inspect._filesbymodname.clear()
         inspect.modulesbyfile.clear()
+        inspect._modulesbyfile_cache = (None, {}, {}, ())
 
     try:
         importlib_metadata = sys.modules['importlib.metadata']