gh-159485: Fix warnings lock deadlock during interpreter shutdown

Author: ngoldbaum

Include/internal/pycore_lock.h

@@ -168,6 +168,9 @@ typedef struct {
     size_t level;
 } _PyRecursiveMutex;
 
+// All other threads that use the mutex must be barred from reattaching.
+extern void _PyRecursiveMutex_at_shutdown_reinit(_PyRecursiveMutex *m);
+
 PyAPI_FUNC(int) _PyRecursiveMutex_IsLockedByCurrentThread(_PyRecursiveMutex *m);
 PyAPI_FUNC(void) _PyRecursiveMutex_Lock(_PyRecursiveMutex *m);
 extern PyLockStatus _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags flags);

Include/internal/pycore_warnings.h

@@ -8,6 +8,8 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+extern void _PyWarnings_PrepareForFinalization(PyInterpreterState *interp);
+
 extern int _PyWarnings_InitState(PyInterpreterState *interp);
 
 extern PyObject* _PyWarnings_Init(void);

Lib/test/test_embed.py

@@ -507,6 +507,68 @@ def test_datetime_reset_strptime(self):
         out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
         self.assertEqual(out, '20000101\n' * INIT_LOOPS)
 
+    @threading_helper.requires_working_threading()
+    @unittest.skipUnless(_testinternalcapi, "requires _testinternalcapi")
+    def test_warnings_at_shutdown(self):
+        import_helper.import_module('_testcapi')
+        code = textwrap.dedent('''
+            import _testinternalcapi
+            import _warnings
+            import threading
+            import warnings
+            from test import support
+
+            warnings.simplefilter('ignore')
+            local = threading.local()
+            ready = threading.Event()
+
+            class WarnOnDelete:
+                def __del__(self):
+                    warnings.warn('during shutdown', ResourceWarning)
+                    if not ready.is_set():
+                        # The finalizing thread still owns all three levels.
+                        for _ in range(3):
+                            _warnings._release_lock()
+                        try:
+                            _warnings._release_lock()
+                        except RuntimeError:
+                            pass
+                        else:
+                            raise AssertionError('unexpected recursion depth')
+                    print('finalized')
+
+            def worker():
+                local.obj = WarnOnDelete()
+                _warnings._acquire_lock()
+                ready.set()
+                threading.Event().wait()
+
+            for _ in range(3):
+                _warnings._acquire_lock()
+            threading.Thread(target=worker, daemon=True).start()
+            for _ in support.sleeping_retry(4):
+                if _testinternalcapi.warnings_lock_has_waiters():
+                    break
+        ''')
+        release = textwrap.dedent('''
+            for _ in range(3):
+                _warnings._release_lock()
+            assert ready.wait(4), 'lock not acquired'
+        ''')
+        subinterp = (
+            'from test import support\n'
+            f'assert support.run_in_subinterp({code + release!r}) == 0'
+        )
+        # Cover Py_EndInterpreter(), then shut down with a waiter. Reuse the
+        # main interpreter's mutex and shut down with a daemon owning it.
+        # All three thread-local finalizers must warn.
+        out, err = self.run_embedded_interpreter(
+            "test_repeated_init_exec", subinterp + '\n' + code, code + release,
+            env=remove_python_envvars(),
+            timeout=10)
+        self.assertEqual(out, 'finalized\n' * 3)
+        self.assertEqual(err, '--- Loop #1 ---\n--- Loop #2 ---\n')
+
     def test_static_types_inherited_slots(self):
         script = textwrap.dedent("""
             import test.support

Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-14-41-28.gh-issue-159485.S3IOyb.rst

@@ -0,0 +1,2 @@
+Prevent a deadlock when emitting warnings during late interpreter shutdown
+if a daemon thread holds the warnings lock or is waiting to acquire it.

Modules/_testinternalcapi/test_lock.c

@@ -1,7 +1,9 @@
 // C Extension module to test pycore_lock.h API
 
 #include "parts.h"
+#include "pycore_interp.h"        // PyInterpreterState.warnings
 #include "pycore_lock.h"
+#include "pycore_pystate.h"      // _PyEval_StopTheWorld()
 #include "pycore_pythread.h"      // PyThread_get_thread_ident_ex()
 
 #include "clinic/test_lock.c.h"
@@ -549,7 +551,21 @@ test_lock_recursive(PyObject *self, PyObject *obj)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+warnings_lock_has_waiters(PyObject *self, PyObject *Py_UNUSED(ignored))
+{
+    PyInterpreterState *interp = PyInterpreterState_Get();
+    PyMutex *mutex = &interp->warnings.lock.mutex;
+    // A contender sets the parked bit before enqueuing, but cannot detach
+    // until it has enqueued. Stop the world to close that window on nogil.
+    _PyEval_StopTheWorld(interp);
+    uint8_t bits = _Py_atomic_load_uint8_relaxed(&mutex->_bits);
+    _PyEval_StartTheWorld(interp);
+    return PyBool_FromLong((bits & _Py_HAS_PARKED) != 0);
+}
+
 static PyMethodDef test_methods[] = {
+    {"warnings_lock_has_waiters", warnings_lock_has_waiters, METH_NOARGS},
     {"test_lock_basic", test_lock_basic, METH_NOARGS},
     {"test_lock_two_threads", test_lock_two_threads, METH_NOARGS},
     {"test_lock_counter", test_lock_counter, METH_NOARGS},

Python/_warnings.c

@@ -254,6 +254,13 @@ get_warnings_attr(PyInterpreterState *interp, PyObject *attr, int try_import)
     return obj;
 }
 
+void
+_PyWarnings_PrepareForFinalization(PyInterpreterState *interp)
+{
+    WarningsState *st = warnings_get_state(interp);
+    _PyRecursiveMutex_at_shutdown_reinit(&st->lock);
+}
+
 static inline void
 warnings_lock(PyInterpreterState *interp)
 {

Python/lock.c

@@ -427,6 +427,21 @@ _PyRecursiveMutex_LockTimed(_PyRecursiveMutex *m, PyTime_t timeout, _PyLockFlags
     return s;
 }
 
+void
+_PyRecursiveMutex_at_shutdown_reinit(_PyRecursiveMutex *m)
+{
+    // Preserve the finalizing thread's ownership and recursion depth.
+    if (_PyRecursiveMutex_IsLockedByCurrentThread(m)) {
+        m->mutex = (PyMutex){._bits = _Py_LOCKED};
+    }
+    else {
+        memset(m, 0, sizeof(*m));
+    }
+    // Remove parked waiters from the queue. After waking, they will hang
+    // when they try to reattach their thread state.
+    _PyParkingLot_UnparkAll(&m->mutex._bits);
+}
+
 void
 _PyRecursiveMutex_Unlock(_PyRecursiveMutex *m)
 {

Python/pylifecycle.c

@@ -2452,6 +2452,7 @@ _Py_Finalize(_PyRuntimeState *runtime)
     for (PyThreadState *p = list; p != NULL; p = p->next) {
         _PyThreadState_SetShuttingDown(p);
     }
+    _PyWarnings_PrepareForFinalization(tstate->interp);
     _PyEval_StartTheWorldAll(runtime);
     PyMutex_Unlock(&tstate->interp->ceval.pending.mutex);
 
@@ -2834,6 +2835,7 @@ Py_EndInterpreter(PyThreadState *tstate)
         _PyThreadState_SetShuttingDown(p);
     }
 
+    _PyWarnings_PrepareForFinalization(interp);
     _PyEval_StartTheWorldAll(interp->runtime);
     PyMutex_Unlock(&interp->ceval.pending.mutex);
     _PyThreadState_DeleteList(list, /*is_after_fork=*/0);