From bb63e8cd60017836f0f21ad006373c811428884e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 9 Oct 2026 16:05:34 +0100 Subject: [PATCH 1/5] gh-159041: Fix asyncio task tracking during thread state cleanup Move lingering asyncio tasks to the interpreter only after all finalizer-capable thread-state cleanup has completed. Also transfer tasks created by finalizers during interpreter shutdown. Add regression tests for both thread-state and interpreter cleanup paths. --- Lib/test/test_asyncio/test_tasks.py | 64 +++++++++++++++++++ ...-10-09-16-04-40.gh-issue-159041.PUP8bU.rst | 2 + Python/pystate.c | 26 ++++++-- 3 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index f460029aff839d8..b0311f6569cc1b6 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -1,5 +1,6 @@ """Tests for tasks.py.""" +import _thread import collections import contextlib import contextvars @@ -20,6 +21,7 @@ from asyncio import tasks from test.test_asyncio import utils as test_utils from test import support +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -3095,6 +3097,68 @@ class CTask_CFuture_Tests(BaseTaskTests, SetMethodsTest, all_tasks = getattr(tasks, '_c_all_tasks', None) current_task = staticmethod(getattr(tasks, '_c_current_task', None)) + @threading_helper.requires_working_threading() + def test_task_created_during_thread_state_clear(self): + loop = self.loop + task = None + var = contextvars.ContextVar('var') + task_context = contextvars.Context() + + async def noop(): + pass + + class CreatesTaskOnClear: + def __del__(self): + nonlocal task + task = loop.create_task(noop(), context=task_context) + + def create_finalizer(): + var.set(CreatesTaskOnClear()) + + handle = _thread.start_joinable_thread(create_finalizer) + handle.join(support.SHORT_TIMEOUT) + self.assertTrue(handle.is_done()) + self.assertIsNotNone(task) + + try: + self.assertEqual(self.all_tasks(loop), {task}) + finally: + loop.run_until_complete(task) + + def test_task_created_during_interpreter_clear(self): + code = """if 1: + import asyncio + import contextvars + import warnings + + async def noop(): + pass + + class Loop: + def get_debug(self): + return False + + def call_soon(self, callback, *args, context=None): + self.callback = callback + + class CreatesTaskOnClear: + def __init__(self): + self.task_type = asyncio.Task + self.coro = noop() + self.loop = Loop() + self.context = contextvars.Context() + + def __del__(self): + self.task_type( + self.coro, loop=self.loop, context=self.context) + + # Leave the only reference to the filters list in the warnings + # state, which is cleared after the final garbage collection. + warnings.filters.append(CreatesTaskOnClear()) + del warnings.filters + """ + assert_python_ok('-c', code) + def test_del__log_destroy_pending_segfault(self): async def coro(): pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst new file mode 100644 index 000000000000000..9f156c70870f5a8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst @@ -0,0 +1,2 @@ +Fix a use-after-free in :mod:`asyncio` task bookkeeping when a finalizer +creates a task while its thread state is being cleared. diff --git a/Python/pystate.c b/Python/pystate.c index 97323ea2ba2bce1..4ed8d7389f15277 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -812,6 +812,16 @@ common_constants_clear(PyInterpreterState *interp) } +static void +move_asyncio_tasks_to_interpreter(PyThreadState *tstate) +{ + PyMutex_Lock(&tstate->interp->asyncio_tasks_lock); + llist_concat(&tstate->interp->asyncio_tasks_head, + &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head); + PyMutex_Unlock(&tstate->interp->asyncio_tasks_lock); +} + + static void interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) { @@ -953,6 +963,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) if (tstate->interp == interp) { /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? + // Finalizers above may have registered new asyncio tasks. + move_asyncio_tasks_to_interpreter(tstate); + assert(llist_empty( + &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head)); tstate->_status.cleared = 1; } @@ -1837,13 +1851,6 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_task); - PyMutex_Lock(&tstate->interp->asyncio_tasks_lock); - // merge any lingering tasks from thread state to interpreter's - // tasks list - llist_concat(&tstate->interp->asyncio_tasks_head, - &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head); - PyMutex_Unlock(&tstate->interp->asyncio_tasks_lock); - Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); @@ -1910,6 +1917,11 @@ PyThreadState_Clear(PyThreadState *tstate) _PyJit_TracerFree((_PyThreadStateImpl *)tstate); #endif + // Merge any lingering tasks from the thread state to the interpreter's + // tasks list. This must happen after all cleanup which can run finalizers, + // since those finalizers may create and register new tasks. + move_asyncio_tasks_to_interpreter(tstate); + tstate->_status.cleared = 1; // XXX Call _PyThreadStateSwap(runtime, NULL) here if "current". From ddf0d98fec96e7f5420dee341644beadea0cd289 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 9 Oct 2026 17:00:55 +0100 Subject: [PATCH 2/5] gh-159041: Strengthen asyncio task cleanup checks --- Lib/test/test_asyncio/test_tasks.py | 7 ++++++- Python/pystate.c | 16 +++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index b0311f6569cc1b6..f8063e9619a7116 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3102,6 +3102,7 @@ def test_task_created_during_thread_state_clear(self): loop = self.loop task = None var = contextvars.ContextVar('var') + # Avoid recreating the thread state's context after it is cleared. task_context = contextvars.Context() async def noop(): @@ -3113,11 +3114,15 @@ def __del__(self): task = loop.create_task(noop(), context=task_context) def create_finalizer(): + # gh-159041: Run the finalizer when PyThreadState_Clear() + # clears the thread's context. var.set(CreatesTaskOnClear()) + # threading.Thread owns its Context, so use _thread to make the + # Context lifetime match the thread state's lifetime. handle = _thread.start_joinable_thread(create_finalizer) handle.join(support.SHORT_TIMEOUT) - self.assertTrue(handle.is_done()) + self.assertTrue(handle.is_done(), 'thread failed to exit') self.assertIsNotNone(task) try: diff --git a/Python/pystate.c b/Python/pystate.c index 4ed8d7389f15277..5706ddb45b7ad20 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,9 +815,14 @@ common_constants_clear(PyInterpreterState *interp) static void move_asyncio_tasks_to_interpreter(PyThreadState *tstate) { + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + if (llist_empty(&tstate_impl->asyncio_tasks_head)) { + return; + } + PyMutex_Lock(&tstate->interp->asyncio_tasks_lock); llist_concat(&tstate->interp->asyncio_tasks_head, - &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head); + &tstate_impl->asyncio_tasks_head); PyMutex_Unlock(&tstate->interp->asyncio_tasks_lock); } @@ -961,12 +966,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #endif if (tstate->interp == interp) { + // gh-159041: Finalizers above may have registered new asyncio tasks. + move_asyncio_tasks_to_interpreter(tstate); + /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? - // Finalizers above may have registered new asyncio tasks. - move_asyncio_tasks_to_interpreter(tstate); - assert(llist_empty( - &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head)); tstate->_status.cleared = 1; } @@ -1936,6 +1940,8 @@ static void tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); + assert(llist_empty( + &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head)); tstate_verify_not_active(tstate); assert(!_PyThreadState_IsRunningMain(tstate)); From a2a41a3850889835d97e8704c66123b2bc64e0a2 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 16 Oct 2026 09:49:32 -0600 Subject: [PATCH 3/5] gh-159041: Transfer asyncio tasks after late interpreter cleanup --- Lib/test/test_asyncio/test_tasks.py | 34 ------------- Lib/test/test_embed.py | 51 +++++++++++++++++++ ...-10-09-16-04-40.gh-issue-159041.PUP8bU.rst | 2 +- Programs/_testembed.c | 31 +++++++++++ Python/pystate.c | 7 +-- 5 files changed, 87 insertions(+), 38 deletions(-) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index f8063e9619a7116..95e1f760123b887 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -3130,40 +3130,6 @@ def create_finalizer(): finally: loop.run_until_complete(task) - def test_task_created_during_interpreter_clear(self): - code = """if 1: - import asyncio - import contextvars - import warnings - - async def noop(): - pass - - class Loop: - def get_debug(self): - return False - - def call_soon(self, callback, *args, context=None): - self.callback = callback - - class CreatesTaskOnClear: - def __init__(self): - self.task_type = asyncio.Task - self.coro = noop() - self.loop = Loop() - self.context = contextvars.Context() - - def __del__(self): - self.task_type( - self.coro, loop=self.loop, context=self.context) - - # Leave the only reference to the filters list in the warnings - # state, which is cleared after the final garbage collection. - warnings.filters.append(CreatesTaskOnClear()) - del warnings.filters - """ - assert_python_ok('-c', code) - def test_del__log_destroy_pending_segfault(self): async def coro(): pass diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 39485afcdb0f57d..fbf97928643a3f0 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -2005,6 +2005,57 @@ def test_open_code_hook(self): def test_audit(self): self.run_embedded_interpreter("test_audit") + def test_asyncio_tasks_during_shutdown(self): + import_helper.import_module('_asyncio') + code = textwrap.dedent(""" + import asyncio + import contextvars + import os + import warnings + + async def noop(): + pass + + class Loop: + debug = False + + def get_debug(self): + return self.debug + + def call_soon(self, callback, *args, context=None): + self.callback = callback + return callback + + class CreatesTaskOnClear: + def __init__(self, message): + self.task_type = asyncio.Task + self.coro = noop() + self.loop = Loop() + self.context = contextvars.Context() + self.write = os.write + self.message = message + + def __del__(self): + self.task_type( + self.coro, loop=self.loop, context=self.context) + return self.write(1, self.message) + + # Release one finalizer when interpreter_clear() clears warnings. + warnings.filters.append(CreatesTaskOnClear(b'warnings cleared\\n')) + del warnings.filters + + # The native audit hook releases this one after interpreter_clear(). + finalizer = CreatesTaskOnClear(b'audit hooks cleared\\n') + """) + # gh-159041: Native audit hooks are cleared after interpreter_clear(). + # The methods above avoid loading common constants (including an + # implicit return None), which have already been cleared at that point. + out, err = self.run_embedded_interpreter( + "test_audit_clear_finalizer", code, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(out, "warnings cleared\naudit hooks cleared\n") + self.assertEqual(err, "") + def test_audit_tuple(self): self.run_embedded_interpreter("test_audit_tuple") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst index 9f156c70870f5a8..7d10d102aeae29a 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst @@ -1,2 +1,2 @@ Fix a use-after-free in :mod:`asyncio` task bookkeeping when a finalizer -creates a task while its thread state is being cleared. +creates a task during thread or interpreter shutdown. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 592c23756201f01..648deabc2ecd623 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1304,6 +1304,36 @@ static int test_audit(void) return result; } +static int +audit_hook_clear_finalizer(const char *event, PyObject *args, void *userdata) +{ + if (strcmp(event, "cpython._PySys_ClearAuditHooks") == 0) { + PyObject **finalizer = userdata; + Py_CLEAR(*finalizer); + } + return 0; +} + +static int test_audit_clear_finalizer(void) +{ + assert(main_argc == 3); + _testembed_initialize(); + if (PyRun_SimpleString(main_argv[2]) < 0) { + return 1; + } + + PyObject *main_module = PyImport_AddModule("__main__"); + assert(main_module != NULL); + PyObject *finalizer = PyObject_GetAttrString(main_module, "finalizer"); + assert(finalizer != NULL); + assert(PyObject_DelAttrString(main_module, "finalizer") == 0); + assert(PySys_AddAuditHook(audit_hook_clear_finalizer, &finalizer) == 0); + + int result = Py_FinalizeEx(); + assert(finalizer == NULL); + return result; +} + static int test_audit_tuple(void) { #define ASSERT(TEST, EXITCODE) \ @@ -3011,6 +3041,7 @@ static struct TestCase TestCases[] = { // Audit {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, + {"test_audit_clear_finalizer", test_audit_clear_finalizer}, {"test_audit_tuple", test_audit_tuple}, {"test_audit_subinterpreter", test_audit_subinterpreter}, {"test_audit_run_command", test_audit_run_command}, diff --git a/Python/pystate.c b/Python/pystate.c index 5706ddb45b7ad20..962d1c6cfa5b661 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -966,9 +966,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #endif if (tstate->interp == interp) { - // gh-159041: Finalizers above may have registered new asyncio tasks. - move_asyncio_tasks_to_interpreter(tstate); - /* We are now safe to fix tstate->_status.cleared. */ // XXX Do this (much) earlier? tstate->_status.cleared = 1; @@ -1039,6 +1036,10 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // we start finalizing the interpreter (or the current thread state). PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { + // Finalizers and C audit hooks can create tasks after interpreter_clear(). + // Transfer them while the current thread state is still attached. + move_asyncio_tasks_to_interpreter(tcur); + /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); } From f37e91cb07c1f043f113ad77b00a0fbc648aebb8 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 16 Oct 2026 11:32:23 -0600 Subject: [PATCH 4/5] gh-159041: Fix asyncio task tracking during detached interpreter deletion --- Lib/test/test_embed.py | 90 ++++++++----------- ...-10-09-16-04-40.gh-issue-159041.PUP8bU.rst | 2 +- Programs/_testembed.c | 49 ++++------ Python/pystate.c | 6 +- 4 files changed, 60 insertions(+), 87 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d3efae2cd2b7ecd..0f6e7ec8004ff42 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -226,6 +226,45 @@ def run_repeated_init_and_subinterpreters(self): class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 100 * 50 + def test_asyncio_task_during_interpreter_clear(self): + import_helper.import_module('_asyncio') + code = textwrap.dedent(""" + import asyncio + import contextvars + import gc + import os + + gc.disable() + + async def noop(): + pass + + class Loop: + def get_debug(self): + return False + + def call_soon(self, callback, *args, context=None): + self.callback = callback + + class CreatesTaskOnClear: + def __del__(self, task_type=asyncio.Task, coro=noop, + loop=Loop(), context=contextvars.Context(), + write=os.write): + task_type(coro(), loop=loop, context=context) + write(1, b'task created\\n') + + # Keep the finalizer alive until interpreter_clear()'s last GC. + finalizer = CreatesTaskOnClear() + finalizer.cycle = finalizer + del finalizer + """) + # gh-159041: Delete the cleared interpreter with no thread state attached. + out, err = self.run_embedded_interpreter( + "test_interpreter_clear_and_delete", code, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(out, "task created\n") + self.assertEqual(err, "") + def test_subinterps_main(self): for run in self.run_repeated_init_and_subinterpreters(): main = run[0] @@ -2009,57 +2048,6 @@ def test_open_code_hook(self): def test_audit(self): self.run_embedded_interpreter("test_audit") - def test_asyncio_tasks_during_shutdown(self): - import_helper.import_module('_asyncio') - code = textwrap.dedent(""" - import asyncio - import contextvars - import os - import warnings - - async def noop(): - pass - - class Loop: - debug = False - - def get_debug(self): - return self.debug - - def call_soon(self, callback, *args, context=None): - self.callback = callback - return callback - - class CreatesTaskOnClear: - def __init__(self, message): - self.task_type = asyncio.Task - self.coro = noop() - self.loop = Loop() - self.context = contextvars.Context() - self.write = os.write - self.message = message - - def __del__(self): - self.task_type( - self.coro, loop=self.loop, context=self.context) - return self.write(1, self.message) - - # Release one finalizer when interpreter_clear() clears warnings. - warnings.filters.append(CreatesTaskOnClear(b'warnings cleared\\n')) - del warnings.filters - - # The native audit hook releases this one after interpreter_clear(). - finalizer = CreatesTaskOnClear(b'audit hooks cleared\\n') - """) - # gh-159041: Native audit hooks are cleared after interpreter_clear(). - # The methods above avoid loading common constants (including an - # implicit return None), which have already been cleared at that point. - out, err = self.run_embedded_interpreter( - "test_audit_clear_finalizer", code, - timeout=support.SHORT_TIMEOUT) - self.assertEqual(out, "warnings cleared\naudit hooks cleared\n") - self.assertEqual(err, "") - def test_audit_tuple(self): self.run_embedded_interpreter("test_audit_tuple") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst index 7d10d102aeae29a..4dd3338e41a4df5 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-09-16-04-40.gh-issue-159041.PUP8bU.rst @@ -1,2 +1,2 @@ Fix a use-after-free in :mod:`asyncio` task bookkeeping when a finalizer -creates a task during thread or interpreter shutdown. +creates a task during thread shutdown. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 4e80a7570990888..7b0d88e4e51294d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -154,6 +154,23 @@ static void _testembed_initialize(void) } +static int test_interpreter_clear_and_delete(void) +{ + assert(main_argc == 3); + _testembed_initialize(); + if (PyRun_SimpleString(main_argv[2]) < 0) { + return 1; + } + + PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState_Clear(interp); + PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); + // The interpreter has already been deleted, so don't call Py_FinalizeEx(). + return 0; +} + + static int test_import_in_subinterpreters(void) { _testembed_initialize(); @@ -1304,36 +1321,6 @@ static int test_audit(void) return result; } -static int -audit_hook_clear_finalizer(const char *event, PyObject *args, void *userdata) -{ - if (strcmp(event, "cpython._PySys_ClearAuditHooks") == 0) { - PyObject **finalizer = userdata; - Py_CLEAR(*finalizer); - } - return 0; -} - -static int test_audit_clear_finalizer(void) -{ - assert(main_argc == 3); - _testembed_initialize(); - if (PyRun_SimpleString(main_argv[2]) < 0) { - return 1; - } - - PyObject *main_module = PyImport_AddModule("__main__"); - assert(main_module != NULL); - PyObject *finalizer = PyObject_GetAttrString(main_module, "finalizer"); - assert(finalizer != NULL); - assert(PyObject_DelAttrString(main_module, "finalizer") == 0); - assert(PySys_AddAuditHook(audit_hook_clear_finalizer, &finalizer) == 0); - - int result = Py_FinalizeEx(); - assert(finalizer == NULL); - return result; -} - static int test_audit_tuple(void) { #define ASSERT(TEST, EXITCODE) \ @@ -3033,6 +3020,7 @@ static struct TestCase TestCases[] = { {"test_repeated_init_exec", test_repeated_init_exec}, {"test_repeated_simple_init", test_repeated_simple_init}, {"test_forced_io_encoding", test_forced_io_encoding}, + {"test_interpreter_clear_and_delete", test_interpreter_clear_and_delete}, {"test_import_in_subinterpreters", test_import_in_subinterpreters}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab}, @@ -3091,7 +3079,6 @@ static struct TestCase TestCases[] = { // Audit {"test_open_code_hook", test_open_code_hook}, {"test_audit", test_audit}, - {"test_audit_clear_finalizer", test_audit_clear_finalizer}, {"test_audit_tuple", test_audit_tuple}, {"test_audit_subinterpreter", test_audit_subinterpreter}, {"test_audit_run_command", test_audit_run_command}, diff --git a/Python/pystate.c b/Python/pystate.c index 962d1c6cfa5b661..7d5c6869aaa4e95 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1036,10 +1036,6 @@ PyInterpreterState_Delete(PyInterpreterState *interp) // we start finalizing the interpreter (or the current thread state). PyThreadState *tcur = current_fast_get(); if (tcur != NULL && interp == tcur->interp) { - // Finalizers and C audit hooks can create tasks after interpreter_clear(). - // Transfer them while the current thread state is still attached. - move_asyncio_tasks_to_interpreter(tcur); - /* Unset current thread. After this, many C API calls become crashy. */ _PyThreadState_Detach(tcur); } @@ -1941,6 +1937,8 @@ static void tstate_delete_common(PyThreadState *tstate, int release_gil) { assert(tstate->_status.cleared && !tstate->_status.finalized); + // Finalizers that run after PyThreadState_Clear() may register new tasks. + move_asyncio_tasks_to_interpreter(tstate); assert(llist_empty( &((_PyThreadStateImpl *)tstate)->asyncio_tasks_head)); tstate_verify_not_active(tstate); From eb4cb96c06007404795146f9483e068058d79c77 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Fri, 16 Oct 2026 11:44:46 -0600 Subject: [PATCH 5/5] gh-159041: Simplify the interpreter-clear regression test and move to more appropriate spot --- Lib/test/test_embed.py | 76 ++++++++++++++++++++---------------------- Programs/_testembed.c | 36 ++++++++++---------- 2 files changed, 55 insertions(+), 57 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 0f6e7ec8004ff42..05c62e7990faff8 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -226,45 +226,6 @@ def run_repeated_init_and_subinterpreters(self): class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 100 * 50 - def test_asyncio_task_during_interpreter_clear(self): - import_helper.import_module('_asyncio') - code = textwrap.dedent(""" - import asyncio - import contextvars - import gc - import os - - gc.disable() - - async def noop(): - pass - - class Loop: - def get_debug(self): - return False - - def call_soon(self, callback, *args, context=None): - self.callback = callback - - class CreatesTaskOnClear: - def __del__(self, task_type=asyncio.Task, coro=noop, - loop=Loop(), context=contextvars.Context(), - write=os.write): - task_type(coro(), loop=loop, context=context) - write(1, b'task created\\n') - - # Keep the finalizer alive until interpreter_clear()'s last GC. - finalizer = CreatesTaskOnClear() - finalizer.cycle = finalizer - del finalizer - """) - # gh-159041: Delete the cleared interpreter with no thread state attached. - out, err = self.run_embedded_interpreter( - "test_interpreter_clear_and_delete", code, - timeout=support.SHORT_TIMEOUT) - self.assertEqual(out, "task created\n") - self.assertEqual(err, "") - def test_subinterps_main(self): for run in self.run_repeated_init_and_subinterpreters(): main = run[0] @@ -479,6 +440,43 @@ def test_finalize_structseq(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS) + def test_asyncio_task_during_interpreter_clear(self): + import_helper.import_module('_asyncio') + code = textwrap.dedent(""" + import asyncio + import contextvars + import gc + import os + + gc.disable() + + async def noop(): + pass + + class Loop: + def get_debug(self): + return False + + def call_soon(self, callback, *args, context=None): + self.callback = callback + + class CreatesTaskOnClear: + def __del__(self): + asyncio.Task(noop(), loop=Loop(), context=contextvars.Context()) + os.write(1, b'task created\\n') + + # Keep the finalizer alive until interpreter_clear()'s last GC. + finalizer = CreatesTaskOnClear() + finalizer.cycle = finalizer + del finalizer + """) + # gh-159041: Delete the cleared interpreter with no thread state attached. + out, err = self.run_embedded_interpreter( + "test_interpreter_clear_and_delete", code, + timeout=support.SHORT_TIMEOUT) + self.assertEqual(out, "task created\n") + self.assertEqual(err, "") + def test_simple_initialization_api(self): # _testembed now uses Py_InitializeFromConfig by default # This case specifically checks Py_Initialize(Ex) still works diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7b0d88e4e51294d..89b12ce7733d40a 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -154,23 +154,6 @@ static void _testembed_initialize(void) } -static int test_interpreter_clear_and_delete(void) -{ - assert(main_argc == 3); - _testembed_initialize(); - if (PyRun_SimpleString(main_argv[2]) < 0) { - return 1; - } - - PyInterpreterState *interp = PyInterpreterState_Get(); - PyInterpreterState_Clear(interp); - PyThreadState_Swap(NULL); - PyInterpreterState_Delete(interp); - // The interpreter has already been deleted, so don't call Py_FinalizeEx(). - return 0; -} - - static int test_import_in_subinterpreters(void) { _testembed_initialize(); @@ -285,6 +268,23 @@ static int test_repeated_init_exec(void) return 0; } + +static int test_interpreter_clear_and_delete(void) +{ + assert(main_argc == 3); + _testembed_initialize(); + if (PyRun_SimpleString(main_argv[2]) < 0) { + return 1; + } + + PyInterpreterState *interp = PyInterpreterState_Get(); + PyInterpreterState_Clear(interp); + PyThreadState_Swap(NULL); + PyInterpreterState_Delete(interp); + // The interpreter has already been deleted, so don't call Py_FinalizeEx(). + return 0; +} + /**************************************************************************** * Test the Py_Initialize(Ex) convenience/compatibility wrappers ***************************************************************************/ @@ -3018,9 +3018,9 @@ struct TestCase static struct TestCase TestCases[] = { // Python initialization {"test_repeated_init_exec", test_repeated_init_exec}, + {"test_interpreter_clear_and_delete", test_interpreter_clear_and_delete}, {"test_repeated_simple_init", test_repeated_simple_init}, {"test_forced_io_encoding", test_forced_io_encoding}, - {"test_interpreter_clear_and_delete", test_interpreter_clear_and_delete}, {"test_import_in_subinterpreters", test_import_in_subinterpreters}, {"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters}, {"test_repeated_init_and_inittab", test_repeated_init_and_inittab},