diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 919100f2d5ce04..7c17879c8317e2 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -330,6 +330,7 @@ def f(): self.assertEqual(gc.collect(), 1) def test_saveall(self): + self.addCleanup(gc.collect) # Verify that cyclic garbage like lists show up in gc.garbage if the # SAVEALL option is enabled. @@ -352,6 +353,31 @@ def test_saveall(self): self.assertEqual(len(gc.garbage), 1) obj = gc.garbage.pop() self.assertEqual(id(obj), id_L) + obj.clear() + + # A no-op generator finalizer must still set the finalized marker. + gen = (item for item in ()) + next(gen, None) + self.assertTrue(gc.is_tracked(gen)) + self.assertIsNone(gen.gi_frame) + self.assertFalse(gc.is_finalized(gen)) + L = [gen] + L.append(L) + id_L = id(L) + del gen + + try: + gc.set_debug(gc.DEBUG_SAVEALL) + del L + gc.collect() + saved = next(obj for obj in gc.garbage if id(obj) == id_L) + self.assertTrue(gc.is_finalized(saved[0])) + finally: + gc.set_debug(debug) + for obj in gc.garbage: + if id(obj) == id_L: + obj.clear() + gc.garbage.clear() def test_del(self): # __del__ methods can trigger collection, make this to happen @@ -658,6 +684,83 @@ def __del__(self): lazarus = storage.pop() self.assertTrue(gc.is_finalized(lazarus)) + def test_weakref_callback_installs_finalizer(self): + """A callback can make a finalizer eligible before the object is cleared.""" + self.addCleanup(gc.collect) + events = [] + + class C: + """A self-cycle whose weakref callback installs its finalizer.""" + __slots__ = ("cycle", "__weakref__") + + def finalize(obj): + """Record whether finalization sees the intact cycle.""" + events.append(("finalizer", getattr(obj, "cycle", None) is obj)) + + def callback(reference): + """Install the finalizer after observing the cleared weakref.""" + events.append(("callback", reference() is None)) + C.__del__ = finalize + + obj = C() + obj.cycle = obj + reference = weakref.ref(obj, callback) + gc.collect() + del obj + + self.assertEqual(gc.collect(), 1) + self.assertEqual(events, [("callback", True), ("finalizer", True)]) + self.assertIsNone(reference()) + + @requires_gil_enabled("tests the generation-list collector") + def test_debug_output_resurrects_without_callbacks(self): + """Debug output can resurrect garbage without pending callbacks.""" + was_enabled = gc.isenabled() + gc.disable() + self.addCleanup(gc.enable if was_enabled else gc.disable) + self.addCleanup(gc.collect) + resurrected = [] + + class C: + """An object with a cycle and no finalizer.""" + __slots__ = ("cycle", "__weakref__") + + class Stderr: + """Retain the target when collection reports it as collectable.""" + + def write(self, text): + """Retain the target on collectable debug output. + + text: Debug text written by the collector. + Returns: Number of characters accepted. + """ + if text.startswith("gc: collectable") and not resurrected: + target = reference() + if target is not None: + resurrected.append(target) + return len(text) + + writer = Stderr() + gc.collect() + obj = C() + obj.cycle = obj + reference = weakref.ref(obj) + del obj + + debug = gc.get_debug() + try: + with unittest.mock.patch.object(sys, "stderr", writer): + gc.set_debug(gc.DEBUG_COLLECTABLE) + self.assertEqual(gc.collect(), 0) + self.assertEqual(len(resurrected), 1) + self.assertIs(reference(), resurrected[0]) + self.assertIs(resurrected[0].cycle, resurrected[0]) + finally: + gc.set_debug(debug) + for obj in resurrected: + obj.cycle = None + resurrected.clear() + def test_bug1055820b(self): # Corresponds to temp2b.py in the bug report. diff --git a/Makefile.pre.in b/Makefile.pre.in index 308fb964a0cd57..6835c9fb4f048c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -102,7 +102,13 @@ CONFIGURE_LDFLAGS= @LDFLAGS@ # command line to append to these values without stomping the pre-set # values. PY_CFLAGS= $(BASECFLAGS) $(OPT) $(CONFIGURE_CFLAGS) $(CFLAGS) $(EXTRA_CFLAGS) -PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include/internal -I$(srcdir)/Include/internal/mimalloc +# Configure validates function alignment once, but make-time CFLAGS can still +# select a size-optimized build after configure has completed. +PY_ALIGN_FUNCTIONS_CFLAGS=@FUNCTION_ALIGNMENT_CFLAGS@ +ifneq ($(filter -Os -Oz,$(PY_CFLAGS) $(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST)),) +PY_ALIGN_FUNCTIONS_CFLAGS= +endif +PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) $(PY_ALIGN_FUNCTIONS_CFLAGS) -I$(srcdir)/Include/internal -I$(srcdir)/Include/internal/mimalloc # Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to # be able to build extension modules using the directories specified in the # environment variables diff --git a/Misc/NEWS.d/next/Build/2026-10-18-21-30-00.gh-issue-159437.align64.rst b/Misc/NEWS.d/next/Build/2026-10-18-21-30-00.gh-issue-159437.align64.rst new file mode 100644 index 00000000000000..023fe865eaf767 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-10-18-21-30-00.gh-issue-159437.align64.rst @@ -0,0 +1 @@ +Request 64-byte function alignment for performance-oriented Linux and macOS builds. This can reduce performance variation caused by changes to the size of preceding functions. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-21-44-05.gh-issue-159437.count1.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-21-44-05.gh-issue-159437.count1.rst new file mode 100644 index 00000000000000..a98b6aee7ef4ff --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-14-21-44-05.gh-issue-159437.count1.rst @@ -0,0 +1 @@ +Count collectable objects while clearing weak references, avoiding a separate pass during garbage collection. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-23-43-15.gh-issue-159437.LHuMLA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-23-43-15.gh-issue-159437.LHuMLA.rst new file mode 100644 index 00000000000000..0052c33ea970de --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-15-23-43-15.gh-issue-159437.LHuMLA.rst @@ -0,0 +1,3 @@ +With the GIL enabled, garbage collection skips the finalization scan when no +finalizers or weakref callbacks are pending and debug output for collectable +objects is disabled. diff --git a/Python/gc.c b/Python/gc.c index 5dc79e20191ebc..dd6d052b9fc8f6 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -756,7 +756,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) } } -/* Handle weakref callbacks. Note that it's possible for such weakrefs to be +/* Prepare weakref callbacks. Note that it's possible for such weakrefs to be * outside the unreachable set -- indeed, those are precisely the weakrefs * whose callbacks must be invoked. See gc_weakref.txt for overview & some * details. @@ -794,17 +794,20 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) * or missing `tp_traverse` methods. When that object goes away, the callback * for weakref can be executed and that could reveal unreachable objects to * Python-level code. See bpo-38006 as an example bug. + * + * Arguments: + * - `unreachable`: normalized collecting list to inspect. + * - `wrcb_to_call`: initialized list receiving weakrefs whose callbacks are safe to invoke. + * + * Return whether any finalizer is pending or a weakref callback is queued. + * This function does not invoke callbacks. */ -static int -handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old, - bool *needs_resurrection_check) +Py_NO_INLINE static bool +prepare_callbacks(PyGC_Head *unreachable, PyGC_Head *wrcb_to_call) { PyGC_Head *gc; - PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */ PyGC_Head *next; - int num_freed = 0; - - gc_list_init(&wrcb_to_call); + bool has_pending_finalizer = false; /* Find all weakrefs with callbacks and move into `wrcb_to_call` if the * callback needs to be invoked. We make another pass over wrcb_to_call, @@ -814,9 +817,13 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old, PyWeakReference **wrlist; PyObject *op = FROM_GC(gc); + PyTypeObject *type = Py_TYPE(op); + if (type->tp_finalize != NULL && !has_pending_finalizer) { + has_pending_finalizer = !_PyGC_FINALIZED(op); + } next = GC_NEXT(gc); - if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) { + if (! _PyType_SUPPORTS_WEAKREFS(type)) { continue; } @@ -891,21 +898,35 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old, PyGC_Head *wrasgc = AS_GC((PyObject *)wr); // wrasgc is reachable, but next isn't, so they can't be the same _PyObject_ASSERT((PyObject *)wr, wrasgc != next); - gc_list_move(wrasgc, &wrcb_to_call); + gc_list_move(wrasgc, wrcb_to_call); } } + return has_pending_finalizer || !gc_list_is_empty(wrcb_to_call); +} + +/* Invoke queued weakref callbacks and move surviving weakrefs to `old`. + * + * Arguments: + * - `wrcb_to_call`: initialized queue to drain, with a temporary reference to each weakref. + * - `old`: generation receiving weakrefs that survive callback invocation. + * + * Return the number of weakrefs reclaimed while callbacks ran. + */ +static int +invoke_weakref_callbacks(PyGC_Head *wrcb_to_call, PyGC_Head *old) +{ + PyGC_Head *gc; + int num_freed = 0; + /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ - if (!*needs_resurrection_check) { - *needs_resurrection_check = !gc_list_is_empty(&wrcb_to_call); - } - while (! gc_list_is_empty(&wrcb_to_call)) { + while (! gc_list_is_empty(wrcb_to_call)) { PyObject *temp; PyObject *callback; - gc = (PyGC_Head*)wrcb_to_call._gc_next; + gc = (PyGC_Head*)wrcb_to_call->_gc_next; PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, PyWeakref_Check(op)); PyWeakReference *wr = (PyWeakReference *)op; @@ -934,7 +955,7 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old, * ours). */ Py_DECREF(op); - if (wrcb_to_call._gc_next == (uintptr_t)gc) { + if (wrcb_to_call->_gc_next == (uintptr_t)gc) { /* object is still alive -- move it */ gc_list_move(gc, old); } @@ -946,25 +967,30 @@ handle_weakref_callbacks(PyGC_Head *unreachable, PyGC_Head *old, return num_freed; } -/* Clear all weakrefs to unreachable objects. When this returns, no object in - * `unreachable` is weakly referenced anymore. See the comments above - * handle_weakref_callbacks() for why these weakrefs need to be cleared. +/* Clear all weakrefs to unreachable objects without invoking callbacks. + * When this returns, no object in `unreachable` is weakly referenced anymore. + * See the comments above prepare_callbacks() for why these weakrefs need to be cleared. + * + * `unreachable`: Final collectable list; membership is unchanged. + * Return: Number of objects in `unreachable`. */ -static void +static Py_ssize_t clear_weakrefs(PyGC_Head *unreachable) { PyGC_Head *gc; PyGC_Head *next; + Py_ssize_t count = 0; for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyWeakReference **wrlist; PyObject *op = FROM_GC(gc); next = GC_NEXT(gc); + count++; if (PyWeakref_Check(op)) { /* A weakref inside the unreachable set is always cleared. See - * the comments above handle_weakref_callbacks() for why these + * the comments above prepare_callbacks() for why these * must be cleared. */ _PyWeakref_ClearRef((PyWeakReference *)op); @@ -994,6 +1020,8 @@ clear_weakrefs(PyGC_Head *unreachable) _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); } } + + return count; } static void @@ -1036,13 +1064,14 @@ handle_legacy_finalizers(PyThreadState *tstate, /* Run first-time finalizers (if any) on all the objects in collectable. * Note that this may remove some (or even all) of the objects from the * list, due to refcounts falling to 0. + * Return whether a finalizer may have run Python. */ -static void -finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable, - bool *needs_resurrection_check) +static bool +finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable) { destructor finalize; PyGC_Head seen; + bool needs_resurrection_check = false; /* While we're going through the loop, `finalize(op)` may cause op, or * other objects, to be reclaimed via refcounts falling to zero. So @@ -1062,7 +1091,7 @@ finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable, (finalize = Py_TYPE(op)->tp_finalize) != NULL) { if (!_PyGen_FinalizerIsNoop(op)) { - *needs_resurrection_check = true; + needs_resurrection_check = true; } _PyGC_SET_FINALIZED(op); Py_INCREF(op); @@ -1072,6 +1101,7 @@ finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable, } } gc_list_merge(&seen, collectable); + return needs_resurrection_check; } /* Break reference cycles by clearing the containers involved. This is @@ -1253,6 +1283,43 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, } +/* Invoke weakref callbacks and finalizers, then handle possible resurrection. + * + * Arguments: + * - `tstate`: the thread state used when invoking finalizers. + * - `collectable`: normalized collecting list whose finalizers may run. + * - `still_unreachable`: distinct, uninitialized list head for remaining garbage. + * - `old`: generation receiving surviving weakrefs and resurrected objects. + * - `wrcb_to_call`: initialized, possibly empty list of callback-bearing weakrefs to drain. + * - `collected_count`: count incremented by weakrefs reclaimed while callbacks run. + * - `needs_resurrection_check`: whether earlier work may have run Python. + * + * Return whether resurrection analysis ran. If true, `collectable` must not be + * used again and `still_unreachable` contains the remaining garbage. Otherwise, + * `collectable` remains selected and `still_unreachable` is untouched. + */ +Py_NO_INLINE static bool +handle_callbacks(PyThreadState *tstate, PyGC_Head *collectable, + PyGC_Head *still_unreachable, PyGC_Head *old, + PyGC_Head *wrcb_to_call, Py_ssize_t *collected_count, + bool needs_resurrection_check) +{ + bool weakref_callbacks_invoked = !gc_list_is_empty(wrcb_to_call); + *collected_count += invoke_weakref_callbacks(wrcb_to_call, old); + needs_resurrection_check |= weakref_callbacks_invoked; + validate_list(old, collecting_clear_unreachable_clear); + validate_list(collectable, collecting_set_unreachable_clear); + + // Weakref callbacks can make finalizers newly eligible after the initial + // scan, so we must recheck even when we previously found no pending finalizers. + needs_resurrection_check |= finalize_garbage(tstate, collectable); + if (needs_resurrection_check) { + handle_resurrected_objects(collectable, still_unreachable, old); + } + return needs_resurrection_check; +} + + /* Invoke progress callbacks to notify clients that garbage collection * is starting or stopping */ @@ -1563,28 +1630,21 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) } } - /* Clear weakrefs and invoke callbacks as necessary. */ - stats.collected += handle_weakref_callbacks(&unreachable, old, - &needs_resurrection_check); - validate_list(old, collecting_clear_unreachable_clear); - validate_list(&unreachable, collecting_set_unreachable_clear); - - /* Call tp_finalize on objects which have one. */ - finalize_garbage(tstate, &unreachable, &needs_resurrection_check); + PyGC_Head wrcb_to_call; + gc_list_init(&wrcb_to_call); + bool has_pending_callbacks = prepare_callbacks(&unreachable, &wrcb_to_call); + has_pending_callbacks |= needs_resurrection_check; - /* Handle any objects that may have resurrected after the call - * to 'finalize_garbage' and continue the collection with the - * objects that are still unreachable */ PyGC_Head final_unreachable; - if (needs_resurrection_check) { - handle_resurrected_objects(&unreachable, &final_unreachable, old); - } - else { - /* No Python code ran during finalization, so the original - * unreachable set is still valid. */ - gc_list_init(&final_unreachable); - gc_list_merge(&unreachable, &final_unreachable); + PyGC_Head *collectable = &unreachable; + if (has_pending_callbacks) { + bool resurrection_checked = handle_callbacks(tstate, collectable, &final_unreachable, old, &wrcb_to_call, &stats.collected, needs_resurrection_check); + if (resurrection_checked) { + collectable = &final_unreachable; + } } + validate_list(old, collecting_clear_unreachable_clear); + validate_list(collectable, collecting_set_unreachable_clear); /* Clear weakrefs to objects in the unreachable set. No Python-level * code must be allowed to access those unreachable objects. During @@ -1592,14 +1652,13 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) * and create new weakrefs. If those weakrefs were not cleared, they * could reveal unreachable objects. Callbacks are not executed. */ - clear_weakrefs(&final_unreachable); + stats.collected += clear_weakrefs(collectable); - /* Call tp_clear on objects in the final_unreachable set. This will cause + /* Call tp_clear on objects in the collectable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. */ - stats.collected += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, old); + delete_garbage(tstate, gcstate, collectable, old); /* Collect statistics on uncollectable objects found and print * debugging information. */ diff --git a/configure b/configure index 217d8e135600e1..c0fd0422bfd8f4 100755 --- a/configure +++ b/configure @@ -922,6 +922,7 @@ WASM_ASSETS_DIR LDFLAGS_NOLTO EXE_LDFLAGS LDFLAGS_NODIST +FUNCTION_ALIGNMENT_CFLAGS CFLAGS_NODIST BASECFLAGS CFLAGS_ALIASING @@ -9960,6 +9961,75 @@ then esac fi +# Keep performance-oriented release builds aligned on validated platforms +# without changing size-optimized, mobile, or third-party extension builds. +FUNCTION_ALIGNMENT_CFLAGS= +case $host in #( + *-*-linux-android*) : + ;; #( + *-*-linux*|*-apple-darwin*) : + + if test "x$Py_DEBUG" = xtrue +then : + +else case e in #( + e) + case " $OPT $CFLAGS " in #( + *" -Os "*|*" -Oz "*) : + ;; #( + *) : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -falign-functions=64" >&5 +printf %s "checking whether C compiler accepts -falign-functions=64... " >&6; } +if test ${ax_cv_check_cflags__Werror__falign_functions_64+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -falign-functions=64" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__falign_functions_64=yes +else case e in #( + e) ax_cv_check_cflags__Werror__falign_functions_64=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__falign_functions_64" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__falign_functions_64" >&6; } +if test "x$ax_cv_check_cflags__Werror__falign_functions_64" = xyes +then : + + FUNCTION_ALIGNMENT_CFLAGS="-falign-functions=64" + +else case e in #( + e) : ;; +esac +fi + ;; +esac + ;; +esac +fi + ;; #( + *) : + ;; +esac + # WASM flags case $ac_sys_system in #( Emscripten) : @@ -10131,6 +10201,7 @@ esac + # The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= diff --git a/configure.ac b/configure.ac index ca804d6448810d..f1714aa2afed88 100644 --- a/configure.ac +++ b/configure.ac @@ -2430,6 +2430,21 @@ then esac fi +# Keep performance-oriented release builds aligned on validated platforms +# without changing size-optimized, mobile, or third-party extension builds. +FUNCTION_ALIGNMENT_CFLAGS= +AS_CASE([$host], + [*-*-linux-android*], [], + [*-*-linux*|*-apple-darwin*], [ + AS_VAR_IF([Py_DEBUG], [true], [], [ + AS_CASE([" $OPT $CFLAGS "], + [*" -Os "*|*" -Oz "*], [], + [AX_CHECK_COMPILE_FLAG([-falign-functions=64], [ + FUNCTION_ALIGNMENT_CFLAGS="-falign-functions=64" + ], [], [-Werror])]) + ]) + ]) + # WASM flags AS_CASE([$ac_sys_system], [Emscripten], [ @@ -2558,6 +2573,7 @@ AS_CASE([$enable_wasm_dynamic_linking], AC_SUBST([BASECFLAGS]) AC_SUBST([CFLAGS_NODIST]) +AC_SUBST([FUNCTION_ALIGNMENT_CFLAGS]) AC_SUBST([LDFLAGS_NODIST]) AC_SUBST([EXE_LDFLAGS]) AC_SUBST([LDFLAGS_NOLTO])