From fd7aab565932bed1d4da1163f8784e620d0c5be4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 23 Oct 2026 18:58:43 +0100 Subject: [PATCH] gh-159137: Traverse dictionary values as spans in the free-threaded GC --- Include/internal/pycore_dict.h | 2 + ...-10-23-19-45-01.gh-issue-159137.gcDict.rst | 2 + Objects/dictobject.c | 8 +- Python/gc_free_threading.c | 105 ++++++++++++------ 4 files changed, 80 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-45-01.gh-issue-159137.gcDict.rst diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 0257aaaf4534d5e..6606bc8ec6d8cce 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -94,6 +94,8 @@ typedef struct { PyObject *me_value; /* This field is only meaningful for combined tables */ } PyDictUnicodeEntry; +extern int _PyDict_Traverse(PyObject *, visitproc, void *); + extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-45-01.gh-issue-159137.gcDict.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-45-01.gh-issue-159137.gcDict.rst new file mode 100644 index 000000000000000..7bd20d70080d677 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-23-19-45-01.gh-issue-159137.gcDict.rst @@ -0,0 +1,2 @@ +Reduce dictionary traversal overhead in the free-threaded garbage collector +with spans of dictionary values. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f00cf75f6a762db..e4087cc143b3c33 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -5089,8 +5089,8 @@ dict_popitem_impl(PyDictObject *self) return res; } -static int -dict_traverse(PyObject *op, visitproc visit, void *arg) +int +_PyDict_Traverse(PyObject *op, visitproc visit, void *arg) { PyDictObject *mp = (PyDictObject *)op; PyDictKeysObject *keys = mp->ma_keys; @@ -5522,7 +5522,7 @@ PyTypeObject PyDict_Type = { Py_TPFLAGS_BASETYPE | Py_TPFLAGS_DICT_SUBCLASS | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_MAPPING, /* tp_flags */ dictionary_doc, /* tp_doc */ - dict_traverse, /* tp_traverse */ + _PyDict_Traverse, /* tp_traverse */ dict_tp_clear, /* tp_clear */ dict_richcompare, /* tp_richcompare */ 0, /* tp_weaklistoffset */ @@ -8632,7 +8632,7 @@ PyTypeObject PyFrozenDict_Type = { | Py_TPFLAGS_BASETYPE | _Py_TPFLAGS_MATCH_SELF | Py_TPFLAGS_MAPPING, .tp_doc = frozendict_doc, - .tp_traverse = dict_traverse, + .tp_traverse = _PyDict_Traverse, .tp_clear = dict_tp_clear, .tp_richcompare = dict_richcompare, .tp_iter = dict_iter, diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 6db010468456eaa..1133b63cc7bc7b5 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -578,10 +578,11 @@ static_assert(BUFFER_HI < BUFFER_SIZE && #define prefetch(ptr) #endif -// a contigous sequence of PyObject pointers, can contain NULLs +// A strided sequence of PyObject pointers, which can contain NULLs. typedef struct { PyObject **start; - PyObject **end; + Py_ssize_t size; + Py_ssize_t stride; } gc_span_t; typedef struct { @@ -660,9 +661,10 @@ gc_mark_stack_push(_PyObjectStack *ms, PyObject *op) } static int -gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) +gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, Py_ssize_t size, + Py_ssize_t stride) { - if (start == end) { + if (size == 0) { return 0; } if (ss->size >= ss->capacity) { @@ -678,9 +680,10 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) } ss->stack = new_stack; } - assert(end > start); + assert(size > 0); ss->stack[ss->size].start = start; - ss->stack[ss->size].end = end; + ss->stack[ss->size].size = size; + ss->stack[ss->size].stride = stride; ss->size++; return 0; } @@ -748,20 +751,21 @@ gc_mark_enqueue(PyObject *op, gc_mark_args_t *args) } } -// Called when we have a contigous sequence of PyObject pointers, either -// a tuple or list object. This will add the items to the buffer if there +// Called for a strided sequence of PyObject pointers, including list and +// tuple items and dictionary values. This will add the items to the buffer if there // is space for them all otherwise push a new "span" on the span stack. Using // spans has the advantage of not creating a deep _PyObjectStack stack when // dealing with long sequences. Those sequences will be processed in smaller // chunks by the gc_prime_from_spans() function. static int -gc_mark_enqueue_span(PyObject **item, Py_ssize_t size, gc_mark_args_t *args) +gc_mark_enqueue_span_stride(PyObject **item, Py_ssize_t size, + Py_ssize_t stride, gc_mark_args_t *args) { Py_ssize_t used = gc_mark_buffer_len(args); Py_ssize_t free = BUFFER_SIZE - used; if (free >= size) { for (Py_ssize_t i = 0; i < size; i++) { - PyObject *op = item[i]; + PyObject *op = *(PyObject **)((char *)item + i * stride); if (op == NULL) { continue; } @@ -770,14 +774,39 @@ gc_mark_enqueue_span(PyObject **item, Py_ssize_t size, gc_mark_args_t *args) } else { assert(size > 0); - PyObject **end = &item[size]; - if (gc_mark_span_push(&args->spans, item, end) < 0) { + if (gc_mark_span_push(&args->spans, item, size, stride) < 0) { return -1; } } return 0; } +static int +gc_mark_enqueue_span(PyObject **item, Py_ssize_t size, gc_mark_args_t *args) +{ + return gc_mark_enqueue_span_stride(item, size, sizeof(PyObject *), args); +} + +static int +gc_mark_traverse_dict(PyObject *self, gc_mark_args_t *args) +{ + PyDictObject *dict = (PyDictObject *)self; + PyDictKeysObject *keys = dict->ma_keys; + if (keys->dk_nentries == 0) { + return 0; + } + if (!DK_IS_UNICODE(keys)) { + return _PyDict_Traverse(self, gc_mark_enqueue_buffer_visitproc, args); + } + if (_PyDict_HasSplitTable(dict)) { + return gc_mark_enqueue_span(dict->ma_values->values, + keys->dk_nentries, args); + } + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + return gc_mark_enqueue_span_stride(&entries[0].me_value, keys->dk_nentries, + sizeof(PyDictUnicodeEntry), args); +} + static bool gc_clear_alive_bits(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) @@ -1250,44 +1279,49 @@ gc_prime_from_spans(gc_mark_args_t *args) assert(space <= BUFFER_HI); gc_span_t entry = args->spans.stack[--args->spans.size]; // spans on the stack should always have one or more elements - assert(entry.start < entry.end); + assert(entry.size > 0); do { PyObject *op = *entry.start; - entry.start++; + entry.size--; + if (entry.size > 0) { + entry.start = (PyObject **)((char *)entry.start + entry.stride); + } if (op != NULL) { gc_mark_buffer_push(op, args); space--; if (space == 0) { // buffer is as full as we want and not done with span - gc_mark_span_push(&args->spans, entry.start, entry.end); + gc_mark_span_push(&args->spans, entry.start, entry.size, entry.stride); return; } } - } while (entry.start < entry.end); + } while (entry.size > 0); } static void gc_prime_buffer(gc_mark_args_t *args) { - if (args->spans.size > 0) { + // Dictionary spans can contain only deleted entries. + while (args->spans.size > 0) { gc_prime_from_spans(args); + if (!gc_mark_buffer_is_empty(args)) { + return; + } } - else { - // When priming, don't fill the buffer too full since that would - // likely cause the stack to be used shortly after when it - // fills. We want to use the buffer as much as possible and so - // we only fill to BUFFER_HI, not BUFFER_SIZE. - Py_ssize_t space = BUFFER_HI - gc_mark_buffer_len(args); - assert(space > 0); - do { - PyObject *op = _PyObjectStack_Pop(&args->stack); - if (op == NULL) { - return; - } - gc_mark_buffer_push(op, args); - space--; - } while (space > 0); - } + // When priming, don't fill the buffer too full since that would + // likely cause the stack to be used shortly after when it + // fills. We want to use the buffer as much as possible and so + // we only fill to BUFFER_HI, not BUFFER_SIZE. + Py_ssize_t space = BUFFER_HI - gc_mark_buffer_len(args); + assert(space > 0); + do { + PyObject *op = _PyObjectStack_Pop(&args->stack); + if (op == NULL) { + return; + } + gc_mark_buffer_push(op, args); + space--; + } while (space > 0); } static int @@ -1331,6 +1365,11 @@ gc_propagate_alive_prefetch(gc_mark_args_t *args) return -1; } } + else if (traverse == _PyDict_Traverse) { + if (gc_mark_traverse_dict(op, args) < 0) { + return -1; + } + } else if (traverse(op, gc_mark_enqueue_buffer_visitproc, args) < 0) { return -1; }