From 79a2ae5c8972cbe6b673d492ff2ed66c90d62a09 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Oct 2026 15:34:15 -0500 Subject: [PATCH 01/10] gh-158660: Add `PyListWriter` C API --- Doc/c-api/list.rst | 82 ++++++ Doc/whatsnew/3.16.rst | 15 + Include/cpython/listobject.h | 22 ++ Lib/test/test_capi/test_list.py | 75 ++++- ...-10-21-16-35-42.gh-issue-158660.KvixNp.rst | 14 + Modules/_testcapi/list.c | 265 +++++++++++++++++- Objects/listobject.c | 189 +++++++++++++ 7 files changed, 660 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 21769c4c5c869c8..2756677ac492f7f 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -227,3 +227,85 @@ List Objects Return a new tuple object containing the contents of *list*; equivalent to ``tuple(list)``. + + +.. _pylistwriter: + +PyListWriter +------------ + +The :c:type:`PyListWriter` API can be used to create a Python :class:`list` +object. + +On Free Threading, it's more efficient than the ``PyList`` C API to create a +:class:`list`, because it can avoid the critical section on the created list. + +.. versionadded:: next + + +.. c:type:: PyListWriter + + A list writer object. + + The API is **not thread safe**. To share a writer with multiple threads, a + critical section or a lock is needed. + + The instance must be destroyed by :c:func:`PyListWriter_Finish` on + success, or :c:func:`PyListWriter_Discard` on error. + + +.. c:function:: PyListWriter* PyListWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyListWriter` and preallocate *size* items. + + On error, set an exception and return ``NULL``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyListWriter_Finish(PyListWriter *writer) + + Finish a :c:type:`PyListWriter` created by + :c:func:`PyListWriter_Create`. + + On success, return a Python :class:`list` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + No API can be called on the writer after :c:func:`PyListWriter_Finish`. + +.. c:function:: void PyListWriter_Discard(PyListWriter *writer) + + Discard a :c:type:`PyListWriter` created by :c:func:`PyListWriter_Create`. + + Do nothing if *writer* is ``NULL``. + + The writer instance is invalid after the call. + No API can be called on the writer after :c:func:`PyListWriter_Discard`. + +.. c:function:: int PyListWriter_Append(PyListWriter *writer, PyObject *item) + + Append an item to the list writer. + Similar to :c:func:`PyList_Append`. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item) + + Similar to :c:func:`PyListWriter_Append`, but steals the reference to + *item*. + +.. c:function:: int PyListWriter_Extend(PyListWriter *writer, PyObject *iterable) + + Extend the list writer with the contents of *iterable*. + Similar to :c:func:`PyList_Extend`. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyListWriter_ExtendArray(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem) + + Extend the list writer with the contents of *items*. + + On success, return ``0``. + On error, set an exception and return ``-1``. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index b29170b5b0b93da..39e47ae4bf55e13 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1248,6 +1248,21 @@ New features * Add :c:func:`Py_HashBuffer` to the limited C API. (Contributed by Victor Stinner in :gh:`148233`.) +* Add a new :ref:`PyListWriter ` C API to create a :class:`list` + object. On Free Threading, it's more efficient than the ``PyList`` C API to + create a :class:`list`, because it can avoid the critical section on the + created list. Add functions: + + * :c:func:`PyListWriter_Create` + * :c:func:`PyListWriter_Finish` + * :c:func:`PyListWriter_Discard` + * :c:func:`PyListWriter_Append` + * :c:func:`PyListWriter_AppendSteal` + * :c:func:`PyListWriter_Extend` + * :c:func:`PyListWriter_ExtendArray` + + (Contributed by Victor Stinner in :gh:`158660`.) + * Add :c:macro:`Py_SETREF` and :c:macro:`Py_XSETREF` macros to the limited C API. (Contributed by Nathan Goldbaum and Victor Stinner in :gh:`158842`.) diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index 49f5e8d6d1a0d6c..cbc4413f594d1e5 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -51,3 +51,25 @@ PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) { PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable); PyAPI_FUNC(int) PyList_Clear(PyObject *self); + + +/* --- PyListWriter API --------------------------------------------------- */ + +typedef struct PyListWriter PyListWriter; + +PyAPI_FUNC(PyListWriter*) PyListWriter_Create(Py_ssize_t size); +PyAPI_FUNC(PyObject*) PyListWriter_Finish(PyListWriter *writer); +PyAPI_FUNC(void) PyListWriter_Discard(PyListWriter *writer); +PyAPI_FUNC(int) PyListWriter_Append( + PyListWriter *writer, + PyObject *item); +PyAPI_FUNC(int) PyListWriter_AppendSteal( + PyListWriter *writer, + PyObject *item); +PyAPI_FUNC(int) PyListWriter_Extend( + PyListWriter *writer, + PyObject *iterable); +PyAPI_FUNC(int) PyListWriter_ExtendArray( + PyListWriter *writer, + PyObject *const *items, + Py_ssize_t nitem); diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index b95b8ba960bd8b2..812ca25ab39bdf1 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -1,9 +1,13 @@ +import threading import unittest -from test.support import import_helper from collections import UserList +from test.support import import_helper +from test.support import threading_helper + _testcapi = import_helper.import_module('_testcapi') _testlimitedcapi = import_helper.import_module('_testlimitedcapi') + NULL = None PY_SSIZE_T_MIN = _testcapi.PY_SSIZE_T_MIN PY_SSIZE_T_MAX = _testcapi.PY_SSIZE_T_MAX @@ -355,5 +359,74 @@ def test_uninitialized_list_repr(self): self.assertEqual(repr(lst), '[, , ]') +class ListWriterTests(unittest.TestCase): + def create_writer(self, size=0): + # Test PyListWriter_Create() + return _testcapi.PyListWriter(size) + + def list_equal(self, result, expected): + if not isinstance(expected, list): + expected = list(expected) + self.assertEqual(result, expected) + self.assertEqual(type(result), list) + + def check_append(self, method): + items = (1, 2, 3) + for size in (0, len(items), len(items) * 10): + with self.subTest(size=size): + writer = self.create_writer(size) + append = getattr(writer, method) + for item in items: + append(item) + self.list_equal(writer.finish(), items) + + def test_append(self): + # Test PyListWriter_Append() + self.check_append('append') + + def test_append_steal(self): + # Test PyListWriter_AppendSteal() + self.check_append('append_steal') + + def check_extend(self, method): + items = (2, 5, 7) + for size in (0, len(items), len(items) * 10): + with self.subTest(size=size): + writer = self.create_writer(size) + extend = getattr(writer, method) + extend(items) + self.list_equal(writer.finish(), items) + + def test_extend(self): + # Test PyListWriter_Extend() + self.check_extend('extend') + + def test_extend_array(self): + # Test PyListWriter_ExtendArray() + self.check_extend('extend_array') + + def test_discard(self): + # Test PyListWriter_Discard(): must not leak memory + writer = self.create_writer() + writer.extend([2, 5, 7]) + writer.discard() + + @threading_helper.requires_working_threading() + def test_thread(self): + # PyListWriter can be used by multiple threads: it's up to the caller + # to implement a lock to prevent concurrent accesses. + writer = self.create_writer() + + def thread_func(writer): + writer.append('Hello') + writer.extend(['World!']) + + thread = threading.Thread(target=thread_func, args=(writer,)) + thread.start() + threading_helper.join_thread(thread) + + self.list_equal(writer.finish(), ['Hello', 'World!']) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst b/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst new file mode 100644 index 000000000000000..b765ccc33c0e865 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst @@ -0,0 +1,14 @@ +Add a new :ref:`PyListWriter ` C API to create a :class:`list` +object. On Free Threading, it's more efficient than the ``PyList`` C API to +create a :class:`list`, because it can avoid the critical section on the +created list. Add functions: + +* :c:func:`PyListWriter_Create` +* :c:func:`PyListWriter_Finish` +* :c:func:`PyListWriter_Discard` +* :c:func:`PyListWriter_Append` +* :c:func:`PyListWriter_AppendSteal` +* :c:func:`PyListWriter_Extend` +* :c:func:`PyListWriter_ExtendArray` + +Patch by Victor Stinner. diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 530b47780ac94e2..cee09359f5bc41d 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -103,6 +103,254 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) } +static PyObject* +test_list_writer(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *result = NULL; + + PyObject* array[3]; + for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { + array[i] = PyLong_FromLong(i); + if (array[i] == NULL) { + for (Py_ssize_t j = 0; j <= i; j++) { + Py_DECREF(array[i]); + } + return NULL; + } + } + + PyListWriter *writer = PyListWriter_Create(3); + if (writer == NULL) { + goto done; + } + + for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { + if (PyListWriter_Append(writer, array[i]) < 0) { + PyListWriter_Discard(writer); + goto done; + } + } + + PyObject *list = PyListWriter_Finish(writer); + if (list == NULL) { + goto done; + } + assert(PyList_CheckExact(list)); + assert(PyList_GET_SIZE(list) == 3); + assert(PyList_GET_ITEM(list, 0) == array[0]); + assert(PyList_GET_ITEM(list, 1) == array[1]); + assert(PyList_GET_ITEM(list, 2) == array[2]); + Py_DECREF(list); + + result = Py_None; + +done: + for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { + Py_DECREF(array[i]); + } + return result; +} + + +typedef struct { + PyObject_HEAD + PyListWriter *writer; +} WriterObject; + + +static inline int +writer_check(WriterObject *self) +{ + if (self->writer == NULL) { + PyErr_SetString(PyExc_ValueError, + "operation on finished/discarded writer"); + return -1; + } + return 0; +} + + +static PyObject * +writer_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + WriterObject *self = (WriterObject *)type->tp_alloc(type, 0); + if (!self) { + return NULL; + } + self->writer = NULL; + return (PyObject*)self; +} + + +// Test PyListWriter_Create() +static int +writer_init(PyObject *self_raw, PyObject *args, PyObject *kwargs) +{ + if (kwargs && PyDict_GET_SIZE(kwargs)) { + PyErr_Format(PyExc_TypeError, + "PyListWriter() takes no keyword arguments"); + return -1; + } + + Py_ssize_t size = 0; + if (!PyArg_ParseTuple(args, "|n", &size)) { + return -1; + } + + WriterObject *self = (WriterObject *)self_raw; + PyListWriter_Discard(self->writer); + + self->writer = PyListWriter_Create(size); + if (self->writer == NULL) { + return -1; + } + + return 0; +} + + +// Test PyListWriter_Discard() +static void +writer_dealloc(PyObject *self_raw) +{ + WriterObject *self = (WriterObject *)self_raw; + PyTypeObject *tp = Py_TYPE(self); + if (self->writer) { + PyListWriter_Discard(self->writer); + } + tp->tp_free(self); + Py_DECREF(tp); +} + + +// Test PyListWriter_Append() +static PyObject* +writer_append(PyObject *self_raw, PyObject *item) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyListWriter_Append(self->writer, item) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +// Test PyListWriter_AppendSteal() +static PyObject* +writer_append_steal(PyObject *self_raw, PyObject *item) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyListWriter_AppendSteal(self->writer, Py_NewRef(item)) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +// Test PyListWriter_Extend() +static PyObject* +writer_extend(PyObject *self_raw, PyObject *iterable) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (PyListWriter_Extend(self->writer, iterable) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +// Test PyListWriter_ExtendArray() +static PyObject* +writer_extend_array(PyObject *self_raw, PyObject *tuple) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (!PyTuple_Check(tuple)) { + PyErr_Format(PyExc_TypeError, + "expect tuple, got %T", tuple); + return NULL; + } + PyObject **items = &PyTuple_GET_ITEM(tuple, 0); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + + if (PyListWriter_ExtendArray(self->writer, items, len) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +// Test PyListWriter_Finish() +static PyObject* +writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyObject *list = PyListWriter_Finish(self->writer); + self->writer = NULL; + return list; +} + + +// Test PyListWriter_Discard() +static PyObject* +writer_discard(PyObject *self_raw, PyObject *Py_UNUSED(args)) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + PyListWriter_Discard(self->writer); + self->writer = NULL; + Py_RETURN_NONE; +} + + +static PyMethodDef writer_methods[] = { + {"append", _PyCFunction_CAST(writer_append), METH_O}, + {"append_steal", _PyCFunction_CAST(writer_append_steal), METH_O}, + {"extend", _PyCFunction_CAST(writer_extend), METH_O}, + {"extend_array", _PyCFunction_CAST(writer_extend_array), METH_O}, + {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, + {"discard", _PyCFunction_CAST(writer_discard), METH_NOARGS}, + {NULL, NULL} /* sentinel */ +}; + +static PyType_Slot Writer_Type_slots[] = { + {Py_tp_new, writer_new}, + {Py_tp_init, writer_init}, + {Py_tp_dealloc, writer_dealloc}, + {Py_tp_methods, writer_methods}, + {0, 0}, /* sentinel */ +}; + +static PyType_Spec Writer_spec = { + .name = "_testcapi.PyListWriter", + .basicsize = sizeof(WriterObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = Writer_Type_slots, +}; + + static PyMethodDef test_methods[] = { {"list_get_size", list_get_size, METH_O}, {"list_get_item", list_get_item, METH_VARARGS}, @@ -110,11 +358,26 @@ static PyMethodDef test_methods[] = { {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, {"test_list_api", test_list_api, METH_NOARGS}, + {"test_list_writer", test_list_writer, METH_NOARGS}, {NULL}, }; int _PyTestCapi_Init_List(PyObject *m) { - return PyModule_AddFunctions(m, test_methods); + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + + PyTypeObject *writer_type = (PyTypeObject *)PyType_FromSpec(&Writer_spec); + if (writer_type == NULL) { + return -1; + } + if (PyModule_AddType(m, writer_type) < 0) { + Py_DECREF(writer_type); + return -1; + } + Py_DECREF(writer_type); + + return 0; } diff --git a/Objects/listobject.c b/Objects/listobject.c index fa251923dfd4951..657c2e47b173f2c 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -1506,6 +1506,7 @@ list_extend_dictitems(PyListObject *self, PyDictObject *dict) return 0; } +// If you update this function, update also PyListWriter_Extend(). static int _list_extend(PyListObject *self, PyObject *iterable) { @@ -4386,3 +4387,191 @@ listiter_reduce_general(void *_it, int forward) } return Py_BuildValue("N(N)", iter, list); } + + +/* --- PyListWriter API --------------------------------------------------- */ + +#ifndef NDEBUG +static int +listwriter_check_consistency(PyListWriter *writer) +{ + assert(writer != NULL); + PyObject *list = _PyObject_CAST(writer); + // Don't test _PyObject_IsUniquelyReferenced() since a list writer can + // be shared between multiple threads with external locking. + assert(Py_REFCNT(list) == 1); + return 1; +} +#endif + + +PyListWriter* +PyListWriter_Create(Py_ssize_t size) +{ + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "size must not be negative"); + return NULL; + } + + PyObject *list = PyList_New(size); + if (list == NULL) { + return NULL; + } + Py_SET_SIZE(list, 0); + + PyListWriter *writer = (PyListWriter*)list; + assert(listwriter_check_consistency(writer)); + return writer; +} + + +void +PyListWriter_Discard(PyListWriter *writer) +{ + if (writer == NULL) { + return; + } + + assert(listwriter_check_consistency(writer)); + PyObject *list = _PyObject_CAST(writer); + Py_DECREF(list); +} + + +PyObject* +PyListWriter_Finish(PyListWriter *writer) +{ + assert(listwriter_check_consistency(writer)); + PyListObject *list = _PyList_CAST(writer); + +#ifdef Py_DEBUG + // Make sure that all items have been initialized + Py_ssize_t size = Py_SIZE(list); + PyObject **list_items = list->ob_item; + for (Py_ssize_t i = 0; i < size; i++) { + assert(list_items[i] != NULL); + } +#endif + + return _PyObject_CAST(list); +} + + +static int +listwriter_resize(PyListWriter *writer, Py_ssize_t new_size) +{ + PyListObject *list = _PyList_CAST(writer); + + // On Free Threading, it can call _PyObject_GC_SET_SHARED() on the list + return list_resize(list, new_size); +} + + +int +PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item) +{ + assert(listwriter_check_consistency(writer)); + PyListObject *list = _PyList_CAST(writer); + + Py_ssize_t len = Py_SIZE(list); + Py_ssize_t allocated = list->allocated; + if (len < allocated) { + PyList_SET_ITEM(list, len, item); + Py_SET_SIZE(list, len + 1); + return 0; + } + + if (listwriter_resize(writer, len + 1) < 0) { + Py_DECREF(item); + return -1; + } + list->ob_item[len] = item; + return 0; +} + + +int +PyListWriter_Append(PyListWriter *writer, PyObject *item) +{ + return PyListWriter_AppendSteal(writer, Py_NewRef(item)); +} + + +// If you update this function, update also _list_extend(). +int +PyListWriter_Extend(PyListWriter *writer, PyObject *iterable) +{ + assert(listwriter_check_consistency(writer)); + assert(_PyObject_CAST(writer) != iterable); + + PyListObject *self = _PyList_CAST(writer); + int res; + if (PyList_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(iterable); + res = list_extend_lock_held(self, iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyTuple_CheckExact(iterable)) { + res = list_extend_lock_held(self, iterable); + } + else if (PyAnySet_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(iterable); + res = list_extend_set(self, (PySetObject *)iterable); + Py_END_CRITICAL_SECTION(); + } + else if (PyDict_CheckExact(iterable)) { + Py_BEGIN_CRITICAL_SECTION(iterable); + res = list_extend_dict(self, (PyDictObject *)iterable, 0 /*keys*/); + Py_END_CRITICAL_SECTION(); + } + else if (Py_IS_TYPE(iterable, &PyDictKeys_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION(dict); + res = list_extend_dict(self, dict, 0 /*keys*/); + Py_END_CRITICAL_SECTION(); + } + else if (Py_IS_TYPE(iterable, &PyDictValues_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION(dict); + res = list_extend_dict(self, dict, 1 /*values*/); + Py_END_CRITICAL_SECTION(); + } + else if (Py_IS_TYPE(iterable, &PyDictItems_Type)) { + PyDictObject *dict = ((_PyDictViewObject *)iterable)->dv_dict; + Py_BEGIN_CRITICAL_SECTION(dict); + res = list_extend_dictitems(self, dict); + Py_END_CRITICAL_SECTION(); + } + else { + res = list_extend_iter_lock_held(self, iterable); + } + return res; +} + + +int +PyListWriter_ExtendArray(PyListWriter *writer, + PyObject *const *items, Py_ssize_t nitem) +{ + assert(listwriter_check_consistency(writer)); + assert(nitem >= 0); + + PyListObject *list = _PyList_CAST(writer); + Py_ssize_t size = Py_SIZE(list); + Py_ssize_t allocated = list->allocated; + Py_ssize_t new_size = size + nitem; + if (new_size > allocated) { + if (listwriter_resize(writer, new_size) < 0) { + return -1; + } + } + else { + Py_SET_SIZE(list, new_size); + } + + PyObject **list_items = &list->ob_item[size]; + for (Py_ssize_t i = 0; i < nitem; i++) { + list_items[i] = Py_NewRef(items[i]); + } + return 0; +} From d8be30db3cfeb0640c05d8d377b32e178ef57c05 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Oct 2026 17:13:51 -0500 Subject: [PATCH 02/10] Inline PyList_SET_ITEM() --- Objects/listobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index 657c2e47b173f2c..333b27306d53bb2 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4476,7 +4476,7 @@ PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item) Py_ssize_t len = Py_SIZE(list); Py_ssize_t allocated = list->allocated; if (len < allocated) { - PyList_SET_ITEM(list, len, item); + list->ob_item[len] = item; Py_SET_SIZE(list, len + 1); return 0; } From 398c4b38844d5a721484c52729f0f90cefb88339 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Oct 2026 17:15:17 -0500 Subject: [PATCH 03/10] Remove redundant test_list_writer() --- Modules/_testcapi/list.c | 50 ---------------------------------------- 1 file changed, 50 deletions(-) diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index cee09359f5bc41d..0160185cd83954a 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -103,55 +103,6 @@ test_list_api(PyObject *self, PyObject *Py_UNUSED(ignored)) } -static PyObject* -test_list_writer(PyObject *self, PyObject *Py_UNUSED(ignored)) -{ - PyObject *result = NULL; - - PyObject* array[3]; - for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { - array[i] = PyLong_FromLong(i); - if (array[i] == NULL) { - for (Py_ssize_t j = 0; j <= i; j++) { - Py_DECREF(array[i]); - } - return NULL; - } - } - - PyListWriter *writer = PyListWriter_Create(3); - if (writer == NULL) { - goto done; - } - - for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { - if (PyListWriter_Append(writer, array[i]) < 0) { - PyListWriter_Discard(writer); - goto done; - } - } - - PyObject *list = PyListWriter_Finish(writer); - if (list == NULL) { - goto done; - } - assert(PyList_CheckExact(list)); - assert(PyList_GET_SIZE(list) == 3); - assert(PyList_GET_ITEM(list, 0) == array[0]); - assert(PyList_GET_ITEM(list, 1) == array[1]); - assert(PyList_GET_ITEM(list, 2) == array[2]); - Py_DECREF(list); - - result = Py_None; - -done: - for (Py_ssize_t i = 0; i < (Py_ssize_t)Py_ARRAY_LENGTH(array); i++) { - Py_DECREF(array[i]); - } - return result; -} - - typedef struct { PyObject_HEAD PyListWriter *writer; @@ -358,7 +309,6 @@ static PyMethodDef test_methods[] = { {"list_clear", list_clear, METH_O}, {"list_extend", list_extend, METH_VARARGS}, {"test_list_api", test_list_api, METH_NOARGS}, - {"test_list_writer", test_list_writer, METH_NOARGS}, {NULL}, }; From 3df6e01c30946b1f3755c871cb47a12f8917be8f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 21 Oct 2026 17:32:26 -0500 Subject: [PATCH 04/10] Move _PyObject_GC_TRACK() to PyListWriter_Finish() --- Objects/listobject.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Objects/listobject.c b/Objects/listobject.c index 333b27306d53bb2..9eabe0fe4656496 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4400,6 +4400,7 @@ listwriter_check_consistency(PyListWriter *writer) // Don't test _PyObject_IsUniquelyReferenced() since a list writer can // be shared between multiple threads with external locking. assert(Py_REFCNT(list) == 1); + assert(!_PyObject_GC_IS_TRACKED(list)); return 1; } #endif @@ -4418,6 +4419,7 @@ PyListWriter_Create(Py_ssize_t size) return NULL; } Py_SET_SIZE(list, 0); + _PyObject_GC_UNTRACK(list); PyListWriter *writer = (PyListWriter*)list; assert(listwriter_check_consistency(writer)); @@ -4453,6 +4455,8 @@ PyListWriter_Finish(PyListWriter *writer) } #endif + // Now that the list is fully initialized, it's safe to track it in the GC + _PyObject_GC_TRACK(list); return _PyObject_CAST(list); } From f9d714500a96208212f37f85ca803ccda5b74a61 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 12:56:29 -0500 Subject: [PATCH 05/10] Clarify PyListWriter_ExtendArray() doc --- Doc/c-api/list.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 2756677ac492f7f..a2064107c2e10e6 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -305,7 +305,8 @@ On Free Threading, it's more efficient than the ``PyList`` C API to create a .. c:function:: int PyListWriter_ExtendArray(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem) - Extend the list writer with the contents of *items*. + Extend the list writer with the contents of *items*: copy references from + *items* to the list writer. On success, return ``0``. On error, set an exception and return ``-1``. From d096f46540645aed0c13b52f0e1aa8ec8c4ecdf1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 16:31:58 -0500 Subject: [PATCH 06/10] Use list_new_prealloc() in PyListWriter_Create() In release mode, no longer initialize items to NULL for better performance. The caller is responsible to initialize all items before calling PyListWriter_Finish(). --- Objects/listobject.c | 63 +++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index 38e82b376b0c622..3fb4ba1e07d7cf9 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -299,30 +299,48 @@ PyList_New(Py_ssize_t size) return (PyObject *) op; } +// Similar to PyList_New() but don't initialize memory and do not track +// the newly created list in the garbage collector. +// +// The list is initialized with a size of 0. +// +// size must be >= 0. static PyObject * list_new_prealloc(Py_ssize_t size) { - assert(size > 0); - PyListObject *op = (PyListObject *) PyList_New(0); + // The caller is responsible to check the size + assert(size >= 0); + + PyListObject *op = _Py_FREELIST_POP(PyListObject, lists); if (op == NULL) { - return NULL; + op = PyObject_GC_New(PyListObject, &PyList_Type); + if (op == NULL) { + return NULL; + } } - assert(op->ob_item == NULL); -#ifdef Py_GIL_DISABLED - _PyListArray *array = list_allocate_array(size); - if (array == NULL) { - Py_DECREF(op); - return PyErr_NoMemory(); + + if (size <= 0) { + op->ob_item = NULL; } - op->ob_item = array->ob_item; + else { +#ifdef Py_GIL_DISABLED + _PyListArray *array = list_allocate_array(size); + if (array == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + op->ob_item = array->ob_item; #else - op->ob_item = PyMem_New(PyObject *, size); - if (op->ob_item == NULL) { - Py_DECREF(op); - return PyErr_NoMemory(); - } + op->ob_item = (PyObject **) PyMem_Malloc(size * sizeof(PyObject *)); #endif + if (op->ob_item == NULL) { + Py_DECREF(op); + return PyErr_NoMemory(); + } + } + Py_SET_SIZE(op, 0); op->allocated = size; + return (PyObject *) op; } @@ -766,6 +784,7 @@ list_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) dest[i] = Py_NewRef(v); } Py_SET_SIZE(np, len); + _PyObject_GC_TRACK(np); return (PyObject *)np; } @@ -848,6 +867,7 @@ list_concat_lock_held(PyListObject *a, PyListObject *b) dest[i] = Py_NewRef(v); } Py_SET_SIZE(np, size); + _PyObject_GC_TRACK(np); return (PyObject *)np; } @@ -918,6 +938,7 @@ list_repeat_impl(PyListObject *a, Py_ssize_t n) } Py_SET_SIZE(np, output_size); + _PyObject_GC_TRACK(np); return (PyObject *) np; } @@ -3723,6 +3744,7 @@ list_slice_step_lock_held(PyListObject *a, Py_ssize_t start, Py_ssize_t step, Py dest[i] = Py_NewRef(v); } Py_SET_SIZE(np, len); + _PyObject_GC_TRACK(np); return (PyObject *)np; } @@ -4415,12 +4437,17 @@ PyListWriter_Create(Py_ssize_t size) return NULL; } - PyObject *list = PyList_New(size); + PyObject *list = list_new_prealloc(size); if (list == NULL) { return NULL; } - Py_SET_SIZE(list, 0); - _PyObject_GC_UNTRACK(list); + assert(Py_SIZE(list) == 0); + +#ifdef Py_DEBUG + // In debug mode, set all items to NULL to be able to check that they are + // non-NULL in PyListWriter_Finish(). + memset(_PyList_CAST(list)->ob_item, 0, size * sizeof(PyObject*)); +#endif PyListWriter *writer = (PyListWriter*)list; assert(listwriter_check_consistency(writer)); From a8c24ee4203a0d9efb22bdaed823d5802f21b731 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 16:31:58 -0500 Subject: [PATCH 07/10] Update PyListWriter_Create() comment --- Objects/listobject.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Objects/listobject.c b/Objects/listobject.c index 3fb4ba1e07d7cf9..2c773e3d3b9be71 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4442,6 +4442,9 @@ PyListWriter_Create(Py_ssize_t size) return NULL; } assert(Py_SIZE(list) == 0); + // In release mode, list items are uninitialized pointers which is safe, + // since the list object is not tracked by the GC, and the writer caller + // must initialize all items. #ifdef Py_DEBUG // In debug mode, set all items to NULL to be able to check that they are From 31e5f1b8fb273fa9942cba538c03feadf93d36bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 16:53:10 -0500 Subject: [PATCH 08/10] Add PyListWriter_ExtendArraySteal() function --- Doc/c-api/list.rst | 14 ++++++- Doc/whatsnew/3.16.rst | 1 + Include/cpython/listobject.h | 4 ++ Lib/test/test_capi/test_list.py | 13 ++++++- ...-10-21-16-35-42.gh-issue-158660.KvixNp.rst | 1 + Modules/_testcapi/list.c | 29 +++++++++++++++ Objects/listobject.c | 37 ++++++++++++++++--- 7 files changed, 90 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index a2064107c2e10e6..c2fc1fb7dbf78e0 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -292,8 +292,8 @@ On Free Threading, it's more efficient than the ``PyList`` C API to create a .. c:function:: int PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item) - Similar to :c:func:`PyListWriter_Append`, but steals the reference to - *item*. + Similar to :c:func:`PyListWriter_Append`, but :term:`steals ` the + reference to *item*, instead of creating a new :ref:`strong reference`. .. c:function:: int PyListWriter_Extend(PyListWriter *writer, PyObject *iterable) @@ -310,3 +310,13 @@ On Free Threading, it's more efficient than the ``PyList`` C API to create a On success, return ``0``. On error, set an exception and return ``-1``. + +.. c:function:: int PyListWriter_ExtendArraySteal(PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem) + + Similar to :c:func:`PyListWriter_ExtendArray`, but :term:`steal` references + instead of copying references. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + On error, all *items* references are also cleared. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 36ccf0e4365f020..0b91987aace7ba1 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -1322,6 +1322,7 @@ New features * :c:func:`PyListWriter_AppendSteal` * :c:func:`PyListWriter_Extend` * :c:func:`PyListWriter_ExtendArray` + * :c:func:`PyListWriter_ExtendArraySteal` (Contributed by Victor Stinner in :gh:`158660`.) diff --git a/Include/cpython/listobject.h b/Include/cpython/listobject.h index cbc4413f594d1e5..a0843bf185e5145 100644 --- a/Include/cpython/listobject.h +++ b/Include/cpython/listobject.h @@ -73,3 +73,7 @@ PyAPI_FUNC(int) PyListWriter_ExtendArray( PyListWriter *writer, PyObject *const *items, Py_ssize_t nitem); +PyAPI_FUNC(int) PyListWriter_ExtendArraySteal( + PyListWriter *writer, + PyObject *const *items, + Py_ssize_t nitem); diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index 812ca25ab39bdf1..c6cf6ab223be9e7 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -370,8 +370,13 @@ def list_equal(self, result, expected): self.assertEqual(result, expected) self.assertEqual(type(result), list) + def create_items(self, nitem): + # Call object() to create unique objects. Don't use small integers + # which are immortal (and so would not check for reference leaks). + return tuple(object() for i in range(nitem)) + def check_append(self, method): - items = (1, 2, 3) + items = self.create_items(3) for size in (0, len(items), len(items) * 10): with self.subTest(size=size): writer = self.create_writer(size) @@ -389,7 +394,7 @@ def test_append_steal(self): self.check_append('append_steal') def check_extend(self, method): - items = (2, 5, 7) + items = self.create_items(5) for size in (0, len(items), len(items) * 10): with self.subTest(size=size): writer = self.create_writer(size) @@ -405,6 +410,10 @@ def test_extend_array(self): # Test PyListWriter_ExtendArray() self.check_extend('extend_array') + def test_extend_array_steal(self): + # Test PyListWriter_ExtendArraySteal() + self.check_extend('extend_array_steal') + def test_discard(self): # Test PyListWriter_Discard(): must not leak memory writer = self.create_writer() diff --git a/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst b/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst index b765ccc33c0e865..bbedca240016512 100644 --- a/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst +++ b/Misc/NEWS.d/next/C_API/2026-10-21-16-35-42.gh-issue-158660.KvixNp.rst @@ -10,5 +10,6 @@ created list. Add functions: * :c:func:`PyListWriter_AppendSteal` * :c:func:`PyListWriter_Extend` * :c:func:`PyListWriter_ExtendArray` +* :c:func:`PyListWriter_ExtendArraySteal` Patch by Victor Stinner. diff --git a/Modules/_testcapi/list.c b/Modules/_testcapi/list.c index 0160185cd83954a..4e1e30c80d63fa9 100644 --- a/Modules/_testcapi/list.c +++ b/Modules/_testcapi/list.c @@ -246,6 +246,34 @@ writer_extend_array(PyObject *self_raw, PyObject *tuple) } +// Test PyListWriter_ExtendArraySteal() +static PyObject* +writer_extend_array_steal(PyObject *self_raw, PyObject *tuple) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { + return NULL; + } + + if (!PyTuple_Check(tuple)) { + PyErr_Format(PyExc_TypeError, + "expect tuple, got %T", tuple); + return NULL; + } + PyObject **items = &PyTuple_GET_ITEM(tuple, 0); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + + for (Py_ssize_t i = 0; i < len; i++) { + Py_INCREF(items[i]); + } + + if (PyListWriter_ExtendArraySteal(self->writer, items, len) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + // Test PyListWriter_Finish() static PyObject* writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) @@ -281,6 +309,7 @@ static PyMethodDef writer_methods[] = { {"append_steal", _PyCFunction_CAST(writer_append_steal), METH_O}, {"extend", _PyCFunction_CAST(writer_extend), METH_O}, {"extend_array", _PyCFunction_CAST(writer_extend_array), METH_O}, + {"extend_array_steal", _PyCFunction_CAST(writer_extend_array_steal), METH_O}, {"finish", _PyCFunction_CAST(writer_finish), METH_NOARGS}, {"discard", _PyCFunction_CAST(writer_discard), METH_NOARGS}, {NULL, NULL} /* sentinel */ diff --git a/Objects/listobject.c b/Objects/listobject.c index 2c773e3d3b9be71..a09759701280076 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4584,9 +4584,10 @@ PyListWriter_Extend(PyListWriter *writer, PyObject *iterable) } -int -PyListWriter_ExtendArray(PyListWriter *writer, - PyObject *const *items, Py_ssize_t nitem) +static int +listwriter_extend_array(PyListWriter *writer, + PyObject *const *items, Py_ssize_t nitem, + int steal) { assert(listwriter_check_consistency(writer)); assert(nitem >= 0); @@ -4597,6 +4598,11 @@ PyListWriter_ExtendArray(PyListWriter *writer, Py_ssize_t new_size = size + nitem; if (new_size > allocated) { if (listwriter_resize(writer, new_size) < 0) { + if (steal) { + for (Py_ssize_t i = 0; i < nitem; i++) { + Py_DECREF(items[i]); + } + } return -1; } } @@ -4605,8 +4611,29 @@ PyListWriter_ExtendArray(PyListWriter *writer, } PyObject **list_items = &list->ob_item[size]; - for (Py_ssize_t i = 0; i < nitem; i++) { - list_items[i] = Py_NewRef(items[i]); + if (steal) { + memcpy(list_items, items, nitem * sizeof(PyObject*)); + } + else { + for (Py_ssize_t i = 0; i < nitem; i++) { + list_items[i] = Py_NewRef(items[i]); + } } return 0; } + + +int +PyListWriter_ExtendArray(PyListWriter *writer, + PyObject *const *items, Py_ssize_t nitem) +{ + return listwriter_extend_array(writer, items, nitem, 0); +} + + +int +PyListWriter_ExtendArraySteal(PyListWriter *writer, + PyObject *const *items, Py_ssize_t nitem) +{ + return listwriter_extend_array(writer, items, nitem, 1); +} From db8eb510bb8cfc9405671e9d0c1e400c0c909ac5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 22:15:29 -0700 Subject: [PATCH 09/10] Fix doc: use :term: not :ref: --- Doc/c-api/list.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c2fc1fb7dbf78e0..4b85e5833cde933 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -293,7 +293,7 @@ On Free Threading, it's more efficient than the ``PyList`` C API to create a .. c:function:: int PyListWriter_AppendSteal(PyListWriter *writer, PyObject *item) Similar to :c:func:`PyListWriter_Append`, but :term:`steals ` the - reference to *item*, instead of creating a new :ref:`strong reference`. + reference to *item*, instead of creating a new :term:`strong reference`. .. c:function:: int PyListWriter_Extend(PyListWriter *writer, PyObject *iterable) From da86200cc0952f9bdb19d6e4fd7ba2936e1c9568 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 22 Oct 2026 22:28:33 -0700 Subject: [PATCH 10/10] Avoid undefined behavior: memset() with size=0 --- Objects/listobject.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Objects/listobject.c b/Objects/listobject.c index a09759701280076..3d7db01f3dc6f60 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -4449,7 +4449,9 @@ PyListWriter_Create(Py_ssize_t size) #ifdef Py_DEBUG // In debug mode, set all items to NULL to be able to check that they are // non-NULL in PyListWriter_Finish(). - memset(_PyList_CAST(list)->ob_item, 0, size * sizeof(PyObject*)); + if (1 <= size) { + memset(_PyList_CAST(list)->ob_item, 0, size * sizeof(PyObject*)); + } #endif PyListWriter *writer = (PyListWriter*)list;