diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index c03816b5727fa4..fbdad0f346550d 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -305,8 +305,8 @@ object. A bytes writer object. - The API is **not thread safe**. A :c:type:`PyBytesWriter` object must only - be used by a single thread, it must not be shared between threads. + The API is **not thread safe**. To share a writer with multiple threads, a + critical section or a lock is needed. The instance must be destroyed by :c:func:`PyBytesWriter_Finish` on success, or :c:func:`PyBytesWriter_Discard` on error. diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 9bf801ad608c77..8ac4e137709555 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1797,6 +1797,9 @@ object. The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on success, or :c:func:`PyUnicodeWriter_Discard` on error. + The API is **not thread safe**. To share a writer with multiple threads, a + critical section or a lock is needed. + .. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) Create a Unicode writer instance. diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 012f5da2869cd5..e9a4aed37030e7 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -16,7 +16,9 @@ extern "C" { #define _Py_MAX_UNICODE 0x10ffff -extern int _PyUnicode_IsModifiable(PyObject *unicode); +// Export for '_multibytecodec' shared extension. _PyUnicodeWriter_CanWrite() +// calls this function when assertions are enabled. +PyAPI_FUNC(int) _PyUnicode_IsModifiable(PyObject *unicode); extern void _PyUnicodeWriter_InitWithBuffer( _PyUnicodeWriter *writer, PyObject *buffer); @@ -105,12 +107,32 @@ _PyUnicode_EnsureUnicode(PyObject *obj) return 0; } +#ifndef NDEBUG +static inline int +_PyUnicodeWriter_CanWrite(_PyUnicodeWriter *writer) +{ + // Code adapted from _PyUnicode_IsModifiable() + assert(!writer->readonly); + PyObject *buffer = writer->buffer; + assert(buffer != NULL); + // Do not use _PyObject_IsUniquelyReferenced(): the caller can have its own + // lock to prevent a writer from being used by two threads at the same + // time. + assert(Py_REFCNT(buffer) == 1); + assert(PyUnstable_Unicode_GET_CACHED_HASH(buffer) == -1); + assert(!PyUnicode_CHECK_INTERNED(buffer)); + assert(!_Py_IsImmortal(buffer)); + return 1; +} +#endif + static inline int _PyUnicodeWriter_WriteCharInline(_PyUnicodeWriter *writer, Py_UCS4 ch) { assert(ch <= _Py_MAX_UNICODE); if (_PyUnicodeWriter_Prepare(writer, 1, ch) < 0) return -1; + assert(_PyUnicodeWriter_CanWrite(writer)); PyUnicode_WRITE(writer->kind, writer->data, writer->pos, ch); writer->pos++; return 0; diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index b68412a02b3228..1356e5d6c51c14 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -1,8 +1,10 @@ import sys import textwrap +import threading import unittest from test import support from test.support import import_helper +from test.support import threading_helper from test.support.script_helper import assert_python_failure _testlimitedcapi = import_helper.import_module('_testlimitedcapi') @@ -461,6 +463,14 @@ def test_resize(self): self.assertEqual(writer.finish(), b's' * small + b'L' * (large - small)) + # Make sure that it's possible to write after a resize to zero + # when a bytes/bytearray object is allocated. + writer = self.create_writer() + writer.resize(self.LARGE_BUFFER) + writer.resize(0) + writer.write_bytes(b'abc', 3) + self.assertEqual(writer.finish(), b'abc') + # invalid size for size in (self.SMALL_BUFFER, self.LARGE_BUFFER): with self.subTest(size=size): @@ -665,6 +675,36 @@ def test_get_data_canary(self): self.assertEqual(get_data_canary(writer), b'abc123' + CANARY_BYTE) + @threading_helper.requires_working_threading() + def test_thread(self): + # PyBytesWriter can be used by multiple threads: it's up to the caller + # to implement a lock to prevent concurrent accesses. + writer = self.create_writer(0) + size = None + data = None + + def thread_func(writer, LARGE_BUFFER): + nonlocal size, data + + # create a bytes object for the buffer + writer.write_bytes(b'x' * LARGE_BUFFER, LARGE_BUFFER) + + # so we can check a write with a bytes object + writer.write_bytes(b'yz', 2) + writer.format_i(b'i=%i', 5) + writer.resize(10) + data = writer.get_data() + size = writer.get_size() + + thread = threading.Thread(target=thread_func, + args=(writer, self.LARGE_BUFFER)) + thread.start() + threading_helper.join_thread(thread) + + self.assertEqual(size, 10) + self.assertEqual(data, b'x' * 10) + self.assertEqual(writer.finish(), b'x' * 10) + class BytesWriterTest(BaseWriterTest, unittest.TestCase): RESULT_TYPE = bytes diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 0dcd8a25ad0128..f2b77e3fdb5fc4 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1,7 +1,9 @@ -import unittest import sys +import textwrap +import unittest from test import support from test.support import threading_helper +from test.support.script_helper import assert_python_failure try: import _testcapi @@ -1981,6 +1983,33 @@ def test_substring_empty(self): writer.write_substring("abc", 1, 1) self.assertEqual(writer.finish(), '') + def test_singletons(self): + writer = self.create_writer(5) + self.assertIs(writer.finish(), '') + + for ch in range(256): + with self.subTest(ch=ch): + ch = chr(ch) + writer = self.create_writer(0) + writer.write_substring(ch + 'xxx', 0, 1) + self.assertIs(writer.finish(), ch) + + @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') + def test_detect_overflow(self): + # Test detection of buffer overflow + code = textwrap.dedent(''' + from test.support import SuppressCrashReport + import _testinternalcapi + + SuppressCrashReport().__enter__() + _testinternalcapi.unicodewriter_overflow() + ''') + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in PyUnicodeWriter', proc.err) + # Do not test the position value since it depends on the overallocation + # strategy which depends on the operating system + self.assertIn(f'at position '.encode(), proc.err) + @unittest.skipIf(ctypes is None, 'need ctypes') class PyUnicodeWriterFormatTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst b/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst new file mode 100644 index 00000000000000..532978485af85a --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst @@ -0,0 +1,3 @@ +When Python is built in debug mode, :c:func:`PyUnicodeWriter_Finish` now +checks if the trailing null byte has been overridden to detect buffer +overflow. Patch by Victor Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae7042098..d30affdbf62139 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -3206,6 +3206,28 @@ test_thread_state_ensure_from_view_interp_switch(PyObject *self, PyObject *unuse Py_RETURN_NONE; } +static PyObject * +unicodewriter_overflow(PyObject *self, PyObject *unused) +{ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } + if (PyUnicodeWriter_WriteASCII(writer, "hello", -1) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + + _PyUnicodeWriter *impl = (_PyUnicodeWriter*)writer; + PyObject *buffer = impl->buffer; + Py_ssize_t index = PyUnicode_GET_LENGTH(buffer); + PyUnicode_WRITE(impl->kind, impl->data, index, '#'); // overflow! + + // Spoiler: the function doesn't return if an overflow is detected + // in debug mode + return PyUnicodeWriter_Finish(writer); +} + /* Self interrupting context manager */ typedef struct { @@ -3393,6 +3415,7 @@ static PyMethodDef module_functions[] = { {"test_interp_guard_countdown", test_interp_guard_countdown, METH_NOARGS}, {"test_interp_view_countdown", test_interp_view_countdown, METH_NOARGS}, {"test_thread_state_ensure_from_view_interp_switch", test_thread_state_ensure_from_view_interp_switch, METH_NOARGS}, + {"unicodewriter_overflow", unicodewriter_overflow, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 38ec7a4aefcf56..7fefd64eefb2ac 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3329,18 +3329,22 @@ PyBytes_ConcatAndDel(PyObject **pv, PyObject *w) // // Usage: assert(_PyBytes_IsMutable(obj)). int -_PyBytes_IsMutable(PyObject *v) +_PyBytes_IsMutable(PyObject *self) { - // Singleton objects must never be modified - assert(!_Py_IsImmortal(v)); + assert(PyBytes_Check(self)); + // Do not use _PyObject_IsUniquelyReferenced(): this function is called + // by bytearray and PyBytesWriter which can be used by multiple threads. + assert(Py_REFCNT(self) == 1); + assert(!_Py_IsImmortal(self)); - Py_ssize_t size = PyBytes_GET_SIZE(v); + // Check that the object is not a singleton + Py_ssize_t size = PyBytes_GET_SIZE(self); if (size == 0) { - assert(v != bytes_get_empty()); + assert(self != bytes_get_empty()); } else if (size == 1) { - unsigned char ch = PyBytes_AS_STRING(v)[0]; - assert(v != (PyObject*)CHARACTER(ch)); + unsigned char ch = PyBytes_AS_STRING(self)[0]; + assert(self != (PyObject*)CHARACTER(ch)); } return 1; } @@ -3675,20 +3679,6 @@ byteswriter_allocated(PyBytesWriter *writer) #ifdef Py_DEBUG -static void -byteswriter_check_canary_byte(PyBytesWriter *writer) -{ - const unsigned char *data = (const unsigned char*)byteswriter_data(writer); - unsigned char canary = data[writer->size]; - if (canary != PyBytesWriter_CANARY_BYTE) { - _Py_FatalErrorFormat(__func__, - "Buffer overflow detected in PyBytesWriter %p " - "at position %zd", - writer, writer->size); - } -} - - static void byteswriter_write_canary_byte(PyBytesWriter *writer) { @@ -3710,6 +3700,45 @@ byteswriter_reset_trailing_byte(PyBytesWriter *writer) #endif +#ifndef NDEBUG +static int +byteswriter_check_consistency(PyBytesWriter *writer) +{ + PyObject *obj = writer->obj; + if (obj != NULL) { + if (writer->use_bytearray) { + assert(PyByteArray_CheckExact(obj)); + // Do not use _PyObject_IsUniquelyReferenced(): the caller can have + // its own lock to prevent a writer from being used by two threads + // at the same time. + assert(Py_REFCNT(obj) == 1); + PyByteArrayObject *bytearray = (PyByteArrayObject*)obj; + obj = bytearray->ob_bytes_object; + assert(obj != NULL); + } + + // Code adapted from _PyBytes_IsMutable() + assert(PyBytes_CheckExact(obj)); + assert(_PyBytes_IsMutable(obj)); + // -1 since the last small buffer byte is used as the canary byte + assert((size_t)PyBytes_GET_SIZE(obj) > (sizeof(writer->small_buffer) - 1)); + } + +#ifdef Py_DEBUG + const unsigned char *data = (const unsigned char*)byteswriter_data(writer); + unsigned char canary = data[writer->size]; + if (canary != PyBytesWriter_CANARY_BYTE) { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in PyBytesWriter %p " + "at position %zd", + writer, writer->size); + } +#endif + return 1; +} +#endif + + #ifdef MS_WINDOWS /* On Windows, overallocate by 50% is the best factor */ # define OVERALLOCATE_FACTOR 2 @@ -3743,6 +3772,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) // bytearray can override the canary byte on error byteswriter_write_canary_byte(writer); #endif + assert(byteswriter_check_consistency(writer)); return -1; } } @@ -3750,6 +3780,7 @@ byteswriter_resize(PyBytesWriter *writer, Py_ssize_t new_size, int resize) // Can raise MemoryError or OverflowError if (_PyBytes_ResizeKeepOnError(&writer->obj, alloc)) { assert(writer->obj != NULL); + assert(byteswriter_check_consistency(writer)); return -1; } assert(_PyBytes_IsMutable(writer->obj)); @@ -3821,7 +3852,7 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) if (size >= 1) { if (byteswriter_resize(writer, size, 0) < 0) { #ifdef Py_DEBUG - // Write the canary byte so byteswriter_check_canary_byte() + // Write the canary byte so byteswriter_check_consistency() // doesn't fail in PyBytesWriter_Discard() byteswriter_write_canary_byte(writer); #endif @@ -3835,6 +3866,7 @@ byteswriter_create(Py_ssize_t size, int use_bytearray) byteswriter_allocated(writer)); byteswriter_write_canary_byte(writer); #endif + assert(byteswriter_check_consistency(writer)); return writer; } @@ -3858,8 +3890,8 @@ PyBytesWriter_Discard(PyBytesWriter *writer) return; } + assert(byteswriter_check_consistency(writer)); #ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); if (writer->obj != NULL) { byteswriter_reset_trailing_byte(writer); } @@ -3873,6 +3905,8 @@ PyBytesWriter_Discard(PyBytesWriter *writer) PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) { + assert(byteswriter_check_consistency(writer)); + // Check for negative size here to raise ValueError in all cases, rather // than having a different exception depending on the code path. For // example, _PyBytes_Resize() raises SystemError on negative size. @@ -3886,10 +3920,6 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) goto error; } -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif - PyObject *result; if (size == 0) { result = bytes_get_empty(); @@ -3938,7 +3968,7 @@ PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) } #ifdef Py_DEBUG - // Reset the writer, so byteswriter_check_canary_byte() doesn't fail + // Reset the writer, so byteswriter_check_consistency() doesn't fail // in PyBytesWriter_Discard(). writer->size = 0; byteswriter_write_canary_byte(writer); @@ -3970,9 +4000,7 @@ PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) void* PyBytesWriter_GetData(PyBytesWriter *writer) { -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif + assert(byteswriter_check_consistency(writer)); return byteswriter_data(writer); } @@ -3981,9 +4009,7 @@ PyBytesWriter_GetData(PyBytesWriter *writer) Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) { -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif + assert(byteswriter_check_consistency(writer)); return _PyBytesWriter_GetSize(writer); } @@ -3992,9 +4018,7 @@ PyBytesWriter_GetSize(PyBytesWriter *writer) int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t new_size) { -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif + assert(byteswriter_check_consistency(writer)); if (new_size < 0) { PyErr_SetString(PyExc_ValueError, "size must be >= 0"); @@ -4012,6 +4036,7 @@ PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t new_size) #ifdef Py_DEBUG byteswriter_write_canary_byte(writer); #endif + assert(byteswriter_check_consistency(writer)); return 0; } @@ -4031,9 +4056,7 @@ _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) { -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif + assert(byteswriter_check_consistency(writer)); if (grow == 0) { // Nothing to do @@ -4064,6 +4087,7 @@ PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) #ifdef Py_DEBUG byteswriter_write_canary_byte(writer); #endif + assert(byteswriter_check_consistency(writer)); return 0; } @@ -4099,6 +4123,8 @@ PyBytesWriter_WriteBytes(PyBytesWriter *writer, } char *buf = byteswriter_data(writer); memcpy(buf + pos, bytes, size); + + assert(byteswriter_check_consistency(writer)); return 0; } @@ -4127,14 +4153,12 @@ PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) static Py_ssize_t _PyBytesWriter_ResizeToAllocated(PyBytesWriter *writer) { -#ifdef Py_DEBUG - byteswriter_check_canary_byte(writer); -#endif - Py_ssize_t allocated = byteswriter_allocated(writer); writer->size = allocated; #ifdef Py_DEBUG byteswriter_write_canary_byte(writer); #endif + + assert(byteswriter_check_consistency(writer)); return allocated; } diff --git a/Objects/longobject.c b/Objects/longobject.c index e35f938629326a..1dac70820d142b 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -2220,6 +2220,7 @@ long_to_decimal_string_internal(PyObject *aa, Py_DECREF(scratch); return -1; } + assert(_PyUnicodeWriter_CanWrite(writer)); } else if (bytes_writer) { *bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, strlen, @@ -2390,8 +2391,10 @@ long_format_binary(PyObject *aa, int base, int alternate, } if (writer) { - if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) + if (_PyUnicodeWriter_Prepare(writer, sz, 'x') == -1) { return -1; + } + assert(_PyUnicodeWriter_CanWrite(writer)); } else if (bytes_writer) { *bytes_str = PyBytesWriter_GrowAndUpdatePointer(bytes_writer, sz, diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index a753c9b971c702..fe1bd97775b3ae 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -350,6 +350,8 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) if (_PyUnicodeWriter_PrepareInternal(writer, len, maxchar) == -1) return -1; } + + assert(_PyUnicodeWriter_CanWrite(writer)); _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, 0, len); writer->pos += len; @@ -428,6 +430,7 @@ _PyUnicodeWriter_WriteSubstring(_PyUnicodeWriter *writer, PyObject *str, if (_PyUnicodeWriter_Prepare(writer, len, maxchar) < 0) { return -1; } + assert(_PyUnicodeWriter_CanWrite(writer)); _PyUnicode_FastCopyCharacters(writer->buffer, writer->pos, str, start, len); @@ -485,8 +488,10 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } - if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) + if (_PyUnicodeWriter_Prepare(writer, len, 127) == -1) { return -1; + } + assert(_PyUnicodeWriter_CanWrite(writer)); switch (writer->kind) { @@ -591,6 +596,7 @@ _PyUnicodeWriter_WriteLatin1String(_PyUnicodeWriter *writer, maxchar = ucs1lib_find_max_char((const Py_UCS1*)str, (const Py_UCS1*)str + len); if (_PyUnicodeWriter_Prepare(writer, len, maxchar) == -1) return -1; + assert(_PyUnicodeWriter_CanWrite(writer)); unicode_write_cstr(writer->buffer, writer->pos, str, len); writer->pos += len; return 0; @@ -602,6 +608,20 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) { PyObject *str; +#ifdef Py_DEBUG + // Check for buffer overflow + if (writer->buffer != NULL) { + Py_ssize_t pos = PyUnicode_GET_LENGTH(writer->buffer); + Py_UCS4 ch = PyUnicode_READ_CHAR(writer->buffer, pos); + if (ch != 0) { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in " + "PyUnicodeWriter %p at position %zd", + writer, pos); + } + } +#endif + if (writer->pos == 0) { Py_CLEAR(writer->buffer); return _PyUnicode_GetEmpty(); @@ -612,6 +632,7 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) if (writer->readonly) { assert(PyUnicode_GET_LENGTH(str) == writer->pos); + assert(_PyUnicode_CheckConsistency(str, 1)); return str; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 86b9baadd0d8aa..7ee9d17b7f77ff 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -601,7 +601,6 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) # define CHECK_IF_FT(expr) (void)(expr) #endif - assert(op != NULL); CHECK(PyUnicode_Check(op)); @@ -647,13 +646,12 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) } /* check that the best kind is used: O(n) operation */ + const void *data = PyUnicode_DATA(ascii); if (check_content) { Py_ssize_t i; Py_UCS4 maxchar = 0; - const void *data; Py_UCS4 ch; - data = PyUnicode_DATA(ascii); for (i=0; i < ascii->length; i++) { ch = PyUnicode_READ(kind, data, i); @@ -676,9 +674,12 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) CHECK(maxchar >= 0x10000); CHECK(maxchar <= MAX_UNICODE); } - CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); } + // Detect buffer overflow: check if the trailing null character + // has been overridden + CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); + /* Check interning state */ #ifdef Py_DEBUG // Note that we do not check `_Py_IsImmortal(op)` in the GIL-enabled build @@ -1743,18 +1744,21 @@ unicode_is_singleton(PyObject *unicode) } #endif +// If this function is updated, update also _PyUnicodeWriter_CanWrite(). int _PyUnicode_IsModifiable(PyObject *unicode) { assert(_PyUnicode_CHECK(unicode)); + if (!PyUnicode_CheckExact(unicode)) + return 0; + // On Free Threading, this test fails if called from a thread other + // than the one which created the str object. if (!_PyObject_IsUniquelyReferenced(unicode)) return 0; if (PyUnicode_HASH(unicode) != -1) return 0; if (PyUnicode_CHECK_INTERNED(unicode)) return 0; - if (!PyUnicode_CheckExact(unicode)) - return 0; #ifdef Py_DEBUG /* singleton refcount is greater than 1 */ assert(!unicode_is_singleton(unicode)); @@ -2008,6 +2012,7 @@ PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *pub_writer, if (_PyUnicodeWriter_Prepare(writer, size - num_surrogates, maxchar) < 0) { return -1; } + assert(_PyUnicodeWriter_CanWrite(writer)); int kind = writer->kind; void *data = (Py_UCS1*)writer->data + writer->pos * kind; @@ -2266,6 +2271,7 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer, if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) { return -1; } + assert(_PyUnicodeWriter_CanWrite(writer)); int kind = writer->kind; void *data = (Py_UCS1*)writer->data + writer->pos * kind; @@ -2552,8 +2558,10 @@ unicode_fromformat_write_str(_PyUnicodeWriter *writer, PyObject *str, else maxchar = writer->maxchar; - if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) + if (_PyUnicodeWriter_Prepare(writer, arglen, maxchar) == -1) { return -1; + } + assert(_PyUnicodeWriter_CanWrite(writer)); fill = Py_MAX(width - length, 0); if (fill && !(flags & F_LJUST)) { @@ -2843,8 +2851,10 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, Py_ssize_t spacepad = Py_MAX(width - precision - sign, 0); Py_ssize_t zeropad = Py_MAX(precision - len, 0); - if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1) + if (_PyUnicodeWriter_Prepare(writer, width, 127) == -1) { return NULL; + } + assert(_PyUnicodeWriter_CanWrite(writer)); if (spacepad && !(flags & F_LJUST)) { if (PyUnicode_Fill(writer->buffer, writer->pos, spacepad, ' ') == -1) @@ -5371,6 +5381,7 @@ _PyUnicode_DecodeUTF8Writer(_PyUnicodeWriter *writer, if (_PyUnicodeWriter_Prepare(writer, size, 127) < 0) { return -1; } + assert(_PyUnicodeWriter_CanWrite(writer)); const char *starts = s; const char *end = s + size;