Skip to content

Commit 09a7956

Browse files
vstinnervstinner
authored
[3.14] gh-158445: Detect invalid UCS4 characters in debug mode (#158502) (#159524) (#159540)
[3.15] gh-158445: Detect invalid UCS4 characters in debug mode (#158502) (#159524) * gh-158445: Detect invalid UCS4 characters in debug mode (#158502) In debug mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) and PyUnicodeWriter_WriteUCS4() now raise a SystemError if a character is not in range [U+0000; U+10ffff], instead of creating an invalid str object. In release mode, PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND) raises a SystemError if the size is 1. * In debug mode, unicode_char() no longer fails with an assertion error if the character is invalid. Instead, raise SystemError. * Add tests on invalid UCS-4 characters. * Add unicode_invalid_character() helper function. (cherry picked from commit a14a4a6) * gh-158445: Skip invalid UCS-4 tests if Python is built with assertions (#158604) If Python is built in release mode with assertions, PyUnicode_FromKindAndData() and PyUnicodeWriter_Finish() fail with an assertion error on invalid characters. (cherry picked from commit 35bdbca) (cherry picked from commit 2c23505)
1 parent 21aae35 commit 09a7956

3 files changed

Lines changed: 138 additions & 30 deletions

File tree

‎Doc/c-api/unicode.rst‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,6 +412,10 @@ APIs:
412412
the UCS1 range, it will be transformed into UCS1
413413
(:c:macro:`PyUnicode_1BYTE_KIND`).
414414
415+
All characters must be in range [U+0000; U+10ffff]. If *kind* is
416+
:c:macro:`PyUnicode_4BYTE_KIND` and the string contains invalid characters,
417+
the behavior is undefined.
418+
415419
.. versionadded:: 3.3
416420
417421
@@ -1880,10 +1884,13 @@ object.
18801884
18811885
.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size)
18821886
1883-
Writer the UCS4 string *str* into *writer*.
1887+
Write the UCS4 string *str* into *writer*.
18841888
18851889
*size* is a number of UCS4 characters.
18861890
1891+
All characters must be in range [U+0000; U+10ffff]. If the string contains
1892+
invalid characters, the behavior is undefined.
1893+
18871894
On success, return ``0``.
18881895
On error, set an exception, leave the writer unchanged, and return ``-1``.
18891896

‎Lib/test/test_capi/test_unicode.py‎

Lines changed: 71 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,11 @@
1717
from _testcapi import PY_SSIZE_T_MIN, PY_SSIZE_T_MAX, SIZEOF_WCHAR_T
1818

1919

20+
MAX_UNICODE = 0x10_ffff
21+
# The first invalid character after MAX_UNICODE
22+
INVALID_CHAR = MAX_UNICODE + 1
23+
# Maximum invalid character which fits into 32-bit Py_UCS4
24+
MAX_INVALID_CHAR = 0xFFFF_FFFF
2025
NULL = None
2126

2227
class Str(str):
@@ -39,6 +44,12 @@ def __str__(self):
3944
SSTATE_INTERNED_IMMORTAL_STATIC = 3
4045

4146

47+
def assert_invalid_string(testcase, text):
48+
# Check that a Unicode string contains invalid characters:
49+
# not in range [U+0000; U+10ffff]
50+
testcase.assertRaises(SystemError, list, text)
51+
52+
4253
class CAPITest(unittest.TestCase):
4354

4455
def _test_check(self, check, *, exact):
@@ -71,14 +82,14 @@ def test_new(self):
7182
self.assertEqual(new(0, maxchar), '')
7283
self.assertEqual(new(5, maxchar), chr(maxchar)*5)
7384
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX, maxchar)
74-
self.assertEqual(new(0, 0x110000), '')
85+
self.assertEqual(new(0, INVALID_CHAR), '')
7586
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x4f60)
7687
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x4f60)
7788
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2, 0x1f600)
7889
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//2+1, 0x1f600)
7990
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4, 0x1f600)
8091
self.assertRaises(MemoryError, new, PY_SSIZE_T_MAX//4+1, 0x1f600)
81-
self.assertRaises(SystemError, new, 5, 0x110000)
92+
self.assertRaises(SystemError, new, 5, INVALID_CHAR)
8293
self.assertRaises(SystemError, new, -1, 0)
8394
self.assertRaises(SystemError, new, PY_SSIZE_T_MIN, 0)
8495

@@ -113,7 +124,7 @@ def test_fill(self):
113124
s = strings[0]
114125
self.assertRaises(IndexError, fill, s, -1, 0, 0x78)
115126
self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78)
116-
self.assertRaises(ValueError, fill, s, 0, 0, 0x110000)
127+
self.assertRaises(ValueError, fill, s, 0, 0, INVALID_CHAR)
117128
self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78)
118129
self.assertRaises(SystemError, fill, [], 0, 0, 0x78)
119130
# CRASHES fill(s, 0, NULL, 0, 0)
@@ -127,7 +138,7 @@ def _test_writechar(self, writechar, *, check):
127138
'\U0001f600\U0001f601\U0001f602'
128139
]
129140
# one character for every kind + out of range code
130-
chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000]
141+
chars = [0x78, 0xa9, 0x20ac, 0x1f638, INVALID_CHAR]
131142
for i, s in enumerate(strings):
132143
for j, c in enumerate(chars):
133144
if j <= i:
@@ -299,7 +310,26 @@ def test_fromkindanddata(self):
299310
self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1)
300311
self.assertRaises(ValueError, fromkindanddata, 1, NULL, PY_SSIZE_T_MIN)
301312
# CRASHES fromkindanddata(1, NULL, 1)
302-
# CRASHES fromkindanddata(4, b'\xff\xff\xff\xff')
313+
314+
# Test invalid UCS-4 string. Create an invalid string in release mode,
315+
# or raise SystemError in debug mode.
316+
built_with_assert = support.built_with_c_assertions()
317+
for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR):
318+
with self.subTest(invalid_char=invalid_char):
319+
# Test single character
320+
ucs4_char = invalid_char.to_bytes(4, byteorder=sys.byteorder)
321+
self.assertRaises(SystemError, fromkindanddata, 4, ucs4_char)
322+
323+
# Test multiple characters
324+
s = 'valid'.encode(enc4) + ucs4_char
325+
if support.Py_DEBUG:
326+
self.assertRaises(SystemError, fromkindanddata, 4, s)
327+
elif not built_with_assert :
328+
result = fromkindanddata(4, s)
329+
assert_invalid_string(self, result)
330+
else:
331+
# PyUnicode_FromKindAndData() fails with an assertion error
332+
pass
303333

304334
def test_substring(self):
305335
"""Test PyUnicode_Substring()"""
@@ -444,7 +474,7 @@ def check_format(expected, format, *args):
444474
check_format('\U0010ffff',
445475
b'%c', c_int(0x10ffff))
446476
with self.assertRaises(OverflowError):
447-
PyUnicode_FromFormat(b'%c', c_int(0x110000))
477+
PyUnicode_FromFormat(b'%c', c_int(INVALID_CHAR))
448478
# Issue #18183
449479
check_format('\U00010000\U00100000',
450480
b'%c%c', c_int(0x10000), c_int(0x100000))
@@ -1013,7 +1043,7 @@ def test_fromordinal(self):
10131043
self.assertEqual(fromordinal(0x20ac), '\u20ac')
10141044
self.assertEqual(fromordinal(0x1f600), '\U0001f600')
10151045

1016-
self.assertRaises(ValueError, fromordinal, 0x110000)
1046+
self.assertRaises(ValueError, fromordinal, INVALID_CHAR)
10171047
self.assertRaises(ValueError, fromordinal, -1)
10181048

10191049
def test_asutf8(self):
@@ -1365,8 +1395,8 @@ def test_findchar(self):
13651395
self.assertEqual(unicode_findchar(str, ord(ch), 0, len(str), -1), i)
13661396

13671397
str = "!>_<!"
1368-
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), 1), -1)
1369-
self.assertEqual(unicode_findchar(str, 0x110000, 0, len(str), -1), -1)
1398+
self.assertEqual(unicode_findchar(str, INVALID_CHAR, 0, len(str), 1), -1)
1399+
self.assertEqual(unicode_findchar(str, INVALID_CHAR, 0, len(str), -1), -1)
13701400
# start < end
13711401
self.assertEqual(unicode_findchar(str, ord('!'), 1, len(str)+1, 1), 4)
13721402
self.assertEqual(unicode_findchar(str, ord('!'), 1, PY_SSIZE_T_MAX, 1), 4)
@@ -1845,8 +1875,8 @@ def test_write_char(self):
18451875
writer.write_char(ord('$'))
18461876
writer.write_char(0x20ac)
18471877
writer.write_char(0x10_ffff)
1848-
self.assertRaises(ValueError, writer.write_char, 0x11_0000)
1849-
self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF)
1878+
self.assertRaises(ValueError, writer.write_char, INVALID_CHAR)
1879+
self.assertRaises(ValueError, writer.write_char, MAX_INVALID_CHAR)
18501880
self.assertEqual(writer.finish(),
18511881
"\0$\u20AC\U0010FFFF")
18521882

@@ -2011,15 +2041,41 @@ def test_ucs4(self):
20112041
writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass'))
20122042
writer.write_char(ord("-"))
20132043
writer.write_ucs4("null[\0]".encode(encoding), 7)
2014-
invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else
2015-
b'\x00\x11\x00\x00')
2016-
# CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid)
20172044
writer.write_ucs4(NULL, 0)
20182045
# CRASHES writer.write_ucs4(NULL, 1)
20192046
self.assertEqual(writer.finish(),
20202047
"lone\udc80-pair\ud83d\udc0d-null[\x00]")
20212048

2022-
# invalid size
2049+
# Invalid UCS-4 characters. Create an invalid string in release mode,
2050+
# or raise SystemError in debug mode.
2051+
built_with_assert = support.built_with_c_assertions()
2052+
writer = self.create_writer(0)
2053+
for invalid_char in (INVALID_CHAR, MAX_INVALID_CHAR):
2054+
with self.subTest(invalid_char=invalid_char):
2055+
# Test single character
2056+
ucs4_char = invalid_char.to_bytes(4, byteorder=sys.byteorder)
2057+
if support.Py_DEBUG:
2058+
self.assertRaises(SystemError, writer.write_ucs4, ucs4_char)
2059+
else:
2060+
writer.write_ucs4(ucs4_char)
2061+
2062+
# Test multiple characters
2063+
s = 'valid'.encode(encoding) + ucs4_char
2064+
if support.Py_DEBUG:
2065+
self.assertRaises(SystemError, writer.write_ucs4, s)
2066+
else:
2067+
writer.write_ucs4(s)
2068+
2069+
if support.Py_DEBUG:
2070+
self.assertEqual(writer.finish(), '')
2071+
elif not built_with_assert:
2072+
result = writer.finish()
2073+
assert_invalid_string(self, result)
2074+
else:
2075+
# PyUnicodeWriter_Finish() fails with an assertion error
2076+
pass
2077+
2078+
# Invalid size
20232079
writer = self.create_writer(0)
20242080
with self.assertRaises(ValueError):
20252081
writer.write_ucs4("text".encode(encoding), -1)

‎Objects/unicodeobject.c‎

Lines changed: 59 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,6 +1681,16 @@ PyUnicode_CopyCharacters(PyObject *to, Py_ssize_t to_start,
16811681
return how_many;
16821682
}
16831683

1684+
1685+
static void
1686+
unicode_invalid_character(PyObject *exc, Py_UCS4 ch)
1687+
{
1688+
PyErr_Format(exc,
1689+
"character U+%x is not in range [U+0000; U+%x]",
1690+
ch, MAX_UNICODE);
1691+
}
1692+
1693+
16841694
/* Find the maximum code point and count the number of surrogate pairs so a
16851695
correct string length can be computed before converting a string to UCS4.
16861696
This function counts single surrogates as a character and not as a pair.
@@ -1716,9 +1726,7 @@ find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end,
17161726
if (ch > *maxchar) {
17171727
*maxchar = ch;
17181728
if (*maxchar > MAX_UNICODE) {
1719-
PyErr_Format(PyExc_ValueError,
1720-
"character U+%x is not in range [U+0000; U+%x]",
1721-
ch, MAX_UNICODE);
1729+
unicode_invalid_character(PyExc_ValueError, ch);
17221730
return -1;
17231731
}
17241732
}
@@ -1972,17 +1980,15 @@ get_latin1_char(Py_UCS1 ch)
19721980
static PyObject*
19731981
unicode_char(Py_UCS4 ch)
19741982
{
1975-
PyObject *unicode;
1976-
1977-
assert(ch <= MAX_UNICODE);
1978-
19791983
if (ch < 256) {
19801984
return get_latin1_char(ch);
19811985
}
19821986

1983-
unicode = PyUnicode_New(1, ch);
1984-
if (unicode == NULL)
1987+
// Raise SystemError if the character is not in range [U+0000; MAX_UNICODE]
1988+
PyObject *unicode = PyUnicode_New(1, ch);
1989+
if (unicode == NULL) {
19851990
return NULL;
1991+
}
19861992

19871993
assert(PyUnicode_KIND(unicode) != PyUnicode_1BYTE_KIND);
19881994
if (PyUnicode_KIND(unicode) == PyUnicode_2BYTE_KIND) {
@@ -2361,15 +2367,36 @@ static PyObject*
23612367
_PyUnicode_FromUCS4(const Py_UCS4 *u, Py_ssize_t size)
23622368
{
23632369
PyObject *res;
2364-
Py_UCS4 max_char;
23652370

23662371
if (size == 0)
23672372
_Py_RETURN_UNICODE_EMPTY();
23682373
assert(size > 0);
2369-
if (size == 1)
2374+
2375+
if (size == 1) {
2376+
// Raise SystemError if the character is invalid
23702377
return unicode_char(u[0]);
2378+
}
2379+
2380+
#ifdef Py_DEBUG
2381+
// Check for invalid characters in debug mode
2382+
Py_UCS4 max_char = 127;
2383+
for (Py_ssize_t i = 0; i < size; i++) {
2384+
Py_UCS4 ch = u[i];
2385+
if (ch > max_char) {
2386+
if (ch > MAX_UNICODE) {
2387+
unicode_invalid_character(PyExc_SystemError, ch);
2388+
return NULL;
2389+
}
2390+
max_char = ch;
2391+
}
2392+
}
2393+
#else
2394+
// gh-158445: Return MAX_UNICODE even if the string contains invalid
2395+
// characters. Checking for invalid characters would make the function
2396+
// slower whereas it's unlikely in practice.
2397+
Py_UCS4 max_char = ucs4lib_find_max_char(u, u + size);
2398+
#endif
23712399

2372-
max_char = ucs4lib_find_max_char(u, u + size);
23732400
res = PyUnicode_New(size, max_char);
23742401
if (!res)
23752402
return NULL;
@@ -2403,9 +2430,27 @@ PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *pub_writer,
24032430
return 0;
24042431
}
24052432

2406-
Py_UCS4 max_char = ucs4lib_find_max_char(str, str + size);
2433+
#ifdef Py_DEBUG
2434+
// Check for invalid characters in debug mode
2435+
Py_UCS4 maxchar = 127;
2436+
for (Py_ssize_t i = 0; i < size; i++) {
2437+
Py_UCS4 ch = str[i];
2438+
if (ch > maxchar) {
2439+
if (ch > MAX_UNICODE) {
2440+
unicode_invalid_character(PyExc_SystemError, ch);
2441+
return -1;
2442+
}
2443+
maxchar = ch;
2444+
}
2445+
}
2446+
#else
2447+
// gh-158445: Return MAX_UNICODE even if the string contains invalid
2448+
// characters. Checking for invalid characters would make the function
2449+
// slower whereas it's unlikely in practice.
2450+
Py_UCS4 maxchar = ucs4lib_find_max_char(str, str + size);
2451+
#endif
24072452

2408-
if (_PyUnicodeWriter_Prepare(writer, size, max_char) < 0) {
2453+
if (_PyUnicodeWriter_Prepare(writer, size, maxchar) < 0) {
24092454
return -1;
24102455
}
24112456

0 commit comments

Comments
 (0)