2222 ctypes = None
2323
2424
25+ MAX_UNICODE = 0x10_ffff
26+ # The first invalid character after MAX_UNICODE
27+ INVALID_CHAR = MAX_UNICODE + 1
28+ # Maximum invalid character which fits into 32-bit Py_UCS4
29+ MAX_INVALID_CHAR = 0xFFFF_FFFF
2530NULL = None
2631
2732class Str (str ):
2833 pass
2934
3035
36+ def assert_invalid_string (testcase , text ):
37+ # Check that a Unicode string contains invalid characters:
38+ # not in range [U+0000; U+10ffff]
39+ testcase .assertRaises (SystemError , list , text )
40+
41+
3142class CAPITest (unittest .TestCase ):
3243
3344 @support .cpython_only
@@ -40,14 +51,14 @@ def test_new(self):
4051 self .assertEqual (new (0 , maxchar ), '' )
4152 self .assertEqual (new (5 , maxchar ), chr (maxchar )* 5 )
4253 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX , maxchar )
43- self .assertEqual (new (0 , 0x110000 ), '' )
54+ self .assertEqual (new (0 , INVALID_CHAR ), '' )
4455 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 2 , 0x4f60 )
4556 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 2 + 1 , 0x4f60 )
4657 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 2 , 0x1f600 )
4758 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 2 + 1 , 0x1f600 )
4859 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 4 , 0x1f600 )
4960 self .assertRaises (MemoryError , new , PY_SSIZE_T_MAX // 4 + 1 , 0x1f600 )
50- self .assertRaises (SystemError , new , 5 , 0x110000 )
61+ self .assertRaises (SystemError , new , 5 , INVALID_CHAR )
5162 self .assertRaises (SystemError , new , - 1 , 0 )
5263 self .assertRaises (SystemError , new , PY_SSIZE_T_MIN , 0 )
5364
@@ -84,7 +95,7 @@ def test_fill(self):
8495 s = strings [0 ]
8596 self .assertRaises (IndexError , fill , s , - 1 , 0 , 0x78 )
8697 self .assertRaises (IndexError , fill , s , PY_SSIZE_T_MIN , 0 , 0x78 )
87- self .assertRaises (ValueError , fill , s , 0 , 0 , 0x110000 )
98+ self .assertRaises (ValueError , fill , s , 0 , 0 , INVALID_CHAR )
8899 self .assertRaises (SystemError , fill , b'abc' , 0 , 0 , 0x78 )
89100 self .assertRaises (SystemError , fill , [], 0 , 0 , 0x78 )
90101 # CRASHES fill(s, 0, NULL, 0, 0)
@@ -103,7 +114,7 @@ def test_writechar(self):
103114 '\U0001f600 \U0001f601 \U0001f602 '
104115 ]
105116 # one character for every kind + out of range code
106- chars = [0x78 , 0xa9 , 0x20ac , 0x1f638 , 0x110000 ]
117+ chars = [0x78 , 0xa9 , 0x20ac , 0x1f638 , INVALID_CHAR ]
107118 for i , s in enumerate (strings ):
108119 for j , c in enumerate (chars ):
109120 if j <= i :
@@ -278,7 +289,26 @@ def test_fromkindanddata(self):
278289 self .assertRaises (ValueError , fromkindanddata , 1 , NULL , - 1 )
279290 self .assertRaises (ValueError , fromkindanddata , 1 , NULL , PY_SSIZE_T_MIN )
280291 # CRASHES fromkindanddata(1, NULL, 1)
281- # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff')
292+
293+ # Test invalid UCS-4 string. Create an invalid string in release mode,
294+ # or raise SystemError in debug mode.
295+ built_with_assert = support .built_with_c_assertions ()
296+ for invalid_char in (INVALID_CHAR , MAX_INVALID_CHAR ):
297+ with self .subTest (invalid_char = invalid_char ):
298+ # Test single character
299+ ucs4_char = invalid_char .to_bytes (4 , byteorder = sys .byteorder )
300+ self .assertRaises (SystemError , fromkindanddata , 4 , ucs4_char )
301+
302+ # Test multiple characters
303+ s = 'valid' .encode (enc4 ) + ucs4_char
304+ if support .Py_DEBUG :
305+ self .assertRaises (SystemError , fromkindanddata , 4 , s )
306+ elif not built_with_assert :
307+ result = fromkindanddata (4 , s )
308+ assert_invalid_string (self , result )
309+ else :
310+ # PyUnicode_FromKindAndData() fails with an assertion error
311+ pass
282312
283313 @support .cpython_only
284314 @unittest .skipIf (_testlimitedcapi is None , 'need _testlimitedcapi module' )
@@ -401,7 +431,7 @@ def check_format(expected, format, *args):
401431 check_format ('\U0010ffff ' ,
402432 b'%c' , c_int (0x10ffff ))
403433 with self .assertRaises (OverflowError ):
404- PyUnicode_FromFormat (b'%c' , c_int (0x110000 ))
434+ PyUnicode_FromFormat (b'%c' , c_int (INVALID_CHAR ))
405435 # Issue #18183
406436 check_format ('\U00010000 \U00100000 ' ,
407437 b'%c%c' , c_int (0x10000 ), c_int (0x100000 ))
@@ -988,7 +1018,7 @@ def test_fromordinal(self):
9881018 self .assertEqual (fromordinal (0x20ac ), '\u20ac ' )
9891019 self .assertEqual (fromordinal (0x1f600 ), '\U0001f600 ' )
9901020
991- self .assertRaises (ValueError , fromordinal , 0x110000 )
1021+ self .assertRaises (ValueError , fromordinal , INVALID_CHAR )
9921022 self .assertRaises (ValueError , fromordinal , - 1 )
9931023
9941024 @support .cpython_only
@@ -1374,8 +1404,8 @@ def test_findchar(self):
13741404 self .assertEqual (unicode_findchar (str , ord (ch ), 0 , len (str ), - 1 ), i )
13751405
13761406 str = "!>_<!"
1377- self .assertEqual (unicode_findchar (str , 0x110000 , 0 , len (str ), 1 ), - 1 )
1378- self .assertEqual (unicode_findchar (str , 0x110000 , 0 , len (str ), - 1 ), - 1 )
1407+ self .assertEqual (unicode_findchar (str , INVALID_CHAR , 0 , len (str ), 1 ), - 1 )
1408+ self .assertEqual (unicode_findchar (str , INVALID_CHAR , 0 , len (str ), - 1 ), - 1 )
13791409 # start < end
13801410 self .assertEqual (unicode_findchar (str , ord ('!' ), 1 , len (str )+ 1 , 1 ), 4 )
13811411 self .assertEqual (unicode_findchar (str , ord ('!' ), 1 , PY_SSIZE_T_MAX , 1 ), 4 )
@@ -1782,8 +1812,8 @@ def test_write_char(self):
17821812 writer .write_char (ord ('$' ))
17831813 writer .write_char (0x20ac )
17841814 writer .write_char (0x10_ffff )
1785- self .assertRaises (ValueError , writer .write_char , 0x11_0000 )
1786- self .assertRaises (ValueError , writer .write_char , 0xFFFF_FFFF )
1815+ self .assertRaises (ValueError , writer .write_char , INVALID_CHAR )
1816+ self .assertRaises (ValueError , writer .write_char , MAX_INVALID_CHAR )
17871817 self .assertEqual (writer .finish (),
17881818 "\0 $\u20AC \U0010FFFF " )
17891819
@@ -1947,15 +1977,41 @@ def test_ucs4(self):
19471977 writer .write_ucs4 ("pair\uD83D \uDC0D " .encode (encoding , 'surrogatepass' ))
19481978 writer .write_char (ord ("-" ))
19491979 writer .write_ucs4 ("null[\0 ]" .encode (encoding ), 7 )
1950- invalid = (b'\x00 \x00 \x11 \x00 ' if sys .byteorder == 'little' else
1951- b'\x00 \x11 \x00 \x00 ' )
1952- # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid)
19531980 writer .write_ucs4 (NULL , 0 )
19541981 # CRASHES writer.write_ucs4(NULL, 1)
19551982 self .assertEqual (writer .finish (),
19561983 "lone\udc80 -pair\ud83d \udc0d -null[\x00 ]" )
19571984
1958- # invalid size
1985+ # Invalid UCS-4 characters. Create an invalid string in release mode,
1986+ # or raise SystemError in debug mode.
1987+ built_with_assert = support .built_with_c_assertions ()
1988+ writer = self .create_writer (0 )
1989+ for invalid_char in (INVALID_CHAR , MAX_INVALID_CHAR ):
1990+ with self .subTest (invalid_char = invalid_char ):
1991+ # Test single character
1992+ ucs4_char = invalid_char .to_bytes (4 , byteorder = sys .byteorder )
1993+ if support .Py_DEBUG :
1994+ self .assertRaises (SystemError , writer .write_ucs4 , ucs4_char )
1995+ else :
1996+ writer .write_ucs4 (ucs4_char )
1997+
1998+ # Test multiple characters
1999+ s = 'valid' .encode (encoding ) + ucs4_char
2000+ if support .Py_DEBUG :
2001+ self .assertRaises (SystemError , writer .write_ucs4 , s )
2002+ else :
2003+ writer .write_ucs4 (s )
2004+
2005+ if support .Py_DEBUG :
2006+ self .assertEqual (writer .finish (), '' )
2007+ elif not built_with_assert :
2008+ result = writer .finish ()
2009+ assert_invalid_string (self , result )
2010+ else :
2011+ # PyUnicodeWriter_Finish() fails with an assertion error
2012+ pass
2013+
2014+ # Invalid size
19592015 writer = self .create_writer (0 )
19602016 with self .assertRaises (ValueError ):
19612017 writer .write_ucs4 ("text" .encode (encoding ), - 1 )
0 commit comments