[pull] main from python:main

Doc/whatsnew/3.16.rst

@@ -1084,7 +1084,8 @@ C API changes
 New features
 ------------
 
-* TODO
+* Add :c:func:`Py_HashBuffer` to the limited C API.
+  (Contributed by Victor Stinner in :gh:`148233`.)
 
 Porting to Python 3.16
 ----------------------

Include/cpython/pyhash.h

@@ -50,5 +50,3 @@ _Py_HashPointer(const void *ptr)
 }
 
 PyAPI_FUNC(Py_hash_t) PyObject_GenericHash(PyObject *);
-
-PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len);

Include/pyhash.h

@@ -47,6 +47,10 @@ extern "C" {
 #  endif /* uint64_t && uint32_t && aligned */
 #endif /* Py_HASH_ALGORITHM */
 
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 16)
+PyAPI_FUNC(Py_hash_t) Py_HashBuffer(const void *ptr, Py_ssize_t len);
+#endif
+
 #ifndef Py_LIMITED_API
 #  define Py_CPYTHON_HASH_H
 #  include "cpython/pyhash.h"

Lib/idlelib/colorizer.py

@@ -42,6 +42,11 @@ def make_pat():
         ]) +
         r"))"
     )
+    type_softkw = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<TYPE_SOFTKW>type)" +
+        r"(?=[ \t]+(?!(?:" + "|".join(keyword.kwlist) + r")\b)[^\W\d])"
+    )
     lazy_softkw = (  # lazy new in 3.15 (+ 2 lines below).
         r"^[ \t]*" +  # at beginning of line + possible indentation
         r"(?P<LAZY_SOFTKW>lazy)" +
@@ -59,7 +64,7 @@ def make_pat():
     dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
     string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
     prog = re.compile("|".join([
-                                builtin, comment, string, kw,
+                                type_softkw, builtin, comment, string, kw,
                                 match_softkw, case_default,
                                 case_softkw_and_pattern, lazy_softkw,
                                 any("SYNC", [r"\n"]),
@@ -75,6 +80,7 @@ def make_pat():
     "CASE_SOFTKW": "KEYWORD",
     "CASE_DEFAULT_UNDERSCORE": "KEYWORD",
     "CASE_SOFTKW2": "KEYWORD",
+    "TYPE_SOFTKW": "KEYWORD",
     "LAZY_SOFTKW": "KEYWORD",
 }
 

Lib/idlelib/idle_test/test_colorizer.py

@@ -52,6 +52,8 @@ async def f(): await g()
     '''
     case _:'''
     "match x:"
+    type Point = tuple[float, float]
+    type = type(1)
     """)
 
 
@@ -404,6 +406,8 @@ def test_recolorize_main(self, mock_notify):
                     ('28.25', ('STRING',)), ('28.38', ('STRING',)),
                     ('30.0', ('STRING',)),
                     ('31.1', ('STRING',)),
+                    ('32.0', ('KEYWORD',)),
+                    ('33.0', ('BUILTIN',)), ('33.1', ('BUILTIN',)),
                     # SYNC at the end of every line.
                     ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)),
                    )
@@ -434,7 +438,7 @@ def test_recolorize_main(self, mock_notify):
         eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17'))
         eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26'))
         eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0'))
-        eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0'))
+        eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '32.0'))
 
     def _assert_highlighting(self, source, tag_ranges):
         """Check highlighting of a given piece of code.

Lib/test/test_capi/test_hash.py

@@ -1,7 +1,9 @@
 import sys
 import unittest
 from test.support import import_helper
+
 _testcapi = import_helper.import_module('_testcapi')
+_testlimitedcapi = import_helper.import_module('_testlimitedcapi')
 
 
 SIZEOF_VOID_P = _testcapi.SIZEOF_VOID_P
@@ -79,7 +81,8 @@ def python_hash_pointer(x):
         self.assertEqual(hash_pointer(VOID_P_MAX), -2)
 
     def test_hash_buffer(self):
-        hash_buffer = _testcapi.hash_buffer
+        # Test Py_HashBuffer()
+        hash_buffer = _testlimitedcapi.hash_buffer
 
         def check(data):
             self.assertEqual(hash_buffer(data), hash(data))

Lib/test/test_capi/test_unicode.py

@@ -1880,6 +1880,36 @@ def test_basic(self):
         self.assertEqual(writer.finish(),
                          "var=long value 'repr'")
 
+    def test_create(self):
+        # Test PyUnicodeWriter_Create() with non-zero size
+        s = 'Monty Python'
+
+        # Preallocate the exact length. Use 2 writes to force the creation
+        # of a buffer:
+        #   1. Use the read-only optimization.
+        #   2. Allocate a buffer of length character.
+        # No resize needed in finish().
+        writer = self.create_writer(len(s))
+        writer.write_str(s[:5])
+        self.assertEqual(writer.get_buffer(), (5, 127, True))
+        writer.write_str(s[5:])
+        self.assertEqual(writer.get_buffer(), (len(s), 127, False))
+        self.assertEqual(writer.finish(), s)
+
+        # Preallocate len(s)-1 characters. Use 3 writes:
+        #   1. Use read-only optimization.
+        #   2. Allocate a buffer of len-1 characters.
+        #   3. Resize the buffer with overallocation.
+        # finish() has to truncate the buffer.
+        writer = self.create_writer(len(s) - 1)
+        writer.write_str(s[:2])
+        self.assertEqual(writer.get_buffer(), (2, 127, True))
+        writer.write_str(s[2:5])
+        self.assertEqual(writer.get_buffer(), (len(s) - 1, 127, False))
+        writer.write_str(s[5:])
+        self.assertGreater(writer.get_buffer()[0], len(s))
+        self.assertEqual(writer.finish(), s)
+
     def test_repr_null(self):
         writer = self.create_writer(0)
         writer.write_utf8(b'var=', -1)
@@ -2087,32 +2117,39 @@ def test_substring_empty(self):
     def test_singletons(self):
         for size in (0, 123):
             with self.subTest(size=size):
+                # PyUnicodeWriter_Finish() returns the empty string singleton
+                # if no character has been written.
                 writer = self.create_writer(size)
                 writer.write_utf8(b'utf8', 0)
                 writer.write_ascii(b'ascii', 0)
                 writer.write_widechar(b'wstr', 0)
                 writer.write_ucs4(b'ucs4', 0)
-                writer.write_substring('text', 0, 0)
+                writer.write_substring('text', 2, 2)
+                self.assertEqual(writer.get_buffer(), (None, 127, False))
                 self.assertIs(writer.finish(), '')
 
         for size in (0, 123):
             for ch in range(256):
                 with self.subTest(size=size, ch=ch):
                     ch = chr(ch)
+                    maxchar = (255 if ord(ch) >= 128 else 127)
 
-                    # If the first write is a Latin1 character and no buffer
-                    # was allocated yet, use the singleton as the read-only
-                    # buffer
+                    # PyUnicodeWriter_WriteChar(ch) uses the read-only
+                    # optimization with the character singleton if ch is a
+                    # Latin1 character and no buffer was allocated yet.
                     writer = self.create_writer(size)
                     writer.write_char(ord(ch))
+                    self.assertEqual(writer.get_buffer(),
+                                     (1, maxchar, True))
                     self.assertIs(writer.finish(), ch)
 
-                    # PyUnicodeWriter_Finish() replaces the buffer
-                    # with the singleton
+                    # PyUnicodeWriter_Finish() replaces the buffer with the
+                    # singleton. Use PyUnicodeWriter_WriteSubstring() to avoid
+                    # the read-only buffer optimization.
                     writer = self.create_writer(size)
-                    # Use PyUnicodeWriter_WriteSubstring() to avoid
-                    # the read-only buffer optimization
-                    writer.write_substring(ch + 'xxx', 0, 1)
+                    writer.write_substring('xxx' + ch + 'y', 3, 4)
+                    self.assertEqual(writer.get_buffer(),
+                                     (size or 1, maxchar, False))
                     self.assertIs(writer.finish(), ch)
 
     @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)')
@@ -2135,60 +2172,83 @@ def test_detect_overflow(self):
     def test_memory_error(self):
         # Inject MemoryError in PyUnicodeWriter_WriteStr()
         writer = self.create_writer(0)
-        writer.write_str("start")
+        writer.write_utf8(b"start", -1)
+        self.assertEqual(writer.get_buffer(), (5, 127, False))
         with self.assertRaises(MemoryError):
             with support.inject_memory_error_cm():
                 # Resize the internal str object
                 writer.write_str("s" * 1024)
         writer.write_str(" end")
         self.assertEqual(writer.finish(), "start end")
 
-        # Inject MemoryError in PyUnicodeWriter_Finish()
+        # Inject MemoryError in PyUnicodeWriter_Finish(). Use write_utf8() to
+        # allocate a buffer of 1024 character. finish() needs to truncate the
+        # buffer to 3 characters.
         writer = self.create_writer(1024)
-        writer.write_str("abc")
+        writer.write_utf8(b"abc", -1)
+        self.assertEqual(writer.get_buffer(), (1024, 127, False))
         with self.assertRaises(MemoryError):
             with support.inject_memory_error_cm():
-                # Need to truncate the internal str object
                 writer.finish()
 
     def test_change_kind(self):
         writer = self.create_writer(0)
+
         # Create an ASCII buffer
         writer.write_str('ascii ')
+        self.assertEqual(writer.get_buffer()[1], 127)
+
         # Change the buffer to UCS1
         writer.write_str('latin1:\xe9 ')
+        self.assertEqual(writer.get_buffer()[1], 255)
+
         # Change the buffer to UCS2
         writer.write_str('ucs2:\u20ac ')
+        self.assertEqual(writer.get_buffer()[1], 0xffff)
+
         # Change the buffer to UCS4
         writer.write_str('ucs4:\U0010ffff')
+        self.assertEqual(writer.get_buffer()[1], 0x10_ffff)
+
         self.assertEqual(writer.finish(),
                          'ascii latin1:\xe9 ucs2:\u20ac ucs4:\U0010ffff')
 
     def test_readonly_optim(self):
         # Read-only optimization: if the first and only write is a Python str
         # object and no buffer was allocated yet, return the object unchanged
         unique_string = 'unique string'
-        writer = self.create_writer(0)
-        writer.write_str(unique_string)
-        self.assertIs(writer.finish(), unique_string)
+        expected = (len(unique_string), 127, True)
+        for size in (0, 123):
+            with self.subTest(size=size):
+                # PyUnicodeWriter_WriteStr() optimization
+                writer = self.create_writer(size)
+                writer.write_str(unique_string)
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertIs(writer.finish(), unique_string)
 
-        writer = self.create_writer(0)
-        writer.write_substring(unique_string, 0, len(unique_string))
-        self.assertIs(writer.finish(), unique_string)
+                # PyUnicodeWriter_WriteSubstring() optimization
+                writer = self.create_writer(size)
+                writer.write_substring(unique_string, 0, len(unique_string))
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertIs(writer.finish(), unique_string)
 
-        class MyStr:
-            def __str__(self):
-                return unique_string
-        writer = self.create_writer(0)
-        writer.write_str(MyStr())
-        self.assertIs(writer.finish(), unique_string)
+                # PyUnicodeWriter_WriteStr() optimization
+                class MyStr:
+                    def __str__(self):
+                        return unique_string
+                writer = self.create_writer(size)
+                writer.write_str(MyStr())
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertIs(writer.finish(), unique_string)
 
-        class MyRepr:
-            def __repr__(self):
-                return unique_string
-        writer = self.create_writer(0)
-        writer.write_repr(MyRepr())
-        self.assertIs(writer.finish(), unique_string)
+                # PyUnicodeWriter_WriteRepr() optimization
+                class MyRepr:
+                    def __repr__(self):
+                        return unique_string
+                writer = self.create_writer(size)
+                writer.write_repr(MyRepr())
+                self.assertEqual(writer.get_buffer(), expected)
+                self.assertIs(writer.finish(), unique_string)
 
 
 # Test PyUnicodeWriter_Format()

Lib/test/test_dtrace.py

@@ -84,21 +84,33 @@ def normalize_trace_output(output):
 
 
 USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
+TERMINATE_TIMEOUT = 10
 
 def create_process_group(*args, **kwargs):
     if USE_PROCESS_GROUP:
         kwargs['start_new_session'] = True
     return subprocess.Popen(*args, **kwargs)
 
-def kill_process_group(proc):
+def terminate_process_group(proc):
     if USE_PROCESS_GROUP:
         try:
-            os.killpg(proc.pid, signal.SIGKILL)
+            os.killpg(proc.pid, signal.SIGTERM)
         except ProcessLookupError:
             pass
     else:
-        proc.kill()
-    proc.communicate()  # Clean up
+        proc.terminate()
+
+    try:
+        proc.communicate(timeout=TERMINATE_TIMEOUT)
+    except subprocess.TimeoutExpired:
+        if USE_PROCESS_GROUP:
+            try:
+                os.killpg(proc.pid, signal.SIGKILL)
+            except ProcessLookupError:
+                pass
+        else:
+            proc.kill()
+        proc.communicate(timeout=TERMINATE_TIMEOUT)  # Clean up
 
 
 def run_readelf(cmd):
@@ -132,6 +144,7 @@ class TraceBackend:
     EXTENSION = None
     COMMAND = None
     COMMAND_ARGS = []
+    USABILITY_TIMEOUT = 10
 
     def run_case(self, name, optimize_python=None):
         try:
@@ -163,7 +176,7 @@ def trace(self, script_file, subcommand=None, *, timeout=None,
         try:
             stdout, _ = proc.communicate(timeout=timeout)
         except subprocess.TimeoutExpired:
-            kill_process_group(proc)
+            terminate_process_group(proc)
             raise
         if check_returncode and proc.returncode:
             raise AssertionError(
@@ -183,7 +196,7 @@ def trace_python(self, script_file, python_file, optimize_python=None):
     def assert_usable(self):
         try:
             output = self.trace(abspath("assert_usable" + self.EXTENSION),
-                                timeout=10)
+                                timeout=self.USABILITY_TIMEOUT)
             output = output.strip()
         except subprocess.TimeoutExpired:
             raise unittest.SkipTest(
@@ -208,6 +221,7 @@ class SystemTapBackend(TraceBackend):
     EXTENSION = ".stp"
     COMMAND = ["stap", "-g"]
     PROBE_PLACEHOLDER = "@PYTHON_SYSTEMTAP_PROBE@"
+    USABILITY_TIMEOUT = 60
 
     @staticmethod
     def quote_systemtap_string(value):
@@ -361,7 +375,7 @@ def run_case(self, name, optimize_python=None):
             )
             stdout, stderr = proc.communicate(timeout=60)
         except subprocess.TimeoutExpired:
-            kill_process_group(proc)
+            terminate_process_group(proc)
             raise AssertionError("bpftrace timed out")
         except (FileNotFoundError, PermissionError) as e:
             raise unittest.SkipTest(f"bpftrace not available: {e}")
@@ -400,7 +414,7 @@ def assert_usable(self):
             )
             stdout, stderr = proc.communicate(timeout=10)
         except subprocess.TimeoutExpired:
-            kill_process_group(proc)
+            terminate_process_group(proc)
             raise unittest.SkipTest("bpftrace timed out during usability check")
         except OSError as e:
             raise unittest.SkipTest(f"bpftrace not available: {e}")