Merge branch 'main' into fix-zoneinfo-159715

Author: pganssle

Doc/library/ctypes.rst

@@ -2052,7 +2052,11 @@ See :ref:`ctypes-callback-functions` for examples.
 .. function:: PYFUNCTYPE(restype, *argtypes)
 
    The returned function prototype creates functions that use the Python calling
-   convention.  The function will *not* release the GIL during the call.
+   convention.  Unlike :func:`CFUNCTYPE` and :func:`WINFUNCTYPE`, ctypes does
+   *not* release the GIL before calling the foreign function, so the function
+   may safely use the Python C API. Note that this does not prevent the GIL
+   from being released *during* the call.
+
 
 Function prototypes created by these factory functions can be instantiated in
 different ways, depending on the type and number of the parameters in the call:

Doc/library/gc.rst

@@ -75,6 +75,12 @@ The :mod:`!gc` module provides the following functions:
    returned. If *generation* is not ``None``, return only the objects tracked by
    the collector that are in that generation.
 
+   .. warning::
+      Care must be taken when using objects returned by :func:`get_objects` because
+      some of them could still be under construction and hence in a temporarily
+      invalid state. Avoid using :func:`get_objects` for any purpose other than
+      debugging.
+
    .. versionchanged:: 3.8
       New *generation* parameter.
 

Include/internal/pycore_global_objects_fini_generated.h

@@ -685,6 +685,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(nlocals)
         STRUCT_FOR_ID(node_depth)
         STRUCT_FOR_ID(node_offset)
+        STRUCT_FOR_ID(nothing)
         STRUCT_FOR_ID(ns)
         STRUCT_FOR_ID(nstype)
         STRUCT_FOR_ID(nt)

Include/internal/pycore_global_strings.h

@@ -34,6 +34,16 @@ class member(object):
     def __init__(self, value):
         self.value = value
 
+def _hashable(obj):
+    """
+    Returns True if obj is hashable, False otherwise.
+    """
+    try:
+        hash(obj)
+        return True
+    except TypeError:
+        return False
+
 def _is_descriptor(obj):
     """
     Returns True if obj is a descriptor, False otherwise.
@@ -313,14 +323,13 @@ def __set_name__(self, enum_class, member_name):
 
         enum_class._add_member_(member_name, enum_member)
         try:
-            # This may fail if value is not hashable. We can't add the value
-            # to the map, and by-value lookups for this value will be
-            # linear.
+            # This may fail if value is not hashable.
             enum_class._value2member_map_.setdefault(value, enum_member)
-            if value not in enum_class._hashable_values_:
-                enum_class._hashable_values_.append(value)
+            enum_class._hashable_values_.add(value)
         except TypeError:
-            # keep track of the value in a list so containment checks are quick
+            # We can't add the value to the map; by-value lookups for this
+            # value will be linear.  keep track of the value in a list so
+            # containment checks are quicker
             enum_class._unhashable_values_.append(value)
             enum_class._unhashable_values_map_.setdefault(member_name, []).append(value)
 
@@ -530,7 +539,7 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k
         classdict['_member_names_'] = []
         classdict['_member_map_'] = {}
         classdict['_value2member_map_'] = {}
-        classdict['_hashable_values_'] = []          # for comparing with non-hashable types
+        classdict['_hashable_values_'] = set()      # for comparing with non-hashable types
         classdict['_unhashable_values_'] = []       # e.g. frozenset() with set()
         classdict['_unhashable_values_map_'] = {}
         classdict['_member_type_'] = member_type
@@ -767,10 +776,10 @@ def __contains__(cls, value):
                 return isinstance(result, cls)
             except ValueError:
                 pass
-        return (
-                value in cls._unhashable_values_    # both structures are lists
-                or value in cls._hashable_values_
-                )
+        try:
+            return value in cls._hashable_values_                   # a set
+        except TypeError:
+            return value in cls._unhashable_values_                 # a list
 
     def __delattr__(cls, attr):
         # nicer error message when someone tries to delete an attribute
@@ -1261,7 +1270,7 @@ def _add_value_alias_(self, value):
             # to the map, and by-value lookups for this value will be
             # linear.
             cls._value2member_map_.setdefault(value, self)
-            cls._hashable_values_.append(value)
+            cls._hashable_values_.add(value)
         except TypeError:
             # keep track of the value in a list so containment checks are quick
             cls._unhashable_values_.append(value)
@@ -1802,7 +1811,7 @@ def convert_class(cls):
         body['_member_names_'] = member_names = []
         body['_member_map_'] = member_map = {}
         body['_value2member_map_'] = value2member_map = {}
-        body['_hashable_values_'] = hashable_values = []
+        body['_hashable_values_'] = hashable_values = set()
         body['_unhashable_values_'] = unhashable_values = []
         body['_unhashable_values_map_'] = {}
         body['_member_type_'] = member_type = etype._member_type_
@@ -1866,7 +1875,7 @@ def convert_class(cls):
                     contained = value2member_map.get(member._value_)
                 except TypeError:
                     contained = None
-                    if member._value_ in unhashable_values or member.value in hashable_values:
+                    if member._value_ in hashable_values or member.value in unhashable_values:
                         for m in enum_class:
                             if m._value_ == member._value_:
                                 contained = m
@@ -1886,7 +1895,7 @@ def convert_class(cls):
                     else:
                         enum_class._add_member_(name, member)
                     value2member_map[value] = member
-                    hashable_values.append(value)
+                    hashable_values.add(value)
                     if _is_single_bit(value):
                         # not a multi-bit alias, record in _member_names_ and _flag_mask_
                         member_names.append(name)
@@ -1923,7 +1932,9 @@ def convert_class(cls):
                     contained = value2member_map.get(member._value_)
                 except TypeError:
                     contained = None
-                    if member._value_ in unhashable_values or member._value_ in hashable_values:
+                    if (_hashable(member._value_) and member._value_ in hashable_values
+                        or member._value_ in unhashable_values
+                        ):
                         for m in enum_class:
                             if m._value_ == member._value_:
                                 contained = m
@@ -1949,8 +1960,7 @@ def convert_class(cls):
                         # to the map, and by-value lookups for this value will be
                         # linear.
                         enum_class._value2member_map_.setdefault(value, member)
-                        if value not in hashable_values:
-                            hashable_values.append(value)
+                        hashable_values.add(value)
                     except TypeError:
                         # keep track of the value in a list so containment checks are quick
                         enum_class._unhashable_values_.append(value)

Include/internal/pycore_runtime_init_generated.h

@@ -267,6 +267,9 @@ def digest(key, msg, digest):
 
 
 def _compute_digest_fallback(key, msg, digest):
+    if not isinstance(key, (bytes, bytearray)):
+        raise TypeError(f"key: expected bytes or bytearray, "
+                        f"but got {type(key).__name__!r}")
     digest_cons = _get_digest_constructor(digest)
     if _is_shake_constructor(digest_cons):
         raise ValueError(f"unsupported hash algorithm {digest}")

Include/internal/pycore_unicodeobject_generated.h

@@ -665,13 +665,15 @@ class _FileInFile(object):
        object.
     """
 
-    def __init__(self, fileobj, offset, size, name, blockinfo=None):
+    def __init__(self, fileobj, offset, size, name, blockinfo=None,
+            exception=OSError):
         self.fileobj = fileobj
         self.offset = offset
         self.size = size
         self.position = 0
         self.name = name
         self.closed = False
+        self.exception = exception
 
         if blockinfo is None:
             blockinfo = [(0, size)]
@@ -748,7 +750,10 @@ def read(self, size=None):
             length = min(size, stop - self.position)
             if data:
                 self.fileobj.seek(offset + (self.position - start))
-                b = self.fileobj.read(length)
+                try:
+                    b = self.fileobj.read(length)
+                except self.exception as e:
+                    raise ReadError(f"invalid compressed data: {e}") from e
                 if len(b) != length:
                     raise ReadError("unexpected end of data")
                 buf += b
@@ -771,7 +776,8 @@ class ExFileObject(io.BufferedReader):
 
     def __init__(self, tarfile, tarinfo):
         fileobj = _FileInFile(tarfile.fileobj, tarinfo.offset_data,
-                tarinfo.size, tarinfo.name, tarinfo.sparse)
+                tarinfo.size, tarinfo.name, tarinfo.sparse,
+                exception=tarfile.exception)
         super().__init__(fileobj)
 #class ExFileObject
 
@@ -1133,7 +1139,7 @@ def create_pax_header(self, info, encoding):
         # be represented in ASCII encoding.
         for name, hname, length in (
                 ("name", "path", LENGTH_NAME), ("linkname", "linkpath", LENGTH_LINK),
-                ("uname", "uname", 32), ("gname", "gname", 32)):
+                ("uname", "uname", 31), ("gname", "gname", 31)):
 
             if hname in pax_headers:
                 # The pax header has priority.
@@ -1177,6 +1183,9 @@ def create_pax_header(self, info, encoding):
         else:
             buf = b""
 
+        # The owner names in the base header must be NUL-terminated.
+        info["uname"] = info["uname"][:31]
+        info["gname"] = info["gname"][:31]
         return buf + self._create_header(info, USTAR_FORMAT, "ascii", "replace")
 
     @classmethod
@@ -1801,6 +1810,12 @@ class TarFile(object):
 
     fileobject = ExFileObject   # The file-object for extractfile().
 
+    exception = OSError         # The exception raised by the underlying
+                                 # fileobj.read() on corrupt compressed
+                                 # data past the header (see gzopen() etc.
+                                 # for the more specific exception types
+                                 # used by each compression format).
+
     extraction_filter = None    # The default filter for extraction.
 
     def __init__(self, name=None, mode="r", fileobj=None, format=None,
@@ -2046,6 +2061,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=6, **kwargs):
 
         try:
             from gzip import GzipFile
+            import zlib
         except ImportError:
             raise CompressionError("gzip module is not available") from None
 
@@ -2068,6 +2084,7 @@ def gzopen(cls, name, mode="r", fileobj=None, compresslevel=6, **kwargs):
             fileobj.close()
             raise
         t._extfileobj = False
+        t.exception = zlib.error
         return t
 
     @classmethod
@@ -2124,6 +2141,7 @@ def xzopen(cls, name, mode="r", fileobj=None, preset=None, **kwargs):
             fileobj.close()
             raise
         t._extfileobj = False
+        t.exception = LZMAError
         return t
 
     @classmethod
@@ -2159,6 +2177,7 @@ def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None,
             fileobj.close()
             raise
         t._extfileobj = False
+        t.exception = ZstdError
         return t
 
     # All *open() methods are registered here.

Lib/enum.py

@@ -1,6 +1,7 @@
 """Tests monitoring, sys.settrace, and sys.setprofile in a multi-threaded
 environment to verify things are thread-safe in a free-threaded build"""
 
+import dis
 import sys
 import threading
 import time
@@ -251,6 +252,62 @@ def append(self, trace):
 
 @threading_helper.requires_working_threading()
 class MonitoringMisc(MonitoringTestMixin, TestCase):
+    def test_disable_line_keeps_instruction_events(self):
+        def func(x):
+            a = x + 1
+            b = a * 2
+            return b
+
+        code = func.__code__
+        expected = [instr.offset for instr in dis.get_instructions(func)
+                    if instr.opname != "RESUME"]
+        # Two threads, each with two lists of instruction offsets (one per call).
+        # Check that disabling LINE preserves INSTRUCTION events in every
+        # thread-local bytecode copy, including on subsequent calls.
+        records = [[[], []] for _ in range(2)]
+        local = threading.local()
+        ready = Barrier(len(records) + 1)
+        start = threading.Event()
+
+        def worker(calls):
+            # Create each thread's bytecode copy before enabling monitoring.
+            func(1)
+            ready.wait()
+            start.wait()
+            for instructions in calls:
+                local.instructions = instructions
+                func(1)
+
+        def instruction(code, offset):
+            local.instructions.append(offset)
+
+        def line(code, lineno):
+            return monitoring.DISABLE
+
+        threads = [Thread(target=worker, args=(calls,)) for calls in records]
+        try:
+            with threading_helper.start_threads(threads, unlock=start.set):
+                ready.wait()
+                monitoring.register_callback(
+                    self.tool_id, monitoring.events.INSTRUCTION, instruction)
+                monitoring.register_callback(
+                    self.tool_id, monitoring.events.LINE, line)
+                monitoring.set_local_events(
+                    self.tool_id, code,
+                    monitoring.events.INSTRUCTION | monitoring.events.LINE)
+                start.set()
+        finally:
+            monitoring.set_local_events(self.tool_id, code, 0)
+            monitoring.register_callback(
+                self.tool_id, monitoring.events.INSTRUCTION, None)
+            monitoring.register_callback(
+                self.tool_id, monitoring.events.LINE, None)
+            monitoring.restart_events()
+
+        for calls in records:
+            for instructions in calls:
+                self.assertEqual(instructions, expected)
+
     def register_callback(self, barrier):
         barrier.wait()