[pull] main from python:main

Doc/c-api/bytes.rst

@@ -249,7 +249,7 @@ called with a non-bytes parameter.
 
    While bytes objects are usually immutable in Python, this special C API
    allows mutating a bytes object in-place. The returned bytes object can still
-   be mutated using :c:func:`PyBytesWriter_GetData`; except if *newsize* is
+   be mutated using :c:func:`PyBytes_AsString`; except if *newsize* is
    zero in which case it returns the immutable empty bytes string.
 
    .. soft-deprecated:: 3.15

Include/internal/pycore_pylifecycle.h

@@ -26,6 +26,7 @@ extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc);
 extern void _Py_InitVersion(void);
 extern PyStatus _PyFaulthandler_Init(int enable);
 extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp);
+extern int _PyBuiltin_InitPythonFunctions(PyObject *dict);
 extern PyStatus _PySys_Create(
     PyThreadState *tstate,
     PyObject **sysmod_p);

Lib/_pybuiltins.py

@@ -0,0 +1,42 @@
+"""Builtins implemented in Python.
+
+This module is frozen into the interpreter and imported during startup,
+before the import system exists.  The names listed in ``__all__`` are
+copied into the ``builtins`` module.
+"""
+
+__all__ = ['anext']
+
+_NOT_GIVEN = sentinel("_NOT_GIVEN")
+
+
+def anext(async_iterator, default=_NOT_GIVEN, /):
+    """Return the next item from the async iterator.
+
+    If default is given and the async iterator is exhausted,
+    it is returned instead of raising StopAsyncIteration.
+    """
+    cls = type(async_iterator)
+    try:
+        # Looked up on the type, like the C slot am_anext.
+        anext_method = cls.__anext__
+    except AttributeError:
+        raise TypeError(
+            f"{cls.__name__!r} object is not an async iterator"
+        ) from None
+    awaitable = anext_method(async_iterator)
+    if default is _NOT_GIVEN:
+        return awaitable
+    return _anext_with_default(awaitable, default)
+
+
+async def _anext_with_default(awaitable, default):
+    try:
+        return await awaitable
+    except StopAsyncIteration:
+        return default
+
+
+for _name in __all__:
+    globals()[_name].__module__ = 'builtins'
+del _name

Lib/idlelib/autocomplete.py

@@ -153,6 +153,10 @@ def open_completions(self, args):
         comp_lists = self.fetch_completions(comp_what, mode)
         if not comp_lists[0]:
             return None
+        if (complete and mode == FILES
+                and not any(name.startswith(comp_start)
+                            for name in comp_lists[0])):
+            return None
         self.autocompletewindow = self._make_autocomplete_window()
         return not self.autocompletewindow.show_window(
                 comp_lists, "insert-%dc" % len(comp_start),

Lib/idlelib/idle_test/test_autocomplete.py

@@ -218,6 +218,12 @@ def make_acw(): return self.dummy_acw()
         self.assertTrue(acp.open_completions(ac.TAB))
         self.text.delete('1.0', 'end')
 
+        # No file name starts with the text (gh-60402).
+        self.text.insert('1.0', '"hello wor')
+        self.assertIsNone(acp.open_completions(ac.TAB))
+        self.assertTrue(acp.open_completions(ac.FORCE))
+        self.text.delete('1.0', 'end')
+
     def test_completion_kwds(self):
         self.assertIn('and', ac.completion_kwds)
         self.assertIn('case', ac.completion_kwds)

Lib/test/libregrtest/refleak.py

@@ -188,34 +188,31 @@ def runtest_refleak(test_name, test_func,
     if not quiet:
         print(file=sys.stderr)
 
-    # These checkers return False on success, True on failure
-    def check_rc_deltas(deltas):
-        # Checker for reference counters and memory blocks.
+    failed = False
+    for raw_deltas, item_name in [
+        (rc_deltas, 'references'),
+        (alloc_deltas, 'memory blocks'),
+        (fd_deltas, 'file descriptors')
+    ]:
+        # Ignore warmup runs; convert to a list for reporting
+        deltas = list(raw_deltas[warmups:])
+
+        # Only consider that a test leaks if all deltas are greater than or
+        # equal to 1. Otherwise, ignore deltas.
         #
-        # bpo-30776: Try to ignore false positives:
+        # For example, ignore deltas:
         #
-        #   [3, 0, 0]
-        #   [0, 1, 0]
-        #   [8, -8, 1]
+        #   [3, 0, 0] references, sum=3
+        #   [0, 1, 0] references, sum=1
+        #   [8, -8, 1] references, sum=1
+        #   [0, 1, -1] file descriptors, sum=0
         #
-        # Expected leaks:
+        # Examples of deltas treated as leaks:
         #
-        #   [5, 5, 6]
-        #   [10, 1, 1]
-        return all(delta >= 1 for delta in deltas)
+        #   [5, 5, 6] references, sum=16
+        #   [10, 1, 1] references, sum=12
+        failing = all(delta >= 1 for delta in deltas)
 
-    def check_fd_deltas(deltas):
-        return any(deltas)
-
-    failed = False
-    for raw_deltas, item_name, checker in [
-        (rc_deltas, 'references', check_rc_deltas),
-        (alloc_deltas, 'memory blocks', check_rc_deltas),
-        (fd_deltas, 'file descriptors', check_fd_deltas)
-    ]:
-        # ignore warmup runs; convert to a list for reporting
-        deltas = list(raw_deltas[warmups:])
-        failing = checker(deltas)
         suspicious = any(deltas)
         if failing or suspicious:
             msg = '%s leaked %s %s, sum=%s' % (

Lib/test/test_asyncgen.py

@@ -1,7 +1,9 @@
 import inspect
+import traceback
 import types
 import unittest
 import contextlib
+import warnings
 
 from test.support.import_helper import import_module
 from test.support import gc_collect, requires_working_socket
@@ -709,7 +711,16 @@ def test_send():
         async def test_throw():
             p = ait_class()
             obj = anext(p, "completed")
-            self.assertRaises(SyntaxError, obj.throw, SyntaxError)
+            with warnings.catch_warnings():
+                # Throwing into the unstarted anext() coroutine leaves the
+                # inner __anext__() awaitable never awaited.
+                warnings.simplefilter("ignore", RuntimeWarning)
+                self.assertRaises(SyntaxError, obj.throw, SyntaxError)
+            if isinstance(p, types.AsyncGeneratorType):
+                # The never-run asend() already registered the async
+                # generator with the loop's finalizer; close it explicitly
+                # so no aclose() task is left pending at loop close.
+                await p.aclose()
             return "completed"
 
         result = self.loop.run_until_complete(test_throw())
@@ -1036,6 +1047,40 @@ async def do_test():
         result = self.loop.run_until_complete(do_test())
         self.assertEqual(result, "completed")
 
+    def test_anext_traceback_filename(self):
+        # anext() is implemented in Python in Lib/_pybuiltins.py, which is
+        # frozen under the builtins ID, so its frames name builtins rather
+        # than the module they are frozen from.
+        def filenames(exc):
+            return [frame.filename
+                    for frame in traceback.extract_tb(exc.__traceback__)]
+
+        class AIter:
+            def __aiter__(self):
+                return self
+            async def __anext__(self):
+                raise ZeroDivisionError
+
+        # assertRaises() drops the traceback, so catch the exceptions here.
+        async def do_test():
+            try:
+                anext(42, "default")
+            except TypeError as exc:
+                self.assertIn("<frozen builtins>", filenames(exc))
+            else:
+                self.fail("TypeError was not raised")
+
+            try:
+                await anext(AIter(), "default")
+            except ZeroDivisionError as exc:
+                self.assertIn("<frozen builtins>", filenames(exc))
+            else:
+                self.fail("ZeroDivisionError was not raised")
+            return "completed"
+
+        result = self.loop.run_until_complete(do_test())
+        self.assertEqual(result, "completed")
+
     def test_anext_iter(self):
         @types.coroutine
         def _async_yield(v):
@@ -1132,9 +1177,13 @@ async def agenfn():
                 yield 'aaa'
 
             agen = agenfn()
-            with contextlib.closing(anext(agen, "default").__await__()) as g:
-                with self.assertRaises(MyError):
-                    g.throw(MyError())
+            with warnings.catch_warnings():
+                # Throwing into the unstarted anext() coroutine leaves the
+                # inner asend() awaitable never awaited.
+                warnings.simplefilter("ignore", RuntimeWarning)
+                with contextlib.closing(anext(agen, "default").__await__()) as g:
+                    with self.assertRaises(MyError):
+                        g.throw(MyError())
 
         def run_test(test):
             with self.subTest('pure-Python anext()'):

Lib/test/test_asyncio/test_graph.py

@@ -148,6 +148,46 @@ async def main():
             'async generator CallStackTestBase.test_stack_async_gen.<locals>.gen()',
             stack_for_gen_nested_call[1])
 
+    async def test_stack_anext_default(self):
+        # anext() with a default wraps the awaitable in a coroutine, so the
+        # call graph of a suspended task sees through it into __anext__().
+
+        loop = asyncio.get_running_loop()
+        blocker = loop.create_future()
+
+        async def inner():
+            await blocker
+
+        class AIter:
+            def __aiter__(self):
+                return self
+
+            async def __anext__(self):
+                await inner()
+                return 1
+
+        async def main():
+            await anext(AIter(), None)
+
+        task = asyncio.create_task(main(), name='anext task')
+        await asyncio.sleep(0)
+        try:
+            stack = capture_test_stack(fut=task)
+        finally:
+            blocker.set_result(None)
+            await task
+
+        self.assertEqual(stack[0], [
+            'T<anext task>',
+            [
+                'a inner',
+                'a __anext__',
+                'a _anext_with_default',
+                'a main',
+            ],
+            []
+        ])
+
     def test_ag_frame_used_for_async_generator(self):
         # Regression test for gh-148736: the ag_await branch of
         # _build_graph_for_future must read ag_frame, not cr_frame.

Lib/test/test_coroutines.py

@@ -1312,8 +1312,12 @@ async def __anext__(self):
             def __aiter__(self):
                 return self
 
-        with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable:
-            self.assertRaises(TypeError, anext_awaitable.close, 1)
+        with warnings.catch_warnings():
+            # Closing the unstarted anext() coroutine leaves the inner
+            # __anext__() coroutine never awaited.
+            warnings.simplefilter("ignore", RuntimeWarning)
+            with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable:
+                self.assertRaises(TypeError, anext_awaitable.close, 1)
 
     def test_with_1(self):
         class Manager:

Lib/test/test_importlib/util.py

@@ -69,7 +69,8 @@ def import_importlib(module_name):
     fresh = ('importlib',) if '.' in module_name else ()
     frozen = import_helper.import_fresh_module(module_name)
     source = import_helper.import_fresh_module(module_name, fresh=fresh,
-                                         blocked=('_frozen_importlib', '_frozen_importlib_external'))
+                                         blocked=('_frozen_importlib', '_frozen_importlib_external',
+                                                  '_pybuiltins'))
     return {'Frozen': frozen, 'Source': source}
 
 

Lib/test/test_inspect/test_inspect.py

@@ -6174,7 +6174,7 @@ def test_builtins_have_signatures(self):
                         "next", "vars"}
         no_signature |= needs_groups
         # These have unrepresentable parameter default values of NULL
-        unsupported_signature = {"anext", "aiter", "iter"}
+        unsupported_signature = {"aiter", "iter"}
         # These need *args support in Argument Clinic
         needs_varargs = {"min", "max", "__build_class__"}
         no_signature |= needs_varargs

Lib/test/test_regrtest.py

@@ -1315,29 +1315,47 @@ def test_run(self):
                                   forever=True)
 
     @support.requires_jit_disabled
-    def check_leak(self, code, what, *, run_workers=False):
-        test = self.create_test('huntrleaks', code=code)
+    def check_leak(self, code, what, *, run_workers=False,
+                   name='huntrleaks', deltas=(1, 1, 1)):
+        test = self.create_test(name, code=code)
 
+        leak = all(delta >= 1 for delta in deltas)
         filename = 'reflog.txt'
         self.addCleanup(os_helper.unlink, filename)
         cmd = ['--huntrleaks', '3:3:']
         if run_workers:
             cmd.append('-j1')
         cmd.append(test)
+        if leak:
+            exitcode = EXITCODE_BAD_TEST
+            kwargs = dict(failed=test)
+        else:
+            exitcode = 0
+            kwargs = {}
+
+        try:
+            os_helper.unlink(filename)
+        except FileNotFoundError:
+            pass
         output = self.run_tests(*cmd,
-                                exitcode=EXITCODE_BAD_TEST,
+                                exitcode=exitcode,
                                 stderr=subprocess.STDOUT)
-        self.check_executed_tests(output, [test], failed=test, stats=1)
+        self.check_executed_tests(output, [test], stats=1, **kwargs)
 
-        line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n'
+        digits = ''.join('1' if delta >= 1 else '.' for delta in deltas)
+        line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} %s\n' % digits
         self.check_line(output, line)
 
-        line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what)
-        self.assertIn(line2, output)
+        if leak:
+            line2 = f'{test} leaked {repr(list(deltas))} {what}, sum=3\n'
+            self.assertIn(line2, output)
 
-        with open(filename) as fp:
-            reflog = fp.read()
-            self.assertIn(line2, reflog)
+        if leak:
+            with open(filename) as fp:
+                reflog = fp.read()
+                self.assertIn(line2, reflog)
+        else:
+            self.assertFalse(os.path.exists(filename))
 
     @unittest.skipUnless(support.Py_DEBUG, 'need a debug build')
     def check_huntrleaks(self, *, run_workers: bool):
@@ -1414,6 +1432,31 @@ def test_leak(self):
         """)
         self.check_leak(code, 'file descriptors')
 
+        # Ignore false positive: deltas [1, -1, 0]
+        code = textwrap.dedent("""
+            import os
+            import unittest
+
+            RUN = 0
+            FD = None
+
+            class FDLeakTest(unittest.TestCase):
+                def test_leak(self):
+                    global RUN, FD
+                    RUN += 1
+                    if RUN == 4:
+                        # Create a fd without closing it: leak! (delta=1)
+                        FD = os.open(__file__, os.O_RDONLY)
+                    elif RUN == 5:
+                        # Close fd created in previous run (delta=-1)
+                        os.close(FD)
+                    else:
+                        # Do nothing at the warmup (steps 1-3) and step 6 (delta=0)
+                        pass
+        """)
+        self.check_leak(code, 'file descriptors',
+                        name='no_fd_leak', deltas=(1, -1, 0))
+
     def test_list_tests(self):
         # test --list-tests
         tests = [self.create_test() for i in range(5)]