diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 72f4a2829d89c7d..c03816b5727fa42 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -249,7 +249,7 @@ called with a non-bytes parameter. While bytes objects are usually immutable in Python, this special C API allows mutating a bytes object in-place. The returned bytes object can still - be mutated using :c:func:`PyBytesWriter_GetData`; except if *newsize* is + be mutated using :c:func:`PyBytes_AsString`; except if *newsize* is zero in which case it returns the immutable empty bytes string. .. soft-deprecated:: 3.15 diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index ab627c28c1fa5ee..bfc94e3e8529b75 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -26,6 +26,7 @@ extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); extern void _Py_InitVersion(void); extern PyStatus _PyFaulthandler_Init(int enable); extern PyObject * _PyBuiltin_Init(PyInterpreterState *interp); +extern int _PyBuiltin_InitPythonFunctions(PyObject *dict); extern PyStatus _PySys_Create( PyThreadState *tstate, PyObject **sysmod_p); diff --git a/Lib/_pybuiltins.py b/Lib/_pybuiltins.py new file mode 100644 index 000000000000000..01da295f53d0c73 --- /dev/null +++ b/Lib/_pybuiltins.py @@ -0,0 +1,42 @@ +"""Builtins implemented in Python. + +This module is frozen into the interpreter and imported during startup, +before the import system exists. The names listed in ``__all__`` are +copied into the ``builtins`` module. +""" + +__all__ = ['anext'] + +_NOT_GIVEN = sentinel("_NOT_GIVEN") + + +def anext(async_iterator, default=_NOT_GIVEN, /): + """Return the next item from the async iterator. + + If default is given and the async iterator is exhausted, + it is returned instead of raising StopAsyncIteration. + """ + cls = type(async_iterator) + try: + # Looked up on the type, like the C slot am_anext. + anext_method = cls.__anext__ + except AttributeError: + raise TypeError( + f"{cls.__name__!r} object is not an async iterator" + ) from None + awaitable = anext_method(async_iterator) + if default is _NOT_GIVEN: + return awaitable + return _anext_with_default(awaitable, default) + + +async def _anext_with_default(awaitable, default): + try: + return await awaitable + except StopAsyncIteration: + return default + + +for _name in __all__: + globals()[_name].__module__ = 'builtins' +del _name diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index 032d31225315fb7..9cb809bc7d46466 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -153,6 +153,10 @@ def open_completions(self, args): comp_lists = self.fetch_completions(comp_what, mode) if not comp_lists[0]: return None + if (complete and mode == FILES + and not any(name.startswith(comp_start) + for name in comp_lists[0])): + return None self.autocompletewindow = self._make_autocomplete_window() return not self.autocompletewindow.show_window( comp_lists, "insert-%dc" % len(comp_start), diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 88af3efc35bbd18..9086c31d2733b61 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -218,6 +218,12 @@ def make_acw(): return self.dummy_acw() self.assertTrue(acp.open_completions(ac.TAB)) self.text.delete('1.0', 'end') + # No file name starts with the text (gh-60402). + self.text.insert('1.0', '"hello wor') + self.assertIsNone(acp.open_completions(ac.TAB)) + self.assertTrue(acp.open_completions(ac.FORCE)) + self.text.delete('1.0', 'end') + def test_completion_kwds(self): self.assertIn('and', ac.completion_kwds) self.assertIn('case', ac.completion_kwds) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index ffb8438d1b0278f..69a9c9d6e5f1d7c 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -188,34 +188,31 @@ def runtest_refleak(test_name, test_func, if not quiet: print(file=sys.stderr) - # These checkers return False on success, True on failure - def check_rc_deltas(deltas): - # Checker for reference counters and memory blocks. + failed = False + for raw_deltas, item_name in [ + (rc_deltas, 'references'), + (alloc_deltas, 'memory blocks'), + (fd_deltas, 'file descriptors') + ]: + # Ignore warmup runs; convert to a list for reporting + deltas = list(raw_deltas[warmups:]) + + # Only consider that a test leaks if all deltas are greater than or + # equal to 1. Otherwise, ignore deltas. # - # bpo-30776: Try to ignore false positives: + # For example, ignore deltas: # - # [3, 0, 0] - # [0, 1, 0] - # [8, -8, 1] + # [3, 0, 0] references, sum=3 + # [0, 1, 0] references, sum=1 + # [8, -8, 1] references, sum=1 + # [0, 1, -1] file descriptors, sum=0 # - # Expected leaks: + # Examples of deltas treated as leaks: # - # [5, 5, 6] - # [10, 1, 1] - return all(delta >= 1 for delta in deltas) + # [5, 5, 6] references, sum=16 + # [10, 1, 1] references, sum=12 + failing = all(delta >= 1 for delta in deltas) - def check_fd_deltas(deltas): - return any(deltas) - - failed = False - for raw_deltas, item_name, checker in [ - (rc_deltas, 'references', check_rc_deltas), - (alloc_deltas, 'memory blocks', check_rc_deltas), - (fd_deltas, 'file descriptors', check_fd_deltas) - ]: - # ignore warmup runs; convert to a list for reporting - deltas = list(raw_deltas[warmups:]) - failing = checker(deltas) suspicious = any(deltas) if failing or suspicious: msg = '%s leaked %s %s, sum=%s' % ( diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index cdae58b3e89ae36..b5e0891feb794e8 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1,7 +1,9 @@ import inspect +import traceback import types import unittest import contextlib +import warnings from test.support.import_helper import import_module from test.support import gc_collect, requires_working_socket @@ -709,7 +711,16 @@ def test_send(): async def test_throw(): p = ait_class() obj = anext(p, "completed") - self.assertRaises(SyntaxError, obj.throw, SyntaxError) + with warnings.catch_warnings(): + # Throwing into the unstarted anext() coroutine leaves the + # inner __anext__() awaitable never awaited. + warnings.simplefilter("ignore", RuntimeWarning) + self.assertRaises(SyntaxError, obj.throw, SyntaxError) + if isinstance(p, types.AsyncGeneratorType): + # The never-run asend() already registered the async + # generator with the loop's finalizer; close it explicitly + # so no aclose() task is left pending at loop close. + await p.aclose() return "completed" result = self.loop.run_until_complete(test_throw()) @@ -1036,6 +1047,40 @@ async def do_test(): result = self.loop.run_until_complete(do_test()) self.assertEqual(result, "completed") + def test_anext_traceback_filename(self): + # anext() is implemented in Python in Lib/_pybuiltins.py, which is + # frozen under the builtins ID, so its frames name builtins rather + # than the module they are frozen from. + def filenames(exc): + return [frame.filename + for frame in traceback.extract_tb(exc.__traceback__)] + + class AIter: + def __aiter__(self): + return self + async def __anext__(self): + raise ZeroDivisionError + + # assertRaises() drops the traceback, so catch the exceptions here. + async def do_test(): + try: + anext(42, "default") + except TypeError as exc: + self.assertIn("", filenames(exc)) + else: + self.fail("TypeError was not raised") + + try: + await anext(AIter(), "default") + except ZeroDivisionError as exc: + self.assertIn("", filenames(exc)) + else: + self.fail("ZeroDivisionError was not raised") + return "completed" + + result = self.loop.run_until_complete(do_test()) + self.assertEqual(result, "completed") + def test_anext_iter(self): @types.coroutine def _async_yield(v): @@ -1132,9 +1177,13 @@ async def agenfn(): yield 'aaa' agen = agenfn() - with contextlib.closing(anext(agen, "default").__await__()) as g: - with self.assertRaises(MyError): - g.throw(MyError()) + with warnings.catch_warnings(): + # Throwing into the unstarted anext() coroutine leaves the + # inner asend() awaitable never awaited. + warnings.simplefilter("ignore", RuntimeWarning) + with contextlib.closing(anext(agen, "default").__await__()) as g: + with self.assertRaises(MyError): + g.throw(MyError()) def run_test(test): with self.subTest('pure-Python anext()'): diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index a442a346ff06d91..1326ef50c149b0b 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -148,6 +148,46 @@ async def main(): 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1]) + async def test_stack_anext_default(self): + # anext() with a default wraps the awaitable in a coroutine, so the + # call graph of a suspended task sees through it into __anext__(). + + loop = asyncio.get_running_loop() + blocker = loop.create_future() + + async def inner(): + await blocker + + class AIter: + def __aiter__(self): + return self + + async def __anext__(self): + await inner() + return 1 + + async def main(): + await anext(AIter(), None) + + task = asyncio.create_task(main(), name='anext task') + await asyncio.sleep(0) + try: + stack = capture_test_stack(fut=task) + finally: + blocker.set_result(None) + await task + + self.assertEqual(stack[0], [ + 'T', + [ + 'a inner', + 'a __anext__', + 'a _anext_with_default', + 'a main', + ], + [] + ]) + def test_ag_frame_used_for_async_generator(self): # Regression test for gh-148736: the ag_await branch of # _build_graph_for_future must read ag_frame, not cr_frame. diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index ab854d56d5a3ebf..c5900da09501b36 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -1312,8 +1312,12 @@ async def __anext__(self): def __aiter__(self): return self - with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable: - self.assertRaises(TypeError, anext_awaitable.close, 1) + with warnings.catch_warnings(): + # Closing the unstarted anext() coroutine leaves the inner + # __anext__() coroutine never awaited. + warnings.simplefilter("ignore", RuntimeWarning) + with contextlib.closing(anext(A(), "a").__await__()) as anext_awaitable: + self.assertRaises(TypeError, anext_awaitable.close, 1) def test_with_1(self): class Manager: diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index 6399f952f9e912b..0cc0e651e45622b 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -69,7 +69,8 @@ def import_importlib(module_name): fresh = ('importlib',) if '.' in module_name else () frozen = import_helper.import_fresh_module(module_name) source = import_helper.import_fresh_module(module_name, fresh=fresh, - blocked=('_frozen_importlib', '_frozen_importlib_external')) + blocked=('_frozen_importlib', '_frozen_importlib_external', + '_pybuiltins')) return {'Frozen': frozen, 'Source': source} diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index df5843abfcb8753..25276fc40cb0287 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -6174,7 +6174,7 @@ def test_builtins_have_signatures(self): "next", "vars"} no_signature |= needs_groups # These have unrepresentable parameter default values of NULL - unsupported_signature = {"anext", "aiter", "iter"} + unsupported_signature = {"aiter", "iter"} # These need *args support in Argument Clinic needs_varargs = {"min", "max", "__build_class__"} no_signature |= needs_varargs diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index c966f8659e2abb0..71edec40ade8079 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1315,29 +1315,47 @@ def test_run(self): forever=True) @support.requires_jit_disabled - def check_leak(self, code, what, *, run_workers=False): - test = self.create_test('huntrleaks', code=code) + def check_leak(self, code, what, *, run_workers=False, + name='huntrleaks', deltas=(1, 1, 1)): + test = self.create_test(name, code=code) + leak = all(delta >= 1 for delta in deltas) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) cmd = ['--huntrleaks', '3:3:'] if run_workers: cmd.append('-j1') cmd.append(test) + if leak: + exitcode = EXITCODE_BAD_TEST + kwargs = dict(failed=test) + else: + exitcode = 0 + kwargs = {} + + try: + os_helper.unlink(filename) + except FileNotFoundError: + pass output = self.run_tests(*cmd, - exitcode=EXITCODE_BAD_TEST, + exitcode=exitcode, stderr=subprocess.STDOUT) - self.check_executed_tests(output, [test], failed=test, stats=1) + self.check_executed_tests(output, [test], stats=1, **kwargs) - line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} 111\n' + digits = ''.join('1' if delta >= 1 else '.' for delta in deltas) + line = r'beginning 6 repetitions. .*\n123:456\n[.0-9X]{3} %s\n' % digits self.check_line(output, line) - line2 = '%s leaked [1, 1, 1] %s, sum=3\n' % (test, what) - self.assertIn(line2, output) + if leak: + line2 = f'{test} leaked {repr(list(deltas))} {what}, sum=3\n' + self.assertIn(line2, output) - with open(filename) as fp: - reflog = fp.read() - self.assertIn(line2, reflog) + if leak: + with open(filename) as fp: + reflog = fp.read() + self.assertIn(line2, reflog) + else: + self.assertFalse(os.path.exists(filename)) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def check_huntrleaks(self, *, run_workers: bool): @@ -1414,6 +1432,31 @@ def test_leak(self): """) self.check_leak(code, 'file descriptors') + # Ignore false positive: deltas [1, -1, 0] + code = textwrap.dedent(""" + import os + import unittest + + RUN = 0 + FD = None + + class FDLeakTest(unittest.TestCase): + def test_leak(self): + global RUN, FD + RUN += 1 + if RUN == 4: + # Create a fd without closing it: leak! (delta=1) + FD = os.open(__file__, os.O_RDONLY) + elif RUN == 5: + # Close fd created in previous run (delta=-1) + os.close(FD) + else: + # Do nothing at the warmup (steps 1-3) and step 6 (delta=0) + pass + """) + self.check_leak(code, 'file descriptors', + name='no_fd_leak', deltas=(1, -1, 0)) + def test_list_tests(self): # test --list-tests tests = [self.create_test() for i in range(5)] diff --git a/Makefile.pre.in b/Makefile.pre.in index 166087f32dff187..72e0ba3267d069d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1623,7 +1623,8 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS) BOOTSTRAP_HEADERS = \ Python/frozen_modules/importlib._bootstrap.h \ Python/frozen_modules/importlib._bootstrap_external.h \ - Python/frozen_modules/zipimport.h + Python/frozen_modules/zipimport.h \ + Python/frozen_modules/builtins.h Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) @@ -1664,6 +1665,7 @@ FROZEN_FILES_IN = \ Lib/importlib/_bootstrap.py \ Lib/importlib/_bootstrap_external.py \ Lib/zipimport.py \ + Lib/_pybuiltins.py \ Lib/abc.py \ Lib/codecs.py \ Lib/io.py \ @@ -1690,6 +1692,7 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/importlib._bootstrap.h \ Python/frozen_modules/importlib._bootstrap_external.h \ Python/frozen_modules/zipimport.h \ + Python/frozen_modules/builtins.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ Python/frozen_modules/io.h \ @@ -1735,6 +1738,9 @@ Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_ Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) $(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h +Python/frozen_modules/builtins.h: Lib/_pybuiltins.py $(FREEZE_MODULE_BOOTSTRAP_DEPS) + $(FREEZE_MODULE_BOOTSTRAP) builtins $(srcdir)/Lib/_pybuiltins.py Python/frozen_modules/builtins.h + Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst new file mode 100644 index 000000000000000..b56307056a7fff5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-12-16-40-00.gh-issue-157361.anextpy.rst @@ -0,0 +1,4 @@ +Implement :func:`anext` in Python instead of C, in a frozen ``_pybuiltins`` +module. The awaitable returned by ``anext(it, default)`` is now a plain +coroutine, so introspection tools such as :func:`asyncio.print_call_graph` +can see through it into :meth:`~object.__anext__`. diff --git a/Misc/NEWS.d/next/IDLE/2026-09-15-21-44-35.gh-issue-60402.iFz6X6.rst b/Misc/NEWS.d/next/IDLE/2026-09-15-21-44-35.gh-issue-60402.iFz6X6.rst new file mode 100644 index 000000000000000..9b876ab0486f20d --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-15-21-44-35.gh-issue-60402.iFz6X6.rst @@ -0,0 +1,2 @@ +In IDLE, Tab in a string that is not the start of a file name inserts a tab +instead of opening the completion list. diff --git a/Misc/NEWS.d/next/Tests/2026-09-16-17-06-54.gh-issue-157628.KAwhlY.rst b/Misc/NEWS.d/next/Tests/2026-09-16-17-06-54.gh-issue-157628.KAwhlY.rst new file mode 100644 index 000000000000000..c232ebca619a21d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-09-16-17-06-54.gh-issue-157628.KAwhlY.rst @@ -0,0 +1,3 @@ +In regrtest, only consider that a test leaks if all test runs leak at least +one file descriptor. For example, ignore "[0, 1, -1] file descriptors, +sum=0" deltas, instead of reporting a leak. Patch by Victor Stinner. diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 07cfba8c8be74be..2291280ec117f2c 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -352,7 +352,8 @@ _close_range_except(int start_fd, int (*closer)(int, int)) { if (end_fd == -1) { - end_fd = Py_MIN(safe_get_max_fd(), INT_MAX); + end_fd = safe_get_max_fd(); + end_fd = Py_MIN(end_fd, INT_MAX); } Py_ssize_t keep_seq_idx; /* As fds_to_keep is sorted we can loop through the list closing diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 199e4ac3db723bf..a79ea5728e72d64 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -571,7 +571,8 @@ ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) if (deadline) { // _PyDeadline_Get will return a negative value if the deadline has // been exceeded. - timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0); + timeout_ns = _PyDeadline_Get(deadline); + timeout_ns = Py_MAX(timeout_ns, 0); } if (timeout_ns) { diff --git a/Objects/dictobject.c b/Objects/dictobject.c index f0feea4b717ec91..9a469f88230f8b2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3412,9 +3412,8 @@ dict_dict_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) PyObject *key; Py_hash_t hash; int unicode = DK_IS_UNICODE(((PyDictObject*)iterable)->ma_keys); - uint8_t new_size = Py_MAX( - estimate_log2_keysize(PyDict_GET_SIZE(iterable)), - DK_LOG_SIZE(mp->ma_keys)); + uint8_t log2_keysize = estimate_log2_keysize(PyDict_GET_SIZE(iterable)); + uint8_t new_size = Py_MAX(log2_keysize, DK_LOG_SIZE(mp->ma_keys)); if (dictresize(mp, new_size, unicode)) { Py_DECREF(mp); return NULL; @@ -3437,9 +3436,8 @@ dict_set_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) Py_ssize_t pos = 0; PyObject *key; Py_hash_t hash; - uint8_t new_size = Py_MAX( - estimate_log2_keysize(PySet_GET_SIZE(iterable)), - DK_LOG_SIZE(mp->ma_keys)); + uint8_t log2_keysize = estimate_log2_keysize(PySet_GET_SIZE(iterable)); + uint8_t new_size = Py_MAX(log2_keysize, DK_LOG_SIZE(mp->ma_keys)); if (dictresize(mp, new_size, 0)) { Py_DECREF(mp); return NULL; diff --git a/Objects/iterobject.c b/Objects/iterobject.c index b5783c92c8eb689..2d5e3709a27dfb0 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -403,33 +403,6 @@ PyTypeObject PyCallIter_Type = { /* -------------------------------------- */ -typedef struct { - PyObject_HEAD - PyObject *wrapped; - PyObject *default_value; -} anextawaitableobject; - -#define anextawaitableobject_CAST(op) ((anextawaitableobject *)(op)) - -static void -anextawaitable_dealloc(PyObject *op) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - _PyObject_GC_UNTRACK(obj); - Py_XDECREF(obj->wrapped); - Py_XDECREF(obj->default_value); - PyObject_GC_Del(obj); -} - -static int -anextawaitable_traverse(PyObject *op, visitproc visit, void *arg) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - Py_VISIT(obj->wrapped); - Py_VISIT(obj->default_value); - return 0; -} - static PyObject * awaitable_getiter(PyObject *owner, PyObject *wrapped) { @@ -461,99 +434,6 @@ awaitable_getiter(PyObject *owner, PyObject *wrapped) return awaitable; } -static PyObject * -anextawaitable_iternext(PyObject *op) -{ - /* Consider the following class: - * - * class A: - * async def __anext__(self): - * ... - * a = A() - * - * Then `await anext(a)` should call - * a.__anext__().__await__().__next__() - * - * On the other hand, given - * - * async def agen(): - * yield 1 - * yield 2 - * gen = agen() - * - * Then `await anext(gen)` can just call - * gen.__anext__().__next__() - */ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - PyObject *awaitable = awaitable_getiter(op, obj->wrapped); - if (awaitable == NULL) { - return NULL; - } - PyObject *result = (*Py_TYPE(awaitable)->tp_iternext)(awaitable); - Py_DECREF(awaitable); - if (result != NULL) { - return result; - } - if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { - PyErr_Clear(); - _PyGen_SetStopIterationValue(obj->default_value); - } - return NULL; -} - - -static PyObject * -anextawaitable_proxy(anextawaitableobject *obj, char *meth, PyObject *arg) -{ - PyObject *awaitable = awaitable_getiter((PyObject *)obj, obj->wrapped); - if (awaitable == NULL) { - return NULL; - } - // When specified, 'arg' may be a tuple (if coming from a METH_VARARGS - // method) or a single object (if coming from a METH_O method). - PyObject *ret = arg == NULL - ? PyObject_CallMethod(awaitable, meth, NULL) - : PyObject_CallMethod(awaitable, meth, "O", arg); - Py_DECREF(awaitable); - if (ret != NULL) { - return ret; - } - if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration)) { - /* `anextawaitableobject` is only used by `anext()` when - * a default value is provided. So when we have a StopAsyncIteration - * exception we replace it with a `StopIteration(default)`, as if - * it was the return value of `__anext__()` coroutine. - */ - PyErr_Clear(); - _PyGen_SetStopIterationValue(obj->default_value); - } - return NULL; -} - - -static PyObject * -anextawaitable_send(PyObject *op, PyObject *arg) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "send", arg); -} - - -static PyObject * -anextawaitable_throw(PyObject *op, PyObject *args) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "throw", args); -} - - -static PyObject * -anextawaitable_close(PyObject *op, PyObject *Py_UNUSED(dummy)) -{ - anextawaitableobject *obj = anextawaitableobject_CAST(op); - return anextawaitable_proxy(obj, "close", NULL); -} - PyDoc_STRVAR(send_doc, "send(arg) -> send 'arg' into the wrapped iterator,\n\ @@ -574,68 +454,6 @@ PyDoc_STRVAR(close_doc, "close() -> raise GeneratorExit inside generator."); -static PyMethodDef anextawaitable_methods[] = { - {"send", anextawaitable_send, METH_O, send_doc}, - {"throw", anextawaitable_throw, METH_VARARGS, throw_doc}, - {"close", anextawaitable_close, METH_NOARGS, close_doc}, - {NULL, NULL} /* Sentinel */ -}; - - -static PyAsyncMethods anextawaitable_as_async = { - PyObject_SelfIter, /* am_await */ - 0, /* am_aiter */ - 0, /* am_anext */ - 0, /* am_send */ -}; - -PyTypeObject _PyAnextAwaitable_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "anext_awaitable", /* tp_name */ - sizeof(anextawaitableobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - anextawaitable_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - &anextawaitable_as_async, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - anextawaitable_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - anextawaitable_iternext, /* tp_iternext */ - anextawaitable_methods, /* tp_methods */ -}; - -PyObject * -PyAnextAwaitable_New(PyObject *awaitable, PyObject *default_value) -{ - anextawaitableobject *anext = PyObject_GC_New( - anextawaitableobject, &_PyAnextAwaitable_Type); - if (anext == NULL) { - return NULL; - } - anext->wrapped = Py_NewRef(awaitable); - anext->default_value = Py_NewRef(default_value); - _PyObject_GC_TRACK(anext); - return (PyObject *)anext; -} - - /* -------------------------------------- */ /* The asynchronous counterpart of calliterobject: the callable is called diff --git a/Objects/longobject.c b/Objects/longobject.c index 6454565aebf6a11..e35f938629326ab 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6269,9 +6269,10 @@ static Py_ssize_t int___sizeof___impl(PyObject *self) /*[clinic end generated code: output=3303f008eaa6a0a5 input=9b51620c76fc4507]*/ { + Py_ssize_t ndigits = _PyLong_DigitCount((PyLongObject *)self); /* using Py_MAX(..., 1) because we always allocate space for at least one digit, even though the integer zero has a digit count of 0 */ - Py_ssize_t ndigits = Py_MAX(_PyLong_DigitCount((PyLongObject *)self), 1); + ndigits = Py_MAX(ndigits, 1); return Py_TYPE(self)->tp_basicsize + Py_TYPE(self)->tp_itemsize * ndigits; } diff --git a/Objects/object.c b/Objects/object.c index a83f8d4c04ca079..e3f29b71301695e 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2522,7 +2522,6 @@ _PyObject_FiniState(PyInterpreterState *interp) extern PyTypeObject _PyACallIter_Type; extern PyTypeObject _PyACallIterAwaitable_Type; -extern PyTypeObject _PyAnextAwaitable_Type; extern PyTypeObject _PyLegacyEventHandler_Type; extern PyTypeObject _PyLineIterator; extern PyTypeObject _PyMemoryIter_Type; @@ -2617,7 +2616,6 @@ static PyTypeObject* static_types[_Py_NUM_MANAGED_PREINITIALIZED_TYPES] = { &Py_GenericAliasType, &_PyACallIter_Type, &_PyACallIterAwaitable_Type, - &_PyAnextAwaitable_Type, &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, diff --git a/Objects/unicode_formatter.c b/Objects/unicode_formatter.c index b8604d1355940a5..2b7681f7ff3ce94 100644 --- a/Objects/unicode_formatter.c +++ b/Objects/unicode_formatter.c @@ -892,8 +892,10 @@ calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix, if (spec->n_lpadding || spec->n_spadding || spec->n_rpadding) *maxchar = Py_MAX(*maxchar, format->fill_char); - if (spec->n_decimal) - *maxchar = Py_MAX(*maxchar, PyUnicode_MAX_CHAR_VALUE(locale->decimal_point)); + if (spec->n_decimal) { + Py_UCS4 point_maxchar = PyUnicode_MAX_CHAR_VALUE(locale->decimal_point); + *maxchar = Py_MAX(*maxchar, point_maxchar); + } return spec->n_lpadding + spec->n_sign + spec->n_prefix + spec->n_spadding + spec->n_grouped_digits + spec->n_decimal + diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 70c54e0e41efc63..69833f132b5e4d4 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -306,6 +306,11 @@ $(IntDir)zipimport.g.h $(GeneratedFrozenModulesDir)Python\frozen_modules\zipimport.h + + builtins + $(IntDir)builtins.g.h + $(GeneratedFrozenModulesDir)Python\frozen_modules\builtins.h + abc $(IntDir)abc.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index b0799b8dc9ecddb..207552113c3dd23 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -549,6 +549,9 @@ Python Files + + Python Files + Python Files diff --git a/Programs/_bootstrap_python.c b/Programs/_bootstrap_python.c index 6443d814a22dabf..d30ef8c879d8153 100644 --- a/Programs/_bootstrap_python.c +++ b/Programs/_bootstrap_python.c @@ -13,6 +13,7 @@ #include "Python/frozen_modules/importlib._bootstrap.h" #include "Python/frozen_modules/importlib._bootstrap_external.h" #include "Python/frozen_modules/zipimport.h" +#include "Python/frozen_modules/builtins.h" /* End includes */ /* Note that a negative size indicates a package. */ @@ -21,6 +22,7 @@ static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)}, + {"_pybuiltins", _Py_M__builtins, (int)sizeof(_Py_M__builtins)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { @@ -36,6 +38,7 @@ const struct _frozen *_PyImport_FrozenTest = test_modules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, {"_frozen_importlib_external", "importlib._bootstrap_external"}, + {"_pybuiltins", "builtins"}, {0, 0} /* aliases sentinel */ }; const struct _module_alias *_PyImport_FrozenAliases = aliases; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index d28e6fa9cd01aed..965cf20fe617787 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1960,52 +1960,6 @@ builtin_aiter_impl(PyObject *module, PyObject *object, PyObject *stop_value, return _PyACallIter_New(object, stop_value, stop_exception); } -PyObject *PyAnextAwaitable_New(PyObject *, PyObject *); - -/*[clinic input] -anext as builtin_anext - - async_iterator as aiterator: object - default: object = NULL - / - -Return the next item from the async iterator. - -If default is given and the async iterator is exhausted, -it is returned instead of raising StopAsyncIteration. -[clinic start generated code]*/ - -static PyObject * -builtin_anext_impl(PyObject *module, PyObject *aiterator, - PyObject *default_value) -/*[clinic end generated code: output=f02c060c163a81fa input=f3dc5a93f073e5ac]*/ -{ - PyTypeObject *t; - PyObject *awaitable; - - t = Py_TYPE(aiterator); - if (t->tp_as_async == NULL || t->tp_as_async->am_anext == NULL) { - PyErr_Format(PyExc_TypeError, - "'%.200s' object is not an async iterator", - t->tp_name); - return NULL; - } - - awaitable = (*t->tp_as_async->am_anext)(aiterator); - if (awaitable == NULL) { - return NULL; - } - if (default_value == NULL) { - return awaitable; - } - - PyObject* new_awaitable = PyAnextAwaitable_New( - awaitable, default_value); - Py_DECREF(awaitable); - return new_awaitable; -} - - /*[clinic input] len as builtin_len @@ -3500,7 +3454,6 @@ static PyMethodDef builtin_methods[] = { {"max", _PyCFunction_CAST(builtin_max), METH_FASTCALL | METH_KEYWORDS, max_doc}, {"min", _PyCFunction_CAST(builtin_min), METH_FASTCALL | METH_KEYWORDS, min_doc}, {"next", _PyCFunction_CAST(builtin_next), METH_FASTCALL, next_doc}, - BUILTIN_ANEXT_METHODDEF BUILTIN_OCT_METHODDEF BUILTIN_ORD_METHODDEF BUILTIN_POW_METHODDEF @@ -3539,6 +3492,57 @@ static struct PyModuleDef builtinsmodule = { }; +/* Builtins implemented in Python. + + Lib/_pybuiltins.py is frozen into the interpreter as a bootstrap module + (see Tools/build/freeze_modules.py), so it can be imported here before + the import system exists. The names in its __all__ are copied into the + builtins dict. */ + +int +_PyBuiltin_InitPythonFunctions(PyObject *dict) +{ + if (PyImport_ImportFrozenModule("_pybuiltins") <= 0) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, + "frozen module _pybuiltins not found"); + } + return -1; + } + PyObject *mod = PyImport_AddModuleRef("_pybuiltins"); + if (mod == NULL) { + return -1; + } + + int rc = -1; + PyObject *all = PyObject_GetAttr(mod, &_Py_ID(__all__)); + if (all == NULL) { + goto done; + } + Py_ssize_t n = PyList_Size(all); + if (n < 0) { + goto done; + } + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *name = PyList_GET_ITEM(all, i); + PyObject *func = PyObject_GetAttr(mod, name); + if (func == NULL) { + goto done; + } + int r = PyDict_SetItem(dict, name, func); + Py_DECREF(func); + if (r < 0) { + goto done; + } + } + rc = 0; + +done: + Py_XDECREF(all); + Py_DECREF(mod); + return rc; +} + PyObject * _PyBuiltin_Init(PyInterpreterState *interp) { diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c10bb03d8178161..5858ca9ff88ec22 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -1011,44 +1011,6 @@ builtin_aiter(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -PyDoc_STRVAR(builtin_anext__doc__, -"anext($module, async_iterator, default=, /)\n" -"--\n" -"\n" -"Return the next item from the async iterator.\n" -"\n" -"If default is given and the async iterator is exhausted,\n" -"it is returned instead of raising StopAsyncIteration."); - -#define BUILTIN_ANEXT_METHODDEF \ - {"anext", _PyCFunction_CAST(builtin_anext), METH_FASTCALL, builtin_anext__doc__}, - -static PyObject * -builtin_anext_impl(PyObject *module, PyObject *aiterator, - PyObject *default_value); - -static PyObject * -builtin_anext(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *aiterator; - PyObject *default_value = NULL; - - if (!_PyArg_CheckPositional("anext", nargs, 1, 2)) { - goto exit; - } - aiterator = args[0]; - if (nargs < 2) { - goto skip_optional; - } - default_value = args[1]; -skip_optional: - return_value = builtin_anext_impl(module, aiterator, default_value); - -exit: - return return_value; -} - PyDoc_STRVAR(builtin_len__doc__, "len($module, obj, /)\n" "--\n" @@ -1539,4 +1501,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=5fb1ac6a4253ee2f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b56739f2e13f616a input=a9049054013a1b77]*/ diff --git a/Python/frozen.c b/Python/frozen.c index 9433d90c15e2eca..1f92ea01dc38bbc 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -44,6 +44,7 @@ #include "frozen_modules/importlib._bootstrap.h" #include "frozen_modules/importlib._bootstrap_external.h" #include "frozen_modules/zipimport.h" +#include "frozen_modules/builtins.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" #include "frozen_modules/io.h" @@ -71,6 +72,7 @@ static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false}, {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false}, {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), false}, + {"_pybuiltins", _Py_M__builtins, (int)sizeof(_Py_M__builtins), false}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { @@ -119,6 +121,7 @@ const struct _frozen *_PyImport_FrozenTest = test_modules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, {"_frozen_importlib_external", "importlib._bootstrap_external"}, + {"_pybuiltins", "builtins"}, {"__hello_alias__", "__hello__"}, {"__phello_alias__", "__hello__"}, {"__phello_alias__.spam", "__hello__"}, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 500a1a1949a5a8a..3feb06915a59c2c 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -924,6 +924,16 @@ pycore_init_builtins(PyThreadState *tstate) return _PyStatus_ERR("failed to add exceptions to builtins"); } + /* The Python-implemented builtins live in the frozen _pybuiltins module. + Programs/_freeze_module has no frozen modules (it's what creates + them) and opts out via _install_importlib, like the import system. */ + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + if (config->_install_importlib) { + if (_PyBuiltin_InitPythonFunctions(builtins_dict) < 0) { + return _PyStatus_ERR("failed to add Python-implemented builtins"); + } + } + interp->builtins_copy = PyDict_Copy(interp->builtins); if (interp->builtins_copy == NULL) { goto error; diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 8937e666bbbdd5b..565be27b7cebda5 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -65,6 +65,7 @@ static const char* _Py_stdlib_module_names[] = { "_posixsubprocess", "_py_abc", "_py_warnings", +"_pybuiltins", "_pydatetime", "_pydecimal", "_pyio", diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index a866336fa78879e..8c817daec0f0b37 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -16,6 +16,9 @@ FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py') STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib') +# Frozen under the "builtins" ID rather than its own name, so that the frames +# of the builtins it defines show up as "" in tracebacks. +PYBUILTINS = os.path.join(STDLIB_DIR, '_pybuiltins.py') # If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the # .gitattributes and .gitignore files needs to be updated. FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') @@ -45,6 +48,8 @@ # This module is important because some Python builds rely # on a builtin zip file instead of a filesystem. 'zipimport', + # Builtins implemented in Python; loaded while builtins is set up. + f'builtins : _pybuiltins = {PYBUILTINS}', ]), # (You can delete entries from here down to the end of the list.) ('stdlib - startup, without site (python -S)', [ @@ -91,6 +96,7 @@ 'importlib._bootstrap', 'importlib._bootstrap_external', 'zipimport', + 'builtins', } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index b8488899c4595de..67ced170243e4a2 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -60,7 +60,6 @@ Objects/iterobject.c - PyCallIter_Type - Objects/iterobject.c - PySeqIter_Type - Objects/iterobject.c - _PyACallIter_Type - Objects/iterobject.c - _PyACallIterAwaitable_Type - -Objects/iterobject.c - _PyAnextAwaitable_Type - Objects/lazyimportobject.c - PyLazyImport_Type - Objects/listobject.c - PyListIter_Type - Objects/listobject.c - PyListRevIter_Type - @@ -77,7 +76,6 @@ Objects/object.c - _PyNone_Type - Objects/object.c - _PyNotImplemented_Type - Objects/object.c - _PyACallIter_Type - Objects/object.c - _PyACallIterAwaitable_Type - -Objects/object.c - _PyAnextAwaitable_Type - Objects/odictobject.c - PyODictItems_Type - Objects/odictobject.c - PyODictIter_Type - Objects/odictobject.c - PyODictKeys_Type -