From 9232c21a1c1eb6a12b96acebbef6c86a698228e0 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Sep 2026 21:29:36 +0300 Subject: [PATCH 1/7] gh-74112: Make Ctrl-C in the IDLE Shell interrupt blocking calls (GH-157662) Send a real SIGINT to the main thread of the user process instead of calling _thread.interrupt_main(), which only sets a flag checked between bytecodes. The signal is sent while holding a new lock which protects sending a message, so that the main thread is not interrupted in the middle of a message. An interrupted wait for a response now releases its lock, so that the socket thread does not deadlock. --- Lib/idlelib/idle_test/test_rpc.py | 14 +++++++ Lib/idlelib/idle_test/test_run.py | 32 +++++++++++++++ Lib/idlelib/rpc.py | 39 +++++++++++-------- Lib/idlelib/run.py | 15 ++++++- ...6-09-17-12-00-00.gh-issue-74112.sigint.rst | 2 + 5 files changed, 85 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-17-12-00-00.gh-issue-74112.sigint.rst diff --git a/Lib/idlelib/idle_test/test_rpc.py b/Lib/idlelib/idle_test/test_rpc.py index 15c3ed14b8f6f3..d929165e2e5ec9 100644 --- a/Lib/idlelib/idle_test/test_rpc.py +++ b/Lib/idlelib/idle_test/test_rpc.py @@ -3,7 +3,9 @@ from idlelib import rpc import socket import struct +import threading import unittest +from unittest import mock class SocketIOTest(unittest.TestCase): @@ -22,6 +24,18 @@ def test_reconnect_discards_partial_packet(self): new_peer.sendall(struct.pack(' 0: - try: - r, w, x = select.select([], [self.sock], []) - n = self.sock.send(s[:BUFSIZE]) - except (AttributeError, TypeError): - raise OSError("socket no longer exists") - s = s[n:] + with self.sendlock: + while len(s) > 0: + try: + r, w, x = select.select([], [self.sock], []) + n = self.sock.send(s[:BUFSIZE]) + except (AttributeError, TypeError): + raise OSError("socket no longer exists") + s = s[n:] def pollpacket(self, wait): self._stage0() diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 2725043b4ed925..c69060620f5f9f 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -9,6 +9,7 @@ import io import linecache import queue +import signal import sys import textwrap import time @@ -678,7 +679,19 @@ def runcode(self, code): def interrupt_the_server(self): if interruptible: - thread.interrupt_main() + handler = signal.getsignal(signal.SIGINT) + if handler not in (signal.SIG_DFL, signal.SIG_IGN, None): + # A real signal interrupts blocking calls such as + # time.sleep() (gh-74112). The lock prevents interrupting + # the main thread in the middle of sending a message. + with self.rpchandler.sendlock: + if hasattr(signal, 'pthread_kill'): + signal.pthread_kill(threading.main_thread().ident, + signal.SIGINT) + else: + signal.raise_signal(signal.SIGINT) + else: + thread.interrupt_main() def start_the_debugger(self, gui_adap_oid): return debugger_r.start_debugger(self.rpchandler, gui_adap_oid) diff --git a/Misc/NEWS.d/next/IDLE/2026-09-17-12-00-00.gh-issue-74112.sigint.rst b/Misc/NEWS.d/next/IDLE/2026-09-17-12-00-00.gh-issue-74112.sigint.rst new file mode 100644 index 00000000000000..18a230e75913b5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-17-12-00-00.gh-issue-74112.sigint.rst @@ -0,0 +1,2 @@ +Ctrl-C in the IDLE Shell now interrupts blocking calls such as +:func:`time.sleep` and :meth:`socket.recv `. From 2f5791df35c66cf5f300d82d86dd6854c71d12bc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Sep 2026 21:21:38 +0200 Subject: [PATCH 2/7] gh-158001: No longer read global config vars in PyConfig_Read() (#158014) PyConfig_Read() and _PyPreConfig_Read() no longer read global config variables (such as Py_BytesWarningFlag). Instead, PyConfig_Read() now copies PyPreConfig members (isolated, use_environment and dev_mode). _PyPreConfig_Read() still reads the last global configuration variable: Py_UTF8Mode. --- Lib/test/test_embed.py | 17 +------- Programs/_testembed.c | 93 ------------------------------------------ Python/initconfig.c | 85 +++++++++++++++++++++++++++----------- Python/preconfig.c | 34 +++++++-------- 4 files changed, 77 insertions(+), 152 deletions(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 9770fac956e649..81241ea1f33733 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1095,24 +1095,11 @@ def test_init_compat_config(self): self.check_all_configs("test_init_compat_config", api=API_COMPAT) def test_init_global_config(self): + # Test Py_UTF8Mode global configuration variable preconfig = { 'utf8_mode': True, } - config = { - 'site_import': False, - 'bytes_warning': True, - 'warnoptions': ['default::BytesWarning'], - 'inspect': True, - 'interactive': True, - 'optimization_level': 2, - 'write_bytecode': False, - 'verbose': True, - 'quiet': True, - 'buffered_stdio': False, - 'remote_debug': True, - 'user_site_directory': False, - 'pathconfig_warnings': False, - } + config = {} self.check_all_configs("test_init_global_config", config, preconfig, api=API_COMPAT) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 63260c9e5f6cc4..17b93ba47caac4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -597,43 +597,9 @@ static int test_init_compat_config(void) static int test_init_global_config(void) { - /* FIXME: test Py_IgnoreEnvironmentFlag */ - putenv("PYTHONUTF8=0"); Py_UTF8Mode = 1; - /* Py_IsolatedFlag is not tested */ - Py_NoSiteFlag = 1; - Py_BytesWarningFlag = 1; - - putenv("PYTHONINSPECT="); - Py_InspectFlag = 1; - - putenv("PYTHONOPTIMIZE=0"); - Py_InteractiveFlag = 1; - - putenv("PYTHONDEBUG=0"); - Py_OptimizeFlag = 2; - - /* Py_DebugFlag is not tested */ - - putenv("PYTHONDONTWRITEBYTECODE="); - Py_DontWriteBytecodeFlag = 1; - - putenv("PYTHONVERBOSE=0"); - Py_VerboseFlag = 1; - - Py_QuietFlag = 1; - Py_NoUserSiteDirectory = 1; - - putenv("PYTHONUNBUFFERED="); - Py_UnbufferedStdioFlag = 1; - - Py_FrozenFlag = 1; - - /* FIXME: test Py_LegacyWindowsFSEncodingFlag */ - /* FIXME: test Py_LegacyWindowsStdioFlag */ - _testembed_initialize(); dump_config(); Py_Finalize(); @@ -734,39 +700,30 @@ static int test_init_from_config(void) config_set_string(&config, &config.platlibdir, L"my_platlibdir"); putenv("PYTHONVERBOSE=0"); - Py_VerboseFlag = 0; config.verbose = 1; - Py_NoSiteFlag = 0; config.site_import = 0; - Py_BytesWarningFlag = 0; config.bytes_warning = 1; putenv("PYTHONINSPECT="); - Py_InspectFlag = 0; config.inspect = 1; - Py_InteractiveFlag = 0; config.interactive = 1; putenv("PYTHONOPTIMIZE=0"); - Py_OptimizeFlag = 1; config.optimization_level = 2; /* FIXME: test parser_debug */ putenv("PYTHONDONTWRITEBYTECODE="); - Py_DontWriteBytecodeFlag = 0; config.write_bytecode = 0; - Py_QuietFlag = 0; config.quiet = 1; config.configure_c_stdio = 1; putenv("PYTHONUNBUFFERED="); - Py_UnbufferedStdioFlag = 0; config.buffered_stdio = 0; putenv("PYTHONIOENCODING=cp424"); @@ -774,12 +731,10 @@ static int test_init_from_config(void) config_set_string(&config, &config.stdio_errors, L"replace"); putenv("PYTHONNOUSERSITE="); - Py_NoUserSiteDirectory = 0; config.user_site_directory = 0; config_set_string(&config, &config.check_hash_pycs_mode, L"always"); - Py_FrozenFlag = 0; config.pathconfig_warnings = 0; config.safe_path = 1; @@ -882,7 +837,6 @@ static void set_all_env_vars(void) static int test_init_compat_env(void) { /* Test initialization from environment variables */ - Py_IgnoreEnvironmentFlag = 0; set_all_env_vars(); _testembed_initialize(); dump_config(); @@ -918,7 +872,6 @@ static void set_all_env_vars_dev_mode(void) static int test_init_env_dev_mode(void) { /* Test initialization from environment variables */ - Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); _testembed_initialize(); dump_config(); @@ -930,7 +883,6 @@ static int test_init_env_dev_mode(void) static int test_init_env_dev_mode_alloc(void) { /* Test initialization from environment variables */ - Py_IgnoreEnvironmentFlag = 0; set_all_env_vars_dev_mode(); #ifndef Py_GIL_DISABLED putenv("PYTHONMALLOC=malloc"); @@ -950,7 +902,6 @@ static int test_init_isolated_flag(void) PyConfig config; PyConfig_InitPythonConfig(&config); - Py_IsolatedFlag = 0; config.isolated = 1; // These options are set to 1 by isolated=1 config.safe_path = 0; @@ -1010,7 +961,6 @@ static int test_preinit_isolated2(void) PyConfig config; _PyConfig_InitCompatConfig(&config); - Py_IsolatedFlag = 0; config.isolated = 1; config_set_program_name(&config); @@ -1081,28 +1031,6 @@ static int test_preinit_parse_argv(void) -static void set_all_global_config_variables(void) -{ - Py_IsolatedFlag = 0; - Py_IgnoreEnvironmentFlag = 0; - Py_BytesWarningFlag = 2; - Py_InspectFlag = 1; - Py_InteractiveFlag = 1; - Py_OptimizeFlag = 1; - Py_DebugFlag = 1; - Py_VerboseFlag = 1; - Py_QuietFlag = 1; - Py_FrozenFlag = 0; - Py_UnbufferedStdioFlag = 1; - Py_NoSiteFlag = 1; - Py_DontWriteBytecodeFlag = 1; - Py_NoUserSiteDirectory = 1; -#ifdef MS_WINDOWS - Py_LegacyWindowsStdioFlag = 1; -#endif -} - - static int check_preinit_isolated_config(int preinit) { PyStatus status; @@ -1111,9 +1039,6 @@ static int check_preinit_isolated_config(int preinit) /* environment variables must be ignored */ set_all_env_vars(); - /* global configuration variables must be ignored */ - set_all_global_config_variables(); - if (preinit) { PyPreConfig preconfig; PyPreConfig_InitIsolatedConfig(&preconfig); @@ -1158,19 +1083,6 @@ static int test_init_isolated_config(void) static int check_init_python_config(int preinit) { - /* global configuration variables must be ignored */ - set_all_global_config_variables(); - Py_IsolatedFlag = 1; - Py_IgnoreEnvironmentFlag = 1; - Py_FrozenFlag = 1; - Py_UnbufferedStdioFlag = 1; - Py_NoSiteFlag = 1; - Py_DontWriteBytecodeFlag = 1; - Py_NoUserSiteDirectory = 1; -#ifdef MS_WINDOWS - Py_LegacyWindowsStdioFlag = 1; -#endif - if (preinit) { PyPreConfig preconfig; PyPreConfig_InitPythonConfig(&preconfig); @@ -1276,7 +1188,6 @@ static int test_open_code_hook(void) return 2; } - Py_IgnoreEnvironmentFlag = 0; _testembed_initialize(); result = 0; @@ -1339,7 +1250,6 @@ static int _test_audit(Py_ssize_t setValue) { Py_ssize_t sawSet = 0; - Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook, &sawSet); _testembed_initialize(); @@ -1451,7 +1361,6 @@ static int _audit_subinterpreter_hook(const char *event, PyObject *args, void *u static int test_audit_subinterpreter(void) { - Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_subinterpreter_hook, NULL); _testembed_initialize(); @@ -1501,7 +1410,6 @@ static int test_audit_run_command(void) AuditRunCommandTest test = {"cpython.run_command"}; wchar_t *argv[] = {PROGRAM_NAME, L"-c", L"pass"}; - Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook_run, (void*)&test); return Py_Main(Py_ARRAY_LENGTH(argv), argv); @@ -1512,7 +1420,6 @@ static int test_audit_run_file(void) AuditRunCommandTest test = {"cpython.run_file"}; wchar_t *argv[] = {PROGRAM_NAME, L"filename.py"}; - Py_IgnoreEnvironmentFlag = 0; PySys_AddAuditHook(_audit_hook_run, (void*)&test); return Py_Main(Py_ARRAY_LENGTH(argv), argv); diff --git a/Python/initconfig.c b/Python/initconfig.c index ac0845b892903c..464c76f9e3df2f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1818,31 +1818,19 @@ config_get_env_dup(PyConfig *config, static void -config_get_global_vars(PyConfig *config) +config_read_preconfig(PyConfig *config) { - if (config->_config_init != _PyConfig_INIT_COMPAT) { - /* Python and Isolated configuration ignore global variables */ - return; - } - - const PyConfigSpec *spec = PYCONFIG_SPEC; - for (; spec->name != NULL; spec++) { - if (spec->global_var.ptr == NULL) { - continue; - } - assert(spec->type == PyConfig_MEMBER_INT - || spec->type == PyConfig_MEMBER_UINT - || spec->type == PyConfig_MEMBER_BOOL); - int *member = config_get_spec_member(config, spec); - if (*member != -1) { - continue; - } - int value = *spec->global_var.ptr; - if (spec->global_var.not) { - value = !value; +#define COPY_FLAG(ATTR) \ + if (config->ATTR == -1) { \ + config->ATTR = preconfig->ATTR; \ } - *member = value; - } + + const PyPreConfig *preconfig = &_PyRuntime.preconfig; + COPY_FLAG(isolated); + COPY_FLAG(use_environment); + COPY_FLAG(dev_mode); + +#undef COPY_FLAG } @@ -3748,7 +3736,56 @@ _PyConfig_Read(PyConfig *config, int compute_path_config) return status; } - config_get_global_vars(config); + config_read_preconfig(config); + + // Set default values + if (config->bytes_warning < 0) { + config->bytes_warning = 0; + } + if (config->inspect < 0) { + config->inspect = 0; + } + if (config->interactive < 0) { + config->interactive = 0; + } + if (config->optimization_level < 0) { + config->optimization_level = 0; + } + if (config->parser_debug < 0) { + config->parser_debug = 0; + } + if (config->quiet < 0) { + config->quiet = 0; + } + if (config->use_environment < 0) { + config->use_environment = 0; + } + if (config->verbose < 0) { + config->verbose = 0; + } + if (config->write_bytecode < 0) { + config->write_bytecode = 1; + } + if (config->buffered_stdio < 0) { + config->buffered_stdio = 1; + } + if (config->isolated < 0) { + config->isolated = 0; + } +#ifdef MS_WINDOWS + if (config->legacy_windows_stdio < 0) { + config->legacy_windows_stdio = 0; + } +#endif + if (config->pathconfig_warnings < 0) { + config->pathconfig_warnings = 1; + } + if (config->site_import < 0) { + config->site_import = 1; + } + if (config->user_site_directory < 0) { + config->user_site_directory = 1; + } #ifdef __CYGWIN__ status = config_argv0_add_exe(config); diff --git a/Python/preconfig.c b/Python/preconfig.c index 2c8c18284c1d2d..16594e545abaed 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -463,36 +463,19 @@ _PyPreConfig_GetConfig(PyPreConfig *preconfig, const PyConfig *config) static void -preconfig_get_global_vars(PyPreConfig *config) +preconfig_get_global_var(PyPreConfig *config) { if (config->_config_init != _PyConfig_INIT_COMPAT) { /* Python and Isolated configuration ignore global variables */ return; } -#define COPY_FLAG(ATTR, VALUE) \ - if (config->ATTR < 0) { \ - config->ATTR = VALUE; \ - } -#define COPY_NOT_FLAG(ATTR, VALUE) \ - if (config->ATTR < 0) { \ - config->ATTR = !(VALUE); \ - } - _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS - COPY_FLAG(isolated, Py_IsolatedFlag); - COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); if (Py_UTF8Mode > 0) { config->utf8_mode = Py_UTF8Mode; } -#ifdef MS_WINDOWS - COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); -#endif _Py_COMP_DIAG_POP - -#undef COPY_FLAG -#undef COPY_NOT_FLAG } @@ -776,7 +759,7 @@ preconfig_read(PyPreConfig *config, _PyPreCmdline *cmdline) - command line arguments - environment variables - - Py_xxx global configuration variables + - Py_UTF8Mode global configuration variable - the LC_CTYPE locale */ PyStatus _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) @@ -788,7 +771,18 @@ _PyPreConfig_Read(PyPreConfig *config, const _PyArgv *args) return status; } - preconfig_get_global_vars(config); + preconfig_get_global_var(config); + if (config->use_environment < 0) { + config->use_environment = 1; + } + if (config->isolated < 0) { + config->isolated = 0; + } +#ifdef MS_WINDOWS + if (config->legacy_windows_fs_encoding < 0) { + config->legacy_windows_fs_encoding = 0; + } +#endif /* Copy LC_CTYPE locale, since it's modified later */ const char *loc = setlocale(LC_CTYPE, NULL); From a9d42dceee5b8d2afba3223d633f5d9b6dad01f3 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:31:57 +0100 Subject: [PATCH 3/7] gh-124697: Represent inlined comprehensions as subscopes in the symbol table (#156819) --- Doc/library/symtable.rst | 10 + Doc/whatsnew/3.16.rst | 10 + Include/internal/pycore_compile.h | 21 +- Include/internal/pycore_symtable.h | 8 +- InternalDocs/README.md | 2 + InternalDocs/compiler.md | 4 + InternalDocs/inlined_comprehensions.md | 132 ++++++++ Lib/symtable.py | 15 +- Lib/test/test_compiler_assemble.py | 3 +- Lib/test/test_listcomps.py | 281 ++++++++++++++++ Lib/test/test_symtable.py | 255 +++++++++++++- Lib/test/test_syntax.py | 17 + ...-09-02-12-11-00.gh-issue-124697.sUbScp.rst | 3 + Modules/_testinternalcapi.c | 5 +- Modules/symtablemodule.c | 3 +- Objects/frameobject.c | 107 +++++- Python/assemble.c | 12 +- Python/codegen.c | 72 ++-- Python/compile.c | 292 +++++++++-------- Python/symtable.c | 310 +++++++++--------- 20 files changed, 1187 insertions(+), 375 deletions(-) create mode 100644 InternalDocs/inlined_comprehensions.md create mode 100644 Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 859687340882de..a5cc0f854ab4c3 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -57,6 +57,16 @@ Examining Symbol Tables Used for the symbol table of a class. + .. attribute:: INLINED_COMPREHENSION + :value: "inlined comprehension" + + Used for the symbol table of a list, set or dict comprehension that + is inlined into the enclosing code unit (see :pep:`709`). A symbol + table of this type represents a sub-scope of the enclosing code unit's + scope, and it does not correspond to a separate compilation unit. + + .. versionadded:: next + The following members refer to different flavors of :ref:`annotation scopes `. diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 8362b1ef7e312b..6c38beb04e8d27 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -573,6 +573,16 @@ symtable like the builtin :func:`compile`. (Contributed by Serhiy Storchaka in :gh:`153844`.) +* Inlined list, set and dict comprehensions (:pep:`709`) are now represented + as their own symbol table entries, of type + :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. Each such entry is + a lexical child of the enclosing scope and records the comprehension's own + locals, cells, and free names. It does not correspond to a separate + compilation unit. For names loaded only inside an inlined list, set, or + dict comprehension, :meth:`symtable.Symbol.is_referenced` can now return + ``True`` on the enclosing function. + (Contributed by Irit Katriel in :gh:`124697`.) + tkinter ------- diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 4597ae2763ad77..31738f2074dbbf 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -69,9 +69,8 @@ typedef struct { PyObject *u_varnames; /* local variables */ PyObject *u_cellvars; /* cell variables */ PyObject *u_freevars; /* free variables */ - PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only - temporarily within an inlined comprehension. When - value is True, treat as fast-local. */ + PyObject *u_fasthidden; /* set of names that are fast-locals only + temporarily within an inlined comprehension. */ Py_ssize_t u_argcount; /* number of arguments for block */ Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ @@ -152,11 +151,10 @@ PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name); int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e); int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name); int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name); -int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope, +int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, _PyCompile_optype *optype, Py_ssize_t *arg); int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c); -int _PyCompile_IsInInlinedComp(struct _PyCompiler *c); int _PyCompile_ScopeType(struct _PyCompiler *c); int _PyCompile_OptimizationLevel(struct _PyCompiler *c); int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name); @@ -180,16 +178,15 @@ enum { typedef struct { PyObject *pushed_locals; - PyObject *temp_symbols; - PyObject *fast_hidden; _PyJumpTargetLabel cleanup; + PySTEntryObject *saved_ste; } _PyCompile_InlinedComprehensionState; -int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state); -int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc, - _PyCompile_InlinedComprehensionState *state); +int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c, + PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state); +int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c, + _PyCompile_InlinedComprehensionState *state); int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s, PyObject **conditional_annotation_index); void _PyCompile_EnterConditionalBlock(struct _PyCompiler *c); diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index c650a94a1eab2e..01afe5bb935b99 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -33,6 +33,10 @@ typedef enum _block_type { // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two // do not support a bound or a constraint tuple). TypeVariableBlock, + // List/set/dict comprehension inlined into the enclosing compilation unit + // (PEP 709). Lexical child of that unit, not a separate code object. + // See InternalDocs/inlined_comprehensions.md. + InlinedComprehensionBlock, } _Py_block_ty; typedef enum _comprehension_type { @@ -119,7 +123,6 @@ typedef struct _symtable_entry { should be created */ unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure over the class dict should be created */ - unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */ unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */ unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an enclosing class scope */ @@ -132,6 +135,7 @@ typedef struct _symtable_entry { int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */ _Py_SourceLocation ste_loc; /* source location of block */ struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ + struct _symtable_entry *ste_parent; /* st entry for the enclosing block if this entry is a sub-scope, NULL otherwise */ struct symtable *ste_table; } PySTEntryObject; @@ -142,6 +146,7 @@ extern PyTypeObject PySTEntry_Type; extern long _PyST_GetSymbol(PySTEntryObject *, PyObject *); extern int _PyST_GetScope(PySTEntryObject *, PyObject *); extern int _PyST_IsFunctionLike(PySTEntryObject *); +extern int _PyST_IsClassClosureName(PyObject *); extern struct symtable* _PySymtable_Build( struct _mod *mod, @@ -172,7 +177,6 @@ _Py_IsPrivateName(PyObject *); #define DEF_ANNOT (2<<7) /* this name is annotated */ #define DEF_COMP_ITER (2<<8) /* this name is a comprehension iteration variable */ #define DEF_TYPE_PARAM (2<<9) /* this name is a type parameter */ -#define DEF_COMP_CELL (2<<10) /* this name is a cell in an inlined comprehension */ #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT) diff --git a/InternalDocs/README.md b/InternalDocs/README.md index 3e8ab442315753..5de575b0f44952 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -23,6 +23,8 @@ Compiling Python Source Code - [Compiler Design](compiler.md) +- [Inlined comprehensions](inlined_comprehensions.md) + - [Changing Python's Grammar](changing_grammar.md) Runtime Objects diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 9ed4d0eb65a0bd..de9e76ec9fcdb1 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -354,6 +354,10 @@ AST node type). Next, the AST tree is walked with the various code blocks that delineate the reach of a local variable as blocks are entered and exited using `symtable_enter_block()` and `symtable_exit_block()`, respectively. +See [Inlined comprehensions](inlined_comprehensions.md) for how list, set, +and dict comprehensions are represented as lexical subscopes of the +enclosing unit. + Once the symbol table is created, the `AST` is transformed by `compiler_codegen()` in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instructions. These are similar to bytecode, but in some cases they are more abstract, and are diff --git a/InternalDocs/inlined_comprehensions.md b/InternalDocs/inlined_comprehensions.md new file mode 100644 index 00000000000000..e1ccd485c165b1 --- /dev/null +++ b/InternalDocs/inlined_comprehensions.md @@ -0,0 +1,132 @@ +Inlined comprehensions +====================== + +Since [PEP 709](https://peps.python.org/pep-0709/), list, set, and dict +comprehensions are compiled into the enclosing compilation unit instead of +creating a nested function and calling it. Generator expressions are not +inlined; they still become their own code object. + +The resulting bytecode lives in the enclosing unit, but the comprehension +still has its own locals: iteration variables must not leak into, or +overwrite, names in the enclosing scope. The symbol table models that as a +nested lexical scope; codegen then emits the comprehension inlined into its +containing compilation unit. + +Which comprehensions are inlined +-------------------------------- + +`symtable_handle_comprehension()` in +[`Python/symtable.c`](../Python/symtable.c) inlines a comprehension when it +is not a generator expression and the current block cannot see class scope +(`!ste_can_see_class_scope`). Annotation scopes that can see a class keep +the historical nested-function compilation so class-local names are not +treated as comprehension locals. + +The outermost iterator expression is always evaluated in the enclosing +scope. The rest of the comprehension (targets, `if` clauses, inner +generators, and the element/value expressions) is visited in the +comprehension's own block. + +Symbol table +------------ + +An inlined comprehension gets an `InlinedComprehensionBlock` entry +([`pycore_symtable.h`](../Include/internal/pycore_symtable.h)). That entry +is a child of the enclosing block, with `ste_parent` pointing at the +enclosing `PySTEntryObject`. It is a lexical subscope, not a compilation +unit: there is no separate code object, `co_consts` entry, or compiler +scope for it. + +Uses and bindings inside the comprehension are recorded on that child +table. Because those loads are in the enclosing compilation unit, a +`FREE` use is also marked `USE` on the parent table. For +`def inner(): return [x for y in ()]`, both `inner.lookup("x")` and the +inlined child's lookup are `FREE|USE`. + +### Analysis + +`analyze_block()` records this block's declarations, analyzes children, +then classifies uses. `finalize_inlined_comprehension()` copies +`USE` from an inlined child onto the parent before that second pass, and +drops inlined-only frees so `analyze_cells()` does not promote those +names to cells. + +A name that is `FREE` in the comprehension and bound in the parent is +dropped from the parent's free set unless: + +* a real nested unit (function, lambda, or genexp) still needs it as a + cell, or +* a sibling nested scope already marked it free. + +That keeps iteration variables as fast locals when they are only used by +nested inlined comprehensions. A nested lambda that captures the name +still forces a cell on the binding comprehension. + +Class-closure names (`__class__` and friends) loaded from an inlined +comprehension do not require a class cell unless a nested function, +lambda, or genexp captures them. Compile treats the inlined loads as +implicit globals. `is_free_in_any_child()` walks through inlined children +and only counts `FREE` on non-inlined descendants. + +Compiler +-------- + +Codegen stays in the enclosing compiler unit. Around the inlined region, +`_PyCompile_EnterInlinedComprehensionScope()` / +`_PyCompile_ExitInlinedComprehensionScope()` in +[`Python/compile.c`](../Python/compile.c) swap `c->u->u_ste` so name +lookup uses the comprehension's symbol table. The saved `u_ste` is +restored on both success and error. + +`_PyCompile_ResolveNameop()` calls `compiler_resolve_inlined_free()`, +which walks `ste_parent` while the current table is inlined and the name +is `FREE` or missing (scope `0`). Missing names include loads synthesized +by codegen, such as the implicit receiver for zero-argument `super()`. +The walk stops at a class: nested scopes do not see class locals. +Class-closure names that would otherwise be free through a class become +`GLOBAL_IMPLICIT`. + +### Isolating iteration variables + +`codegen_push_inlined_comprehension_locals()` in +[`Python/codegen.c`](../Python/codegen.c) isolates names bound in the +comprehension: + +* `LOAD_FAST_AND_CLEAR` saves the enclosing value (possibly `NULL`) and + clears the slot. +* `MAKE_CELL` runs if the name is a cell for this comprehension. +* In module and class units the name is added to `u_fasthidden` so + assemble can set `CO_FAST_HIDDEN`. + +A `SETUP_FINALLY` / `COMPILE_FBLOCK_INLINED_COMPREHENSION` handler +restores those slots if the comprehension raises, so an enclosing `except` +or `finally` sees the original values. + +Runtime +------- + +An inlined comprehension cell can share a localsplus name with an +enclosing free variable (for example `[lambda: x for x in x]` inside a +nested function). `FrameLocalsProxy` keys, values, items, and `len` +keep the first slot of each name so they agree with `getitem`. + +Source +------ + +* [`Python/symtable.c`](../Python/symtable.c): + `symtable_handle_comprehension()`, `analyze_block()`, + `finalize_inlined_comprehension()`, `is_free_in_any_child()` +* [`Python/compile.c`](../Python/compile.c): + `compiler_resolve_inlined_free()`, + `_PyCompile_EnterInlinedComprehensionScope()`, + `compiler_cellvars()` +* [`Python/codegen.c`](../Python/codegen.c): + `codegen_comprehension()`, + `push_inlined_comprehension_state()`, + `codegen_push_inlined_comprehension_locals()` +* [`Include/internal/pycore_symtable.h`](../Include/internal/pycore_symtable.h): + `InlinedComprehensionBlock` +* [`Include/internal/pycore_compile.h`](../Include/internal/pycore_compile.h): + `_PyCompile_InlinedComprehensionState` +* [`Objects/frameobject.c`](../Objects/frameobject.c): + `FrameLocalsProxy` duplicate-name handling diff --git a/Lib/symtable.py b/Lib/symtable.py index 18bb355d86b09e..3e56e8ab99c1c3 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -7,7 +7,7 @@ DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, DEF_TYPE_PARAM, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, DEF_ANNOT, - DEF_COMP_ITER, DEF_COMP_CELL, + DEF_COMP_ITER, SCOPE_OFF, SCOPE_MASK, FREE, LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL ) @@ -56,6 +56,7 @@ class SymbolTableType(StrEnum): TYPE_ALIAS = "type alias" TYPE_PARAMETERS = "type parameters" TYPE_VARIABLE = "type variable" + INLINED_COMPREHENSION = "inlined comprehension" class SymbolTable: @@ -98,6 +99,8 @@ def get_type(self): return SymbolTableType.TYPE_PARAMETERS if self._table.type == _symtable.TYPE_TYPE_VARIABLE: return SymbolTableType.TYPE_VARIABLE + if self._table.type == _symtable.TYPE_INLINED_COMPREHENSION: + return SymbolTableType.INLINED_COMPREHENSION assert False, f"unexpected type: {self._table.type}" def get_id(self): @@ -151,8 +154,10 @@ def lookup(self, name): flags = self._table.symbols[name] namespaces = self.__check_children(name) module_scope = (self._table.name == "top") + inlined = (self._table.type == _symtable.TYPE_INLINED_COMPREHENSION) sym = self._symbols[name] = Symbol(name, flags, namespaces, - module_scope=module_scope) + module_scope=module_scope, + inlined_comprehension=inlined) return sym def get_symbols(self): @@ -246,12 +251,14 @@ class Class(SymbolTable): class Symbol: - def __init__(self, name, flags, namespaces=None, *, module_scope=False): + def __init__(self, name, flags, namespaces=None, *, module_scope=False, + inlined_comprehension=False): self.__name = name self.__flags = flags self.__scope = _get_scope(flags) self.__namespaces = namespaces or () self.__module_scope = module_scope + self.__inlined_comprehension = inlined_comprehension def __repr__(self): flags_str = '|'.join(self._flags_str()) @@ -345,7 +352,7 @@ def is_comp_iter(self): def is_comp_cell(self): """Return *True* if the symbol is a cell in an inlined comprehension. """ - return bool(self.__flags & DEF_COMP_CELL) + return self.is_cell() and self.__inlined_comprehension def is_namespace(self): """Returns *True* if name binding introduces new namespace. diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index 99a11e99d56485..6e04df99b453ec 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -17,8 +17,9 @@ def complete_metadata(self, metadata, filename="myfile.py"): metadata.setdefault(key, key) for key in ['consts']: metadata.setdefault(key, []) - for key in ['names', 'varnames', 'cellvars', 'freevars', 'fasthidden']: + for key in ['names', 'varnames', 'cellvars', 'freevars']: metadata.setdefault(key, {}) + metadata.setdefault('fasthidden', None) for key in ['argcount', 'posonlyargcount', 'kwonlyargcount']: metadata.setdefault(key, 0) metadata.setdefault('firstlineno', 1) diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index fca9acbc6b1ef6..f02f3223a513d2 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -4,6 +4,7 @@ import types import unittest +from test import support from test.support import BrokenIter @@ -165,6 +166,30 @@ def test_references_super(self): """ self._check_in_scopes(code, outputs={"res": [super]}) + def test_zero_arg_super_in_inlined_comprehension(self): + class A: + def f(self): + return 42 + + class B(A): + def f(self): + return [super().f() for _ in (0,)] + + def nested(self): + return [[super().f() for _ in (0,)] for _ in (0,)] + + def setcomp(self): + return [{super().f() for _ in (0,)}] + + def dictcomp(self): + return {0: {1: super().f() for _ in (0,)} for _ in (0,)} + + b = B() + self.assertEqual(b.f(), [42]) + self.assertEqual(b.nested(), [[42]]) + self.assertEqual(b.setcomp(), [{42}]) + self.assertEqual(b.dictcomp(), {0: {1: 42}}) + def test_references___class__(self): code = """ res = [__class__ for x in [1]] @@ -277,6 +302,20 @@ def f(): outputs = {"y": [1]} self._check_in_scopes(code, outputs, scopes=["module", "function"]) + def test_inlined_comp_cell_with_enclosing_free(self): + # The listcomp cell and the enclosing free must not share an index. + code = """ + def outer(y): + def inner(): + return [lambda: x for x in (1, 2)], y + return inner() + funcs, val = outer(99) + z = [f() for f in funcs] + w = val + """ + outputs = {"z": [2, 2], "w": 99} + self._check_in_scopes(code, outputs) + def test_free_inner_cell_outer(self): code = """ g = 2 @@ -407,6 +446,175 @@ def test_nested(self): outputs = {"y": [[0, 1], [0, 1, 4]]} self._check_in_scopes(code, outputs) + def test_nested_inner_uses_outer_iter(self): + # Inner comprehension reads the outer iteration variable. In a class + # this must not be treated as a class-level name of the same name. + code = """ + x = 99 + y = [[x for _ in (0,)] for x in (42,)] + """ + outputs = {"y": [[42]]} + self._check_in_scopes(code, outputs) + + def test_nested_mixed_comprehensions_use_outer_iter(self): + cases = [ + (""" + x = 99 + y = [{x for _ in (0,)} for x in (42,)] + """, {"y": [{42}]}), + (""" + x = 99 + y = [{x: x for _ in (0,)} for x in (42,)] + """, {"y": [{42: 42}]}), + (""" + x = 99 + y = {[x for _ in (0,)][0] for x in (42,)} + """, {"y": {42}}), + (""" + x = 99 + y = {x: [x for _ in (0,)] for x in (42,)} + """, {"y": {42: [42]}}), + ] + for code, outputs in cases: + with self.subTest(code=code): + self._check_in_scopes(code, outputs) + + def test_nested_triple_inner_uses_outer_iter(self): + code = """ + x = 99 + y = [[[x for _ in (0,)] for _ in (0,)] for x in (42,)] + """ + outputs = {"y": [[[42]]]} + self._check_in_scopes(code, outputs) + + def test_nested_inner_uses_outer_iter_in_iter(self): + code = """ + x = 99 + y = [[_ for _ in (x,)] for x in (42,)] + """ + outputs = {"y": [[42]]} + self._check_in_scopes(code, outputs) + + def test_nested_inner_uses_outer_iter_in_if(self): + code = """ + x = 99 + y = [[1 for _ in (0,) if x] for x in (42,)] + """ + outputs = {"y": [[1]]} + self._check_in_scopes(code, outputs) + + def test_nested_sibling_inners_use_outer_iter(self): + code = """ + x = 99 + y = [([x for _ in (0,)], [x for _ in (1,)]) for x in (42,)] + """ + outputs = {"y": [([42], [42])]} + self._check_in_scopes(code, outputs) + + def test_nested_lambda_captures_outer_iter(self): + code = """ + x = 99 + y = [[lambda: x for _ in (0,)] for x in (42,)] + z = y[0][0]() + """ + outputs = {"z": 42} + self._check_in_scopes(code, outputs) + + def test_nested_inlined_comp_iter_var_is_fast_local(self): + def f(n): + return [[x for _ in range(2)] for x in range(n)] + self.assertEqual(f.__code__.co_cellvars, ()) + self.assertEqual(f(2), [[0, 0], [1, 1]]) + + def g(n): + return [[(lambda: x) for _ in range(2)] for x in range(n)] + self.assertEqual(g.__code__.co_cellvars, ("x",)) + self.assertEqual([fn() for fn in g(2)[1]], [1, 1]) + + @support.requires_working_socket() + def test_nested_inlined_async_comp_iter_var_is_fast_local(self): + import asyncio + + async def agen(n): + for i in range(n): + yield i + + async def f(n): + return [[x async for _ in agen(2)] async for x in agen(n)] + + self.assertEqual(f.__code__.co_cellvars, ()) + self.assertEqual(asyncio.run(f(2)), [[0, 0], [1, 1]]) + + async def g(n): + return [[(lambda: x) async for _ in agen(2)] async for x in agen(n)] + + self.assertEqual(g.__code__.co_cellvars, ("x",)) + out = asyncio.run(g(2)) + self.assertEqual([fn() for fn in out[1]], [1, 1]) + + def test_inlined_comprehension_name_mangling_in_method_scope(self): + class C: + def f(self): + __x = 42 + return [__x for _ in (0,)] + + self.assertEqual(C().f(), [42]) + + def test_nested_references___class__(self): + code = """ + res = [[__class__ for _ in (0,)] for _ in (1,)] + """ + self._check_in_scopes(code, raises=NameError) + + def test_nested_references___class___via_lambda(self): + class _C: + res = [[lambda: __class__ for _ in (0,)] for _ in (1,)] + self.assertIs(_C.res[0][0](), _C) + + def test_nested_references_super(self): + code = """ + res = [[super for _ in (0,)] for _ in (1,)] + """ + self._check_in_scopes(code, outputs={"res": [[super]]}) + + def test_nested_inlined_super_does_not_require_class_cell(self): + # Nested inlined comps compile super/__class__ as global lookups. + # They must not inject __classcell__ the way a nested function would. + class Meta(type): + def __new__(mcls, name, bases, ns): + ns.pop('__classcell__', None) + return type.__new__(mcls, name, bases, ns) + + cases = [ + ("[[super for _ in (0,)] for _ in (0,)]", [[super]]), + ("[{super for _ in (0,)} for _ in (0,)]", [{super}]), + ("{0: {1: super for _ in (0,)} for _ in (0,)}", {0: {1: super}}), + ] + for expr, expected in cases: + with self.subTest(expr=expr): + ns = {"Meta": Meta} + exec(f"class C(metaclass=Meta):\n result = {expr}", ns) + self.assertEqual(ns["C"].result, expected) + + def test_nested_inlined_lambda_class_ref_requires_class_cell(self): + class Meta(type): + def __new__(mcls, name, bases, ns): + ns.pop('__classcell__', None) + return type.__new__(mcls, name, bases, ns) + + exprs = [ + "[[lambda: __class__ for _ in (0,)] for _ in (0,)]", + "[{lambda: __class__ for _ in (0,)} for _ in (0,)]", + "{0: {1: (lambda: __class__) for _ in (0,)} for _ in (0,)}", + ] + for expr in exprs: + with self.subTest(expr=expr): + with self.assertRaisesRegex( + RuntimeError, + r"__class__ not set.*__classcell__ propagated"): + exec(f"class C(metaclass=Meta):\n result = {expr}", + {"Meta": Meta}) + def test_nested_2(self): code = """ l = [1, 2, 3] @@ -703,6 +911,79 @@ def test_frame_locals(self): """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) + def test_frame_locals_comp_cell_and_enclosing_free(self): + # The inlined listcomp cell and the enclosing free share a name. + # f_locals keys must still be unique so dict(**f_locals) works. + # keys(), values(), items(), and len() must agree (first slot wins). + code = """ + def outer(x): + def inner(): + return [(lambda: x, + dict(**sys._getframe().f_locals), + len(sys._getframe().f_locals), + list(sys._getframe().f_locals.keys()), + list(sys._getframe().f_locals.values()), + list(sys._getframe().f_locals.items()), + dict(sys._getframe().f_locals.items())) + for x in x] + return inner() + result = outer([1, 2]) + snaps = [d['x'] for _, d, *_ in result] + vals = [fn() for fn, *_ in result] + consistent = [] + for _, d, n, ks, vs, it, d_items in result: + consistent.append( + n == len(ks) == len(vs) == len(it) + and ks.count('x') == 1 + and d == d_items == dict(zip(ks, vs)) + ) + """ + import sys + self._check_in_scopes( + code, + {"snaps": [1, 2], "vals": [2, 2], "consistent": [True, True]}, + ns={"sys": sys}, scopes=["module", "function"]) + + def test_frame_locals_nested_comp_cell_and_enclosing_free(self): + # Stress a nested inlined shape where a comp cell and enclosing free + # share a name; all f_locals views must stay consistent. + code = """ + def outer(x): + def inner(): + return [( + lambda: x, + [[x for _ in (0,)] for _ in (0,)][0][0], + dict(**sys._getframe().f_locals), + len(sys._getframe().f_locals), + list(sys._getframe().f_locals.keys()), + list(sys._getframe().f_locals.values()), + list(sys._getframe().f_locals.items()), + dict(sys._getframe().f_locals.items())) + for x in x] + return inner() + result = outer([1, 2]) + snaps = [d['x'] for _, _, d, *_ in result] + vals = [fn() for fn, *_ in result] + nested_vals = [nested for _, nested, *_ in result] + consistent = [] + for _, _, d, n, ks, vs, it, d_items in result: + consistent.append( + n == len(ks) == len(vs) == len(it) + and ks.count('x') == 1 + and d == d_items == dict(zip(ks, vs)) + ) + """ + import sys + self._check_in_scopes( + code, + { + "snaps": [1, 2], + "vals": [2, 2], + "nested_vals": [1, 2], + "consistent": [True, True], + }, + ns={"sys": sys}, scopes=["module", "function"]) + def _recursive_replace(self, maybe_code): if not isinstance(maybe_code, types.CodeType): return maybe_code diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ce02b27c599c42..da00b0bfa74b12 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -426,12 +426,12 @@ def test_symbol_repr(self): "") st1 = symtable.symtable("[x for x in [1]]", "?", "exec") - self.assertEqual(repr(st1.lookup("x")), + self.assertEqual(repr(st1.get_children()[0].lookup("x")), "") st2 = symtable.symtable("[(lambda: x) for x in [1]]", "?", "exec") - self.assertEqual(repr(st2.lookup("x")), - "") + self.assertEqual(repr(st2.get_children()[0].lookup("x")), + "") st3 = symtable.symtable("def f():\n" " x = 1\n" @@ -502,6 +502,255 @@ def test_nested_genexpr(self): self.assertEqual(sorted(st.get_identifiers()), [".0", "y"]) self.assertEqual(st.get_children(), []) + def test_inlined_comprehension_in_genexpr(self): + st = symtable.symtable("([y for y in x] for x in a)", "?", "exec") + self.assertEqual(len(st.get_children()), 1) + st = st.get_children()[0] + self.assertIs(st.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(st.get_name(), "") + self.assertFalse(st.is_nested()) + self.assertEqual(sorted(st.get_identifiers()), [".0", "x"]) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["y"], nested=True) + + def check_inlined_listcomp(self, st, identifiers, *, nested, nchildren=0): + self.assertIs(st.get_type(), symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertEqual(st.get_name(), "") + self.assertEqual(st.is_nested(), nested) + self.assertEqual(sorted(st.get_identifiers()), identifiers) + children = st.get_children() + self.assertEqual(len(children), nchildren) + return children + + def check_nested_inlined_listcomp(self, outer, outer_ids, inner_ids, *, nested): + inner, = self.check_inlined_listcomp( + outer, outer_ids, nested=nested, nchildren=1) + self.check_inlined_listcomp(inner, inner_ids, nested=True) + return inner + + def test_inlined_comprehension(self): + st = symtable.symtable("[x for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_inlined_listcomp(children[0], ["x"], nested=False) + + def test_inlined_nested_comprehension(self): + st = symtable.symtable("[[y for y in x] for x in [1]]", "?", "exec") + self.assertEqual(sorted(st.get_identifiers()), []) + children = st.get_children() + self.assertEqual(len(children), 1) + self.check_nested_inlined_listcomp( + children[0], ["x"], ["y"], nested=False) + + def test_inlined_comprehension_use_of_enclosing_free_in_function(self): + # x is used only in the inlined listcomp; inner still reports USE. + st = symtable.symtable( + "def outer(x):\n" + " def inner():\n" + " return [x for y in ()]", + "?", "exec") + inner = find_block(find_block(st, "outer"), "inner") + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) + comp, = inner.get_children() + self.assertTrue(comp.lookup("x").is_free()) + self.assertTrue(comp.lookup("x").is_referenced()) + + def test_inlined_comprehension_use_of_nonlocal_in_function(self): + st = symtable.symtable( + "def outer():\n" + " x = 1\n" + " def inner():\n" + " nonlocal x\n" + " return [x for _ in ()]", + "?", "exec") + inner = find_block(find_block(st, "outer"), "inner") + self.assertTrue(inner.lookup("x").is_nonlocal()) + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) + comp, = inner.get_children() + self.assertTrue(comp.lookup("x").is_free()) + self.assertTrue(comp.lookup("x").is_referenced()) + + def test_inlined_comprehension_use_of_explicit_global_in_function(self): + st = symtable.symtable( + "def f():\n" + " global g\n" + " return [g for _ in ()]", + "?", "exec") + f = find_block(st, "f") + self.assertTrue(f.lookup("g").is_global()) + self.assertTrue(f.lookup("g").is_declared_global()) + self.assertFalse(f.lookup("g").is_referenced()) + comp, = f.get_children() + self.assertTrue(comp.lookup("g").is_global()) + self.assertTrue(comp.lookup("g").is_referenced()) + + def test_inlined_comprehension_nested_function_use_not_on_enclosing(self): + # The load of x is in the lambda's code object, not inner's. + st = symtable.symtable( + "def outer(x):\n" + " def inner():\n" + " return [(lambda: x) for y in ()]", + "?", "exec") + inner = find_block(find_block(st, "outer"), "inner") + self.assertTrue(inner.lookup("x").is_free()) + self.assertFalse(inner.lookup("x").is_referenced()) + comp, = inner.get_children() + self.assertTrue(comp.lookup("x").is_free()) + self.assertFalse(comp.lookup("x").is_referenced()) + lam, = comp.get_children() + self.assertTrue(lam.lookup("x").is_free()) + self.assertTrue(lam.lookup("x").is_referenced()) + + def test_inlined_comprehension_comp_cell_not_on_enclosing(self): + st = symtable.symtable( + "def f():\n" + " x = 1\n" + " return [(lambda: x) for x in [1]]", + "?", "exec") + f = find_block(st, "f") + self.assertTrue(f.lookup("x").is_cell()) + self.assertFalse(f.lookup("x").is_comp_cell()) + comp, = (c for c in f.get_children() + if c.get_type() is symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertTrue(comp.lookup("x").is_cell()) + self.assertTrue(comp.lookup("x").is_comp_cell()) + + def test_inlined_comprehension_use_of_enclosing_free_in_class(self): + st = symtable.symtable( + "def f():\n" + " y = 1\n" + " class C:\n" + " y = 2\n" + " vals = [(x, y) for x in range(2)]", + "?", "exec") + f = find_block(st, "f") + self.assertTrue(f.lookup("y").is_cell()) + C = find_block(f, "C") + self.assertTrue(C.lookup("y").is_local()) + self.assertFalse(C.lookup("y").is_free()) + self.assertTrue(C.lookup("y").is_free_class()) + comp, = C.get_children() + self.assertTrue(comp.lookup("y").is_free()) + self.assertTrue(comp.lookup("y").is_referenced()) + self.assertTrue(comp.lookup("x").is_local()) + + def test_inlined_comprehension_class_closure_names_are_free(self): + st = symtable.symtable( + "class C:\n" + " [__class__ for x in [1]]", + "?", "exec") + C = find_block(st, "C") + comp, = C.get_children() + self.assertTrue(comp.lookup("__class__").is_free()) + self.assertTrue(comp.lookup("__class__").is_referenced()) + with self.assertRaises(KeyError): + C.lookup("__class__") + + def test_inlined_nested_comprehension_class_iter_var(self): + st = symtable.symtable( + "class C:\n" + " x = 99\n" + " [[x for _ in (0,)] for x in (42,)]", + "?", "exec") + C = find_block(st, "C") + children = C.get_children() + self.assertEqual(len(children), 1) + inner = self.check_nested_inlined_listcomp( + children[0], ["x"], ["_", "x"], nested=False) + self.assertFalse(C.lookup("x").is_free()) + self.assertTrue(C.lookup("x").is_local()) + self.assertFalse(C.lookup("x").is_comp_cell()) + self.assertFalse(children[0].lookup("x").is_free()) + self.assertTrue(children[0].lookup("x").is_local()) + self.assertFalse(children[0].lookup("x").is_cell()) + self.assertFalse(children[0].lookup("x").is_comp_cell()) + self.assertTrue(children[0].lookup("x").is_referenced()) + self.assertFalse(inner.lookup("_").is_free()) + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) + + def test_inlined_nested_comprehension_iter_var_is_fast_local(self): + st = symtable.symtable( + "def f():\n" + " return [[x for _ in (0,)] for x in (42,)]", + "?", "exec") + f = find_block(st, "f") + with self.assertRaises(KeyError): + f.lookup("x") + outer, = f.get_children() + inner = self.check_nested_inlined_listcomp( + outer, ["x"], ["_", "x"], nested=True) + self.assertTrue(outer.lookup("x").is_local()) + self.assertFalse(outer.lookup("x").is_cell()) + self.assertFalse(outer.lookup("x").is_comp_cell()) + self.assertTrue(outer.lookup("x").is_referenced()) + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) + + def test_inlined_nested_mixed_comprehension_iter_var_is_referenced(self): + st = symtable.symtable( + "def f():\n" + " return [{x for _ in (0,)} for x in (42,)]", + "?", "exec") + f = find_block(st, "f") + with self.assertRaises(KeyError): + f.lookup("x") + outer, = f.get_children() + self.assertTrue(outer.lookup("x").is_local()) + self.assertTrue(outer.lookup("x").is_referenced()) + inner, = outer.get_children() + self.assertTrue(inner.lookup("x").is_free()) + self.assertTrue(inner.lookup("x").is_referenced()) + + def test_inlined_nested_comprehension_captured_iter_is_cell(self): + st = symtable.symtable( + "def f():\n" + " return [[(lambda: x) for _ in (0,)] for x in (1, 2)]", + "?", "exec") + f = find_block(st, "f") + with self.assertRaises(KeyError): + f.lookup("x") + outer, = (c for c in f.get_children() + if c.get_type() is symtable.SymbolTableType.INLINED_COMPREHENSION) + inner, = (c for c in outer.get_children() + if c.get_type() is symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertTrue(outer.lookup("x").is_cell()) + self.assertTrue(outer.lookup("x").is_comp_cell()) + self.assertTrue(inner.lookup("x").is_free()) + lam, = inner.get_children() + self.assertTrue(lam.lookup("x").is_free()) + + def test_inlined_sibling_nested_comprehensions(self): + st = symtable.symtable( + "def f(): [[y for y in x] for x in [1]]; [[w for w in z] for z in [2]]", + "?", "exec") + f = find_block(st, "f") + self.assertIs(f.get_type(), symtable.SymbolTableType.FUNCTION) + self.assertEqual(sorted(f.get_identifiers()), []) + children = f.get_children() + self.assertEqual(len(children), 2) + self.check_nested_inlined_listcomp( + children[0], ["x"], ["y"], nested=True) + self.check_nested_inlined_listcomp( + children[1], ["z"], ["w"], nested=True) + + def test_deeply_nested_inlined_comprehensions(self): + depth = 24 + source = "[" * depth + "0" + " for x in ()]" * depth + st = symtable.symtable(source, "?", "exec") + cur = st + for _ in range(depth): + children = cur.get_children() + self.assertEqual(len(children), 1) + cur = children[0] + self.assertIs(cur.get_type(), + symtable.SymbolTableType.INLINED_COMPREHENSION) + self.assertEqual(cur.get_children(), []) + def test__symtable_refleak(self): # Regression test for reference leak in PyUnicode_FSDecoder. # See https://github.com/python/cpython/issues/139748. diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 5358b6f5ee8fcd..2001b70739fd26 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -3538,6 +3538,23 @@ def src(depth): self._check_error(src(CO_MAXBLOCKS + 1), "too many statically nested blocks") + def test_invalid_starred_for_target_in_async_comprehension(self): + sources = [ + "async def f():\n {a async for b in d for *(b,) in e}", + "async def f():\n [a async for b in d for *(b,) in e]", + "async def f():\n {a: a async for b in d for *(b,) in e}", + ] + for src in sources: + with self.subTest(src=src): + self._check_error( + src, "starred assignment target must be in a list or tuple") + + def test_syntax_error_in_nested_inlined_async_comprehension(self): + self._check_error( + "async def f(it):\n" + " return [[f(a=1, a=2) for y in z] async for x in it]\n", + "keyword argument repeated") + @support.cpython_only def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" diff --git a/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst new file mode 100644 index 00000000000000..8dfdbeaf1856fc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-02-12-11-00.gh-issue-124697.sUbScp.rst @@ -0,0 +1,3 @@ +The :mod:`symtable` module now represents inlined list, set and dict +comprehensions (:pep:`709`) as their own symbol table entries of type +:attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d30affdbf62139..c01ac65dd4cc04 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1357,13 +1357,16 @@ _testinternalcapi_assemble_code_object_impl(PyObject *module, umd.u_cellvars = PyDict_GetItemString(metadata, "cellvars"); umd.u_freevars = PyDict_GetItemString(metadata, "freevars"); umd.u_fasthidden = PyDict_GetItemString(metadata, "fasthidden"); + if (umd.u_fasthidden == Py_None) { + umd.u_fasthidden = NULL; + } assert(PyDict_Check(umd.u_consts)); assert(PyDict_Check(umd.u_names)); assert(PyDict_Check(umd.u_varnames)); assert(PyDict_Check(umd.u_cellvars)); assert(PyDict_Check(umd.u_freevars)); - assert(PyDict_Check(umd.u_fasthidden)); + assert(umd.u_fasthidden == NULL || PySet_Check(umd.u_fasthidden)); umd.u_argcount = get_nonnegative_int_from_dict(metadata, "argcount"); umd.u_posonlyargcount = get_nonnegative_int_from_dict(metadata, "posonlyargcount"); diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index 7e20b5c7173ae5..3028158fc917e1 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -128,7 +128,6 @@ symtable_init_constants(PyObject *m) if (PyModule_AddIntMacro(m, DEF_BOUND) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_ANNOT) < 0) return -1; if (PyModule_AddIntMacro(m, DEF_COMP_ITER) < 0) return -1; - if (PyModule_AddIntMacro(m, DEF_COMP_CELL) < 0) return -1; if (PyModule_AddIntConstant(m, "TYPE_FUNCTION", FunctionBlock) < 0) return -1; @@ -144,6 +143,8 @@ symtable_init_constants(PyObject *m) return -1; if (PyModule_AddIntConstant(m, "TYPE_TYPE_VARIABLE", TypeVariableBlock) < 0) return -1; + if (PyModule_AddIntConstant(m, "TYPE_INLINED_COMPREHENSION", InlinedComprehensionBlock) < 0) + return -1; if (PyModule_AddIntMacro(m, LOCAL) < 0) return -1; if (PyModule_AddIntMacro(m, GLOBAL_EXPLICIT) < 0) return -1; diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c920e6cfc89c3b..a4cc14a6eaad45 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -94,6 +94,22 @@ framelocalsproxy_hasval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) return true; } +static int +framelocalsproxy_is_first_occurrence(PyObject *seen, PyObject *name) +{ + int found = PySet_Contains(seen, name); + if (found < 0) { + return -1; + } + if (found) { + return 0; + } + if (PySet_Add(seen, name) < 0) { + return -1; + } + return 1; +} + static int framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject *key, bool read, PyObject **value_ptr) { @@ -380,16 +396,28 @@ framelocalsproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) if (names == NULL) { return NULL; } + // An inlined comprehension cell can share a name with a free var. + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + Py_DECREF(names); + return NULL; + } for (int i = 0; i < co->co_nlocalsplus; i++) { if (framelocalsproxy_hasval(frame->f_frame, co, i)) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - if (PyList_Append(names, name) < 0) { - Py_DECREF(names); - return NULL; + int first = framelocalsproxy_is_first_occurrence(seen, name); + if (first < 0) { + goto error; + } + if (first) { + if (PyList_Append(names, name) < 0) { + goto error; + } } } } + Py_DECREF(seen); // Iterate through the extra locals if (frame->f_extra_locals) { @@ -408,6 +436,11 @@ framelocalsproxy_keys(PyObject *self, PyObject *Py_UNUSED(ignored)) } return names; + +error: + Py_DECREF(seen); + Py_DECREF(names); + return NULL; } static void @@ -589,18 +622,30 @@ framelocalsproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) if (values == NULL) { return NULL; } + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + Py_DECREF(values); + return NULL; + } for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { - if (PyList_Append(values, value) < 0) { - Py_DECREF(values); - Py_DECREF(value); - return NULL; + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + int first = framelocalsproxy_is_first_occurrence(seen, name); + if (first == 1) { + if (PyList_Append(values, value) < 0) { + Py_DECREF(value); + goto error; + } } Py_DECREF(value); + if (first < 0) { + goto error; + } } } + Py_DECREF(seen); // Iterate through the extra locals if (frame->f_extra_locals) { @@ -616,6 +661,11 @@ framelocalsproxy_values(PyObject *self, PyObject *Py_UNUSED(ignored)) } return values; + +error: + Py_DECREF(seen); + Py_DECREF(values); + return NULL; } static PyObject * @@ -627,22 +677,37 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) if (items == NULL) { return NULL; } + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + Py_DECREF(items); + return NULL; + } for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { - PyObject *pair = _PyTuple_FromPairSteal(Py_NewRef(name), value); - if (pair == NULL) { - goto error; + int first = framelocalsproxy_is_first_occurrence(seen, name); + if (first == 1) { + PyObject *pair = _PyTuple_FromPairSteal(Py_NewRef(name), value); + if (pair == NULL) { + goto error; + } + if (_PyList_AppendTakeRef((PyListObject *)items, pair) < 0) { + goto error; + } } - - if (_PyList_AppendTakeRef((PyListObject *)items, pair) < 0) { - goto error; + else { + Py_DECREF(value); + if (first < 0) { + goto error; + } } } } + Py_DECREF(seen); + seen = NULL; // Iterate through the extra locals if (frame->f_extra_locals) { @@ -664,6 +729,7 @@ framelocalsproxy_items(PyObject *self, PyObject *Py_UNUSED(ignored)) return items; error: + Py_XDECREF(seen); Py_DECREF(items); return NULL; } @@ -680,11 +746,24 @@ framelocalsproxy_length(PyObject *self) size += PyDict_Size(frame->f_extra_locals); } + PyObject *seen = PySet_New(NULL); + if (seen == NULL) { + return -1; + } for (int i = 0; i < co->co_nlocalsplus; i++) { if (framelocalsproxy_hasval(frame->f_frame, co, i)) { - size++; + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + int first = framelocalsproxy_is_first_occurrence(seen, name); + if (first < 0) { + Py_DECREF(seen); + return -1; + } + else if (first) { + size++; + } } } + Py_DECREF(seen); return size; } diff --git a/Python/assemble.c b/Python/assemble.c index 8b92042345f150..db9efff5e08ca9 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -534,13 +534,15 @@ compute_localsplus_info(_PyCompile_CodeUnitMetadata *umd, int nlocalsplus, _PyLocals_Kind kind = CO_FAST_LOCAL | argvarkinds[i].kind; - int has_key = PyDict_Contains(umd->u_fasthidden, k); - RETURN_IF_ERROR(has_key); - if (has_key) { - kind |= CO_FAST_HIDDEN; + if (umd->u_fasthidden != NULL) { + int hidden = PySet_Contains(umd->u_fasthidden, k); + RETURN_IF_ERROR(hidden); + if (hidden) { + kind |= CO_FAST_HIDDEN; + } } - has_key = PyDict_Contains(umd->u_cellvars, k); + int has_key = PyDict_Contains(umd->u_cellvars, k); RETURN_IF_ERROR(has_key); if (has_key) { kind |= CO_FAST_CELL; diff --git a/Python/codegen.c b/Python/codegen.c index c4b749a539efdd..0ae13e40d4a1ee 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3326,27 +3326,20 @@ codegen_nameop(compiler *c, location loc, return ERROR; } - int scope = _PyST_GetScope(SYMTABLE_ENTRY(c), mangled); - if (scope == -1) { - goto error; - } - _PyCompile_optype optype; Py_ssize_t arg = 0; - if (_PyCompile_ResolveNameop(c, mangled, scope, &optype, &arg) < 0) { + int scope = _PyCompile_ResolveNameop(c, mangled, &optype, &arg); + if (scope < 0) { Py_DECREF(mangled); return ERROR; } - /* XXX Leave assert here, but handle __doc__ and the like better */ - assert(scope || PyUnicode_READ_CHAR(name, 0) == '_'); - int op = 0; switch (optype) { case COMPILE_OP_DEREF: switch (ctx) { case Load: - if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock && !_PyCompile_IsInInlinedComp(c)) { + if (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) { op = LOAD_FROM_DICT_OR_DEREF; // First load the locals if (codegen_addop_noarg(INSTR_SEQUENCE(c), LOAD_LOCALS, loc) < 0) { @@ -3399,8 +3392,9 @@ codegen_nameop(compiler *c, location loc, case COMPILE_OP_NAME: switch (ctx) { case Load: - op = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock - && _PyCompile_IsInInlinedComp(c)) + /* LOAD_NAME in a class reads the class dict; inlined comps must not. */ + op = (SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS + && SYMTABLE_ENTRY(c)->ste_type == InlinedComprehensionBlock) ? LOAD_GLOBAL : LOAD_NAME; break; @@ -4949,9 +4943,6 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, PySTEntryObject *comp, _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (SYMTABLE_ENTRY(c)->ste_type == ClassBlock) && - !_PyCompile_IsInInlinedComp(c); - PySTEntryObject *outer = SYMTABLE_ENTRY(c); // iterate over names bound in the comprehension and ensure we isolate // them from the outer scope as needed PyObject *k, *v; @@ -4962,11 +4953,7 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, RETURN_IF_ERROR(symbol); long scope = SYMBOL_TO_SCOPE(symbol); - long outsymbol = _PyST_GetSymbol(outer, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { + if ((symbol & DEF_LOCAL) && !(symbol & DEF_NONLOCAL)) { // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack @@ -4981,15 +4968,17 @@ codegen_push_inlined_comprehension_locals(compiler *c, location loc, // comprehension and restore the original one after ADDOP_NAME(c, loc, LOAD_FAST_AND_CLEAR, k, varnames); if (scope == CELL) { - if (outsc == FREE) { - ADDOP_NAME(c, loc, MAKE_CELL, k, freevars); - } else { - ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); - } + ADDOP_NAME(c, loc, MAKE_CELL, k, cellvars); } if (PyList_Append(state->pushed_locals, k) < 0) { return ERROR; } + if (METADATA(c)->u_fasthidden != NULL) { + /* For Module/Class scopes, assemble needs to set CO_FAST_HIDDEN on these names */ + if (PySet_Add(METADATA(c)->u_fasthidden, k) < 0) { + return ERROR; + } + } } } if (state->pushed_locals) { @@ -5021,9 +5010,11 @@ push_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { RETURN_IF_ERROR( - _PyCompile_TweakInlinedComprehensionScopes(c, loc, comp, state)); - RETURN_IF_ERROR( - codegen_push_inlined_comprehension_locals(c, loc, comp, state)); + _PyCompile_EnterInlinedComprehensionScope(c, comp, state)); + if (codegen_push_inlined_comprehension_locals(c, loc, comp, state) < 0){ + _PyCompile_ExitInlinedComprehensionScope(c, state); + return ERROR; + } return SUCCESS; } @@ -5080,9 +5071,9 @@ static int pop_inlined_comprehension_state(compiler *c, location loc, _PyCompile_InlinedComprehensionState *state) { - RETURN_IF_ERROR(codegen_pop_inlined_comprehension_locals(c, loc, state)); - RETURN_IF_ERROR(_PyCompile_RevertInlinedComprehensionScopes(c, loc, state)); - return SUCCESS; + int result = codegen_pop_inlined_comprehension_locals(c, loc, state); + RETURN_IF_ERROR(_PyCompile_ExitInlinedComprehensionScope(c, state)); + return result; } static int @@ -5123,13 +5114,13 @@ codegen_comprehension(compiler *c, expr_ty e, int type, expr_ty val, bool avoid_creation) { PyCodeObject *co = NULL; - _PyCompile_InlinedComprehensionState inline_state = {NULL, NULL, NULL, NO_LABEL}; + _PyCompile_InlinedComprehensionState inline_state = {NULL, NO_LABEL, NULL}; comprehension_ty outermost; PySTEntryObject *entry = _PySymtable_Lookup(SYMTABLE(c), (void *)e); if (entry == NULL) { goto error; } - int is_inlined = entry->ste_comp_inlined; + int is_inlined = (entry->ste_type == InlinedComprehensionBlock); int is_async_comprehension = entry->ste_coroutine; location loc = LOC(e); @@ -5138,7 +5129,7 @@ codegen_comprehension(compiler *c, expr_ty e, int type, IterStackPosition iter_state; if (is_inlined) { VISIT(c, expr, outermost->iter); - if (push_inlined_comprehension_state(c, loc, entry, &inline_state)) { + if (push_inlined_comprehension_state(c, loc, entry, &inline_state) < 0) { goto error; } iter_state = ITERABLE_ON_STACK; @@ -5191,8 +5182,8 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } if (is_inlined) { - if (pop_inlined_comprehension_state(c, loc, &inline_state)) { - goto error; + if (pop_inlined_comprehension_state(c, loc, &inline_state) < 0) { + goto error_in_scope; } return SUCCESS; } @@ -5232,15 +5223,18 @@ codegen_comprehension(compiler *c, expr_ty e, int type, return SUCCESS; error_in_scope: - if (!is_inlined) { + if (is_inlined) { + if (inline_state.saved_ste != NULL) { + _PyCompile_ExitInlinedComprehensionScope(c, &inline_state); + } + } + else { _PyCompile_ExitScope(c); } error: Py_XDECREF(co); Py_XDECREF(entry); Py_XDECREF(inline_state.pushed_locals); - Py_XDECREF(inline_state.temp_symbols); - Py_XDECREF(inline_state.fast_hidden); return ERROR; } diff --git a/Python/compile.c b/Python/compile.c index f3852041bce69c..ee29f7a9a5d589 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -68,7 +68,6 @@ struct compiler_unit { instr_sequence *u_stashed_instr_sequence; /* temporarily stashed parent instruction sequence */ int u_nfblocks; - int u_in_inlined_comp; int u_in_conditional_block; _PyCompile_FBlockInfo u_fblock[CO_MAXBLOCKS]; @@ -596,6 +595,75 @@ dictbytype(PyObject *src, int scope_type, int flag, Py_ssize_t offset) return dest; } +static int +add_cell_names_from_symbols(PyObject *symbols, PyObject *names) +{ + Py_ssize_t pos = 0; + PyObject *k, *v; + while (PyDict_Next(symbols, &pos, &k, &v)) { + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { + return ERROR; + } + if (SYMBOL_TO_SCOPE(flags) == CELL) { + if (PySet_Add(names, k) < 0) { + return ERROR; + } + } + } + return SUCCESS; +} + +static int +add_inlined_comprehension_cell_names(PySTEntryObject *ste, PyObject *names) +{ + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(ste->ste_children); i++) { + PySTEntryObject *child = + (PySTEntryObject *)PyList_GET_ITEM(ste->ste_children, i); + if (child->ste_type != InlinedComprehensionBlock) { + continue; + } + if (add_cell_names_from_symbols(child->ste_symbols, names) < 0) { + return ERROR; + } + if (add_inlined_comprehension_cell_names(child, names) < 0) { + return ERROR; + } + } + return SUCCESS; +} + +/* Cells of the shared unit: this table's CELL names, plus cells that live + * only on inlined comprehension children. */ +static PyObject * +compiler_cellvars(PySTEntryObject *ste) +{ + PyObject *names = PySet_New(NULL); + if (names == NULL) { + return NULL; + } + if (add_cell_names_from_symbols(ste->ste_symbols, names) < 0) { + Py_DECREF(names); + return NULL; + } + if (add_inlined_comprehension_cell_names(ste, names) < 0) { + Py_DECREF(names); + return NULL; + } + PyObject *sorted = PySequence_List(names); + Py_DECREF(names); + if (sorted == NULL) { + return NULL; + } + if (PyList_Sort(sorted) < 0) { + Py_DECREF(sorted); + return NULL; + } + PyObject *cellvars = list2dict(sorted); + Py_DECREF(sorted); + return cellvars; +} + int _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, void *key, int lineno, PyObject *private, @@ -627,7 +695,7 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, compiler_unit_free(u); return ERROR; } - u->u_metadata.u_cellvars = dictbytype(u->u_ste->ste_symbols, CELL, DEF_COMP_CELL, 0); + u->u_metadata.u_cellvars = compiler_cellvars(u->u_ste); if (!u->u_metadata.u_cellvars) { compiler_unit_free(u); return ERROR; @@ -670,14 +738,18 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, return ERROR; } - u->u_metadata.u_fasthidden = PyDict_New(); - if (!u->u_metadata.u_fasthidden) { - compiler_unit_free(u); - return ERROR; + if (scope_type == COMPILE_SCOPE_MODULE || scope_type == COMPILE_SCOPE_CLASS) { + u->u_metadata.u_fasthidden = PySet_New(NULL); + if (!u->u_metadata.u_fasthidden) { + compiler_unit_free(u); + return ERROR; + } + } + else { + u->u_metadata.u_fasthidden = NULL; } u->u_nfblocks = 0; - u->u_in_inlined_comp = 0; u->u_metadata.u_firstlineno = lineno; u->u_metadata.u_consts = PyDict_New(); if (!u->u_metadata.u_consts) { @@ -907,17 +979,46 @@ compiler_mod(compiler *c, mod_ty mod) return co; } +/* Inlined comprehensions are compiled in the enclosing unit. If a name is + * FREE in the comprehension, or is absent from its table (scope 0), resolve + * it in enclosing tables until it is bound. Stop if the next table is a class: + * nested scopes (including inlined comprehensions) do not see class locals, so + * the name stays FREE. __class__ and friends are not allowed to be free + * through a class; treat those loads as implicit globals. + * + * Names with no entry (scope 0) include loads synthesized by codegen, such as + * the implicit receiver for zero-arg super(). */ +static int +compiler_resolve_inlined_free(PySTEntryObject **ste, PyObject *name) +{ + int scope = _PyST_GetScope(*ste, name); + RETURN_IF_ERROR(scope); + while ((*ste)->ste_type == InlinedComprehensionBlock && + (scope == FREE || scope == 0)) { + PySTEntryObject *parent = (*ste)->ste_parent; + assert(parent != NULL); + if (parent->ste_type == ClassBlock) { + if (_PyST_IsClassClosureName(name)) { + return GLOBAL_IMPLICIT; + } + break; + } + *ste = parent; + scope = _PyST_GetScope(*ste, name); + RETURN_IF_ERROR(scope); + } + return scope; +} + int _PyCompile_GetRefType(compiler *c, PyObject *name) { - if (c->u->u_scope_type == COMPILE_SCOPE_CLASS && - (_PyUnicode_EqualToASCIIString(name, "__class__") || - _PyUnicode_EqualToASCIIString(name, "__classdict__") || - _PyUnicode_EqualToASCIIString(name, "__conditional_annotations__"))) { + if (c->u->u_scope_type == COMPILE_SCOPE_CLASS && _PyST_IsClassClosureName(name)) { return CELL; } PySTEntryObject *ste = c->u->u_ste; - int scope = _PyST_GetScope(ste, name); + int scope = compiler_resolve_inlined_free(&ste, name); + RETURN_IF_ERROR(scope); if (scope == 0) { PyErr_Format(PyExc_SystemError, "_PyST_GetScope(name=%R) failed: " @@ -1007,13 +1108,18 @@ _PyCompile_StaticAttributesAsTuple(compiler *c) } int -_PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, +_PyCompile_ResolveNameop(compiler *c, PyObject *mangled, _PyCompile_optype *optype, Py_ssize_t *arg) { PyObject *dict = c->u->u_metadata.u_names; *optype = COMPILE_OP_NAME; - assert(scope >= 0); + PySTEntryObject *ste = c->u->u_ste; + assert(ste != NULL); + + int scope = compiler_resolve_inlined_free(&ste, mangled); + RETURN_IF_ERROR(scope); + switch (scope) { case FREE: dict = c->u->u_metadata.u_freevars; @@ -1024,24 +1130,24 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, *optype = COMPILE_OP_DEREF; break; case LOCAL: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + /* Inlined comprehensions isolate their locals as FAST, even when + * nested in class or module scope. */ + if (_PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock) { *optype = COMPILE_OP_FAST; } - else { - PyObject *item; - RETURN_IF_ERROR(PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, mangled, - &item)); - if (item == Py_True) { - *optype = COMPILE_OP_FAST; - } - Py_XDECREF(item); - } break; - case GLOBAL_IMPLICIT: - if (_PyST_IsFunctionLike(c->u->u_ste)) { + case GLOBAL_IMPLICIT: { + /* Opcode depends on the enclosing non-inlined scope. */ + PySTEntryObject *enclosing = ste; + while (enclosing->ste_parent != NULL) { + assert(enclosing->ste_type == InlinedComprehensionBlock); + enclosing = enclosing->ste_parent; + } + if (_PyST_IsFunctionLike(enclosing)) { *optype = COMPILE_OP_GLOBAL; } break; + } case GLOBAL_EXPLICIT: *optype = COMPILE_OP_GLOBAL; break; @@ -1049,128 +1155,33 @@ _PyCompile_ResolveNameop(compiler *c, PyObject *mangled, int scope, /* scope can be 0 */ break; } + /* XXX Handle __doc__ and the like better */ + assert(scope || PyUnicode_READ_CHAR(mangled, 0) == '_'); if (*optype != COMPILE_OP_FAST) { *arg = _PyCompile_DictAddObj(dict, mangled); RETURN_IF_ERROR(*arg); } - return SUCCESS; + return scope; } int -_PyCompile_TweakInlinedComprehensionScopes(compiler *c, location loc, - PySTEntryObject *entry, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_EnterInlinedComprehensionScope(compiler *c, PySTEntryObject *entry, + _PyCompile_InlinedComprehensionState *state) { - int in_class_block = (c->u->u_ste->ste_type == ClassBlock) && !c->u->u_in_inlined_comp; - c->u->u_in_inlined_comp++; - - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { - long symbol = PyLong_AsLong(v); - assert(symbol >= 0 || PyErr_Occurred()); - RETURN_IF_ERROR(symbol); - long scope = SYMBOL_TO_SCOPE(symbol); - - long outsymbol = _PyST_GetSymbol(c->u->u_ste, k); - RETURN_IF_ERROR(outsymbol); - long outsc = SYMBOL_TO_SCOPE(outsymbol); - - // If a name has different scope inside than outside the comprehension, - // we need to temporarily handle it with the right scope while - // compiling the comprehension. If it's free in the comprehension - // scope, no special handling; it should be handled the same as the - // enclosing scope. (If it's free in outer scope and cell in inner - // scope, we can't treat it as both cell and free in the same function, - // but treating it as free throughout is fine; it's *_DEREF - // either way.) - if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) - || in_class_block) { - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - return ERROR; - } - PyObject *outv = PyLong_FromLong(outsymbol); - if (outv == NULL) { - return ERROR; - } - int res = PyDict_SetItem(state->temp_symbols, k, outv); - Py_DECREF(outv); - RETURN_IF_ERROR(res); - } - // locals handling for names bound in comprehension (DEF_LOCAL | - // DEF_NONLOCAL occurs in assignment expression to nonlocal) - if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { - if (!_PyST_IsFunctionLike(c->u->u_ste)) { - // non-function scope: override this name to use fast locals - PyObject *orig; - if (PyDict_GetItemRef(c->u->u_metadata.u_fasthidden, k, &orig) < 0) { - return ERROR; - } - assert(orig == NULL || orig == Py_True || orig == Py_False); - if (orig != Py_True) { - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_True) < 0) { - Py_XDECREF(orig); - return ERROR; - } - if (state->fast_hidden == NULL) { - state->fast_hidden = PySet_New(NULL); - if (state->fast_hidden == NULL) { - Py_XDECREF(orig); - return ERROR; - } - } - if (PySet_Add(state->fast_hidden, k) < 0) { - Py_XDECREF(orig); - return ERROR; - } - } - Py_XDECREF(orig); - } - } - } + assert(state->saved_ste == NULL); + state->saved_ste = c->u->u_ste; + c->u->u_ste = (PySTEntryObject *)Py_NewRef(entry); return SUCCESS; } int -_PyCompile_RevertInlinedComprehensionScopes(compiler *c, location loc, - _PyCompile_InlinedComprehensionState *state) +_PyCompile_ExitInlinedComprehensionScope(compiler *c, + _PyCompile_InlinedComprehensionState *state) { - c->u->u_in_inlined_comp--; - if (state->temp_symbols) { - PyObject *k, *v; - Py_ssize_t pos = 0; - while (PyDict_Next(state->temp_symbols, &pos, &k, &v)) { - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v)) { - return ERROR; - } - } - Py_CLEAR(state->temp_symbols); - } - if (state->fast_hidden) { - while (PySet_Size(state->fast_hidden) > 0) { - PyObject *k = PySet_Pop(state->fast_hidden); - if (k == NULL) { - return ERROR; - } - // we set to False instead of clearing, so we can track which names - // were temporarily fast-locals and should use CO_FAST_HIDDEN - if (PyDict_SetItem(c->u->u_metadata.u_fasthidden, k, Py_False)) { - Py_DECREF(k); - return ERROR; - } - Py_DECREF(k); - } - Py_CLEAR(state->fast_hidden); - } + assert(state->saved_ste != NULL); + Py_DECREF(c->u->u_ste); + c->u->u_ste = state->saved_ste; + state->saved_ste = NULL; return SUCCESS; } @@ -1362,12 +1373,6 @@ _PyCompile_ScopeType(compiler *c) return c->u->u_scope_type; } -int -_PyCompile_IsInInlinedComp(compiler *c) -{ - return c->u->u_in_inlined_comp; -} - PyObject * _PyCompile_Qualname(compiler *c) { @@ -1513,10 +1518,7 @@ _PyCompile_OptimizeAndAssemble(compiler *c, int addNone) PyObject *filename = c->c_filename; int code_flags = compute_code_flags(c); - if (code_flags < 0) { - return NULL; - } - + assert(code_flags >= 0); if (_PyCodegen_AddReturnAtEnd(c, addNone) < 0) { return NULL; } diff --git a/Python/symtable.c b/Python/symtable.c index 8da04b40e8ad14..b97778ada20c04 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -5,8 +5,10 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_symtable.h" // PySTEntryObject #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString +#include "setobject.h" #include // offsetof() +#include // Set this to 1 to dump all symtables to stdout for debugging @@ -89,6 +91,12 @@ #define IS_ASYNC_DEF(st) ((st)->st_cur->ste_type == FunctionBlock && (st)->st_cur->ste_coroutine) +static int +ste_uses_fast_locals(PySTEntryObject *ste) +{ + return _PyST_IsFunctionLike(ste) || ste->ste_type == InlinedComprehensionBlock; +} + static PySTEntryObject * ste_new(struct symtable *st, identifier name, _Py_block_ty block, void *key, _Py_SourceLocation loc) @@ -128,14 +136,14 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, if (st->st_cur != NULL && (st->st_cur->ste_nested || - _PyST_IsFunctionLike(st->st_cur))) + ste_uses_fast_locals(st->st_cur))) ste->ste_nested = 1; ste->ste_generator = 0; ste->ste_coroutine = 0; ste->ste_comprehension = NoComprehension; ste->ste_returns_value = 0; ste->ste_needs_class_closure = 0; - ste->ste_comp_inlined = 0; + ste->ste_parent = (block == InlinedComprehensionBlock) ? st->st_cur : NULL; ste->ste_comp_iter_target = 0; ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; @@ -295,6 +303,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) case TypeVariableBlock: blocktype = "TypeVariableBlock"; break; case TypeAliasBlock: blocktype = "TypeAliasBlock"; break; case TypeParametersBlock: blocktype = "TypeParametersBlock"; break; + case InlinedComprehensionBlock: blocktype = "InlinedComprehensionBlock"; break; } const char *comptype = ""; switch (ste->ste_comprehension) { @@ -308,7 +317,7 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ( "%U=== Symtable for %U ===\n" "%U%s%s\n" - "%U%s%s%s%s%s%s%s%s%s%s%s\n" + "%U%s%s%s%s%s%s%s%s%s%s\n" "%Ulineno: %d col_offset: %d\n" "%U--- Symbols ---\n" ), @@ -326,7 +335,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) ste->ste_returns_value ? " returns_value" : "", ste->ste_needs_class_closure ? " needs_class_closure" : "", ste->ste_needs_classdict ? " needs_classdict" : "", - ste->ste_comp_inlined ? " comp_inlined" : "", ste->ste_comp_iter_target ? " comp_iter_target" : "", ste->ste_can_see_class_scope ? " can_see_class_scope" : "", prefix, @@ -353,7 +361,6 @@ static void _dump_symtable(PySTEntryObject* ste, PyObject* prefix) if (flags & DEF_ANNOT) printf(" DEF_ANNOT"); if (flags & DEF_COMP_ITER) printf(" DEF_COMP_ITER"); if (flags & DEF_TYPE_PARAM) printf(" DEF_TYPE_PARAM"); - if (flags & DEF_COMP_CELL) printf(" DEF_COMP_CELL"); switch (scope) { case LOCAL: printf(" LOCAL"); break; case GLOBAL_EXPLICIT: printf(" GLOBAL_EXPLICIT"); break; @@ -541,7 +548,7 @@ _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) if (PyDict_GetItemRef(ste->ste_symbols, name, &v) < 0) { return -1; } - if (!v) { + if (v == NULL) { return 0; } long symbol = PyLong_AsLong(v); @@ -575,6 +582,14 @@ _PyST_IsFunctionLike(PySTEntryObject *ste) || ste->ste_type == TypeParametersBlock; } +int +_PyST_IsClassClosureName(PyObject *name) +{ + return _PyUnicode_EqualToASCIIString(name, "__class__") + || _PyUnicode_EqualToASCIIString(name, "__classdict__") + || _PyUnicode_EqualToASCIIString(name, "__conditional_annotations__"); +} + static int error_at_directive(PySTEntryObject *ste, PyObject *name) { @@ -784,12 +799,22 @@ analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags, return 1; } +/* See InternalDocs/inlined_comprehensions.md. */ + static int is_free_in_any_child(PySTEntryObject *entry, PyObject *key) { for (Py_ssize_t i = 0; i < PyList_GET_SIZE(entry->ste_children); i++) { PySTEntryObject *child_ste = (PySTEntryObject *)PyList_GET_ITEM( entry->ste_children, i); + if (child_ste->ste_type == InlinedComprehensionBlock) { + /* this is the same scope, so we check its children */ + int nested = is_free_in_any_child(child_ste, key); + if (nested != 0) { + return nested; + } + continue; + } long scope = _PyST_GetScope(child_ste, key); if (scope < 0) { return -1; @@ -802,109 +827,115 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) } static int -inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, - PyObject *scopes, PyObject *comp_free, - PyObject *inlined_cells) +symtable_add_flag(PyObject *dict, PyObject *name, int flag) +{ + PyObject *o = PyDict_GetItemWithError(dict, name); + long val; + if (o != NULL) { + val = PyLong_AsLong(o); + if (val == -1 && PyErr_Occurred()) { + return 0; + } + val |= flag; + } + else if (PyErr_Occurred()) { + return 0; + } + else { + val = flag; + } + o = PyLong_FromLong(val); + if (o == NULL) { + return 0; + } + int rc = PyDict_SetItem(dict, name, o); + Py_DECREF(o); + return rc >= 0; +} + +static int +finalize_inlined_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, + PyObject *comp_free, PyObject *outer_newfree, + PyObject *inlined_cells) { PyObject *k, *v; Py_ssize_t pos = 0; - int remove_dunder_class = 0; - int remove_dunder_classdict = 0; - int remove_dunder_cond_annotations = 0; + + assert(comp->ste_type == InlinedComprehensionBlock); + assert(comp->ste_parent != NULL); while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { - // skip comprehension parameter long comp_flags = PyLong_AsLong(v); if (comp_flags == -1 && PyErr_Occurred()) { - return 0; - } - if (comp_flags & DEF_PARAM) { - assert(_PyUnicode_EqualToASCIIString(k, ".0")); - continue; + goto error; } int scope = SYMBOL_TO_SCOPE(comp_flags); - int only_flags = comp_flags & ((1 << SCOPE_OFFSET) - 1); - if (scope == CELL || only_flags & DEF_COMP_CELL) { + if (scope == CELL) { if (PySet_Add(inlined_cells, k) < 0) { - return 0; + goto error; } } PyObject *existing = PyDict_GetItemWithError(ste->ste_symbols, k); if (existing == NULL && PyErr_Occurred()) { - return 0; + goto error; } - // __class__, __classdict__ and __conditional_annotations__ are - // not allowed to be free through a class scope (see - // drop_class_free) unless children scopes need it + // These names are not allowed to be free through a class (see + // drop_class_free) unless a nested child needs them. Keep FREE + // on this table; compile treats the load as a global. if (scope == FREE && ste->ste_type == ClassBlock && - (_PyUnicode_EqualToASCIIString(k, "__class__") || - _PyUnicode_EqualToASCIIString(k, "__classdict__") || - _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { - scope = GLOBAL_IMPLICIT; + _PyST_IsClassClosureName(k)) { int child_needs_free = is_free_in_any_child(comp, k); if (child_needs_free < 0) { - return 0; + goto error; } if (!child_needs_free) { if (PySet_Discard(comp_free, k) < 0) { - return 0; + goto error; } } - if (_PyUnicode_EqualToASCIIString(k, "__class__")) { - remove_dunder_class = 1; - } - else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { - remove_dunder_cond_annotations = 1; - } - else { - remove_dunder_classdict = 1; - } - } - if (!existing) { - // name does not exist in scope, copy from comprehension - assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - PyObject *v_flags = PyLong_FromLong(only_flags); - if (v_flags == NULL) { - return 0; - } - int ok = PyDict_SetItem(ste->ste_symbols, k, v_flags); - Py_DECREF(v_flags); - if (ok < 0) { - return 0; - } - SET_SCOPE(scopes, k, scope); + continue; } - else { + int is_def_bound = 0; + if (existing) { long flags = PyLong_AsLong(existing); if (flags == -1 && PyErr_Occurred()) { - return 0; + goto error; } - if ((flags & DEF_BOUND) && ste->ste_type != ClassBlock) { - // free vars in comprehension that are locals in outer scope can - // now simply be locals, unless they are free in comp children, - // or if the outer scope is a class block - int ok = is_free_in_any_child(comp, k); - if (ok < 0) { - return 0; + is_def_bound = flags & DEF_BOUND; + } + // Loads here are in the enclosing compilation unit. + if (scope == FREE && (comp_flags & USE) && + !symtable_add_flag(ste->ste_symbols, k, USE)) { + goto error; + } + if (is_def_bound && ste->ste_type != ClassBlock) { + // free vars in comprehension that are locals in outer scope can + // now simply be locals, unless they are free in comp children, + // needed as cells by sibling nested scopes, or if the outer + // scope is a class block + int ok = is_free_in_any_child(comp, k); + if (ok < 0) { + goto error; + } + if (!ok) { + int in_newfree = PySet_Contains(outer_newfree, k); + if (in_newfree < 0) { + goto error; } - if (!ok) { + if (!in_newfree) { if (PySet_Discard(comp_free, k) < 0) { - return 0; + goto error; } } } } - } - if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { - return 0; - } - if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { - return 0; - } - if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { - return 0; + else if (!existing) { + assert(scope != FREE || PySet_Contains(comp_free, k) == 1); + } } return 1; +error: + return 0; } #undef SET_SCOPE @@ -992,7 +1023,7 @@ drop_class_free(PySTEntryObject *ste, PyObject *free) static int update_symbols(PyObject *symbols, PyObject *scopes, PyObject *bound, PyObject *free, - PyObject *inlined_cells, int classflag) + int classflag) { PyObject *name = NULL, *itr = NULL; PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL; @@ -1004,13 +1035,6 @@ update_symbols(PyObject *symbols, PyObject *scopes, if (flags == -1 && PyErr_Occurred()) { return 0; } - int contains = PySet_Contains(inlined_cells, name); - if (contains < 0) { - return 0; - } - if (contains) { - flags |= DEF_COMP_CELL; - } if (PyDict_GetItemRef(scopes, name, &v_scope) < 0) { return 0; } @@ -1158,7 +1182,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, ClassBlocks, the bound and global names are initialized before analyzing names, because class bindings aren't visible in methods. For other blocks, they are initialized - after names are analyzed. + after this block's declarations are recorded. */ /* TODO(jhylton): Package these dicts in a struct so that we @@ -1176,7 +1200,6 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, inlined_cells = PySet_New(NULL); if (!inlined_cells) goto error; - /* Class namespace has no effect on names visible in nested functions, so populate the global and bound sets to be passed to child blocks before analyzing @@ -1197,11 +1220,17 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } + /* Record bindings and global/nonlocal declarations first so child + blocks see this scope's locals. Uses are classified after children + so inlined comprehensions can add USE flags to this table. */ while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { long flags = PyLong_AsLong(v); if (flags == -1 && PyErr_Occurred()) { goto error; } + if (!(flags & (DEF_GLOBAL | DEF_NONLOCAL | DEF_BOUND))) { + continue; + } if (!analyze_name(ste, scopes, name, flags, bound, local, free, global, type_params, class_entry)) goto error; @@ -1210,7 +1239,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, /* Populate global and bound sets to be passed to children. */ if (ste->ste_type != ClassBlock) { /* Add function locals to bound set */ - if (_PyST_IsFunctionLike(ste)) { + if (ste_uses_fast_locals(ste)) { temp = PyNumber_InPlaceOr(newbound, local); if (!temp) goto error; @@ -1262,24 +1291,20 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } } - // we inline all non-generator-expression comprehensions, - // except those in annotation scopes that are nested in classes - int inline_comp = - entry->ste_comprehension && - !entry->ste_generator && - !ste->ste_can_see_class_scope; - + // Finalize inlined children before classifying this block's uses + // and analyze_cells, so their loads are USE here and do not force + // a cell unless a real nested unit needs one. if (!analyze_child_block(entry, newbound, newfree, newglobal, type_params, new_class_entry, &child_free)) { goto error; } - if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) { + if (entry->ste_type == InlinedComprehensionBlock) { + if (!finalize_inlined_comprehension(ste, entry, child_free, newfree, + inlined_cells)) { Py_DECREF(child_free); goto error; } - entry->ste_comp_inlined = 1; } temp = PyNumber_InPlaceOr(newfree, child_free); Py_DECREF(child_free); @@ -1288,30 +1313,36 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_DECREF(temp); } - /* Splice children of inlined comprehensions into our children list */ - for (i = PyList_GET_SIZE(ste->ste_children) - 1; i >= 0; --i) { - PyObject* c = PyList_GET_ITEM(ste->ste_children, i); - PySTEntryObject* entry; - assert(c && PySTEntry_Check(c)); - entry = (PySTEntryObject*)c; - if (entry->ste_comp_inlined && - PyList_SetSlice(ste->ste_children, i, i + 1, - entry->ste_children) < 0) - { + /* Complete the classification. */ + pos = 0; + while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) { + long flags = PyLong_AsLong(v); + if (flags == -1 && PyErr_Occurred()) { goto error; } + int contains = PyDict_Contains(scopes, name); + if (contains < 0) { + goto error; + } + if (contains) { + continue; + } + if (!analyze_name(ste, scopes, name, flags, + bound, local, free, global, type_params, class_entry)) + goto error; } /* Check if any local variables must be converted to cell variables */ - if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells)) + if (ste_uses_fast_locals(ste) && !analyze_cells(scopes, newfree, inlined_cells)) { goto error; - else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) + } + else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) { goto error; + } /* Records the results of the analysis in the symbol table entry */ - if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, inlined_cells, + if (!update_symbols(ste->ste_symbols, scopes, bound, newfree, (ste->ste_type == ClassBlock) || ste->ste_can_see_class_scope)) goto error; - temp = PyNumber_InPlaceOr(free, newfree); if (!temp) goto error; @@ -1515,14 +1546,12 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s _Py_SourceLocation loc) { PyObject *o; - PyObject *dict; - long val; + long val = 0; PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name); if (!mangled) return 0; - dict = ste->ste_symbols; - if ((o = PyDict_GetItemWithError(dict, mangled))) { + if ((o = PyDict_GetItemWithError(ste->ste_symbols, mangled))) { val = PyLong_AsLong(o); if (val == -1 && PyErr_Occurred()) { goto error; @@ -1538,62 +1567,40 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s SET_ERROR_LOCATION(st->st_filename, loc); goto error; } - val |= flag; } else if (PyErr_Occurred()) { goto error; } - else { - val = flag; - } + + int to_add = flag; if (ste->ste_comp_iter_target) { /* This name is an iteration variable in a comprehension, * so check for a binding conflict with any named expressions. * Otherwise, mark it as an iteration variable so subsequent * named expressions can check for conflicts. */ - if (val & (DEF_GLOBAL | DEF_NONLOCAL)) { + if ((val | flag) & (DEF_GLOBAL | DEF_NONLOCAL)) { PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name); SET_ERROR_LOCATION(st->st_filename, loc); goto error; } - val |= DEF_COMP_ITER; + to_add |= DEF_COMP_ITER; } - o = PyLong_FromLong(val); - if (o == NULL) - goto error; - if (PyDict_SetItem(dict, mangled, o) < 0) { - Py_DECREF(o); + if (!symtable_add_flag(ste->ste_symbols, mangled, to_add)) { goto error; } - Py_DECREF(o); if (flag & DEF_PARAM) { if (PyList_Append(ste->ste_varnames, mangled) < 0) goto error; - } else if (flag & DEF_GLOBAL) { + } + else if (flag & DEF_GLOBAL) { /* XXX need to update DEF_GLOBAL for other flags too; perhaps only DEF_FREE_GLOBAL */ - val = 0; - if ((o = PyDict_GetItemWithError(st->st_global, mangled))) { - val = PyLong_AsLong(o); - if (val == -1 && PyErr_Occurred()) { - goto error; - } - } - else if (PyErr_Occurred()) { + if (!symtable_add_flag(st->st_global, mangled, flag)) { goto error; } - val |= flag; - o = PyLong_FromLong(val); - if (o == NULL) - goto error; - if (PyDict_SetItem(st->st_global, mangled, o) < 0) { - Py_DECREF(o); - goto error; - } - Py_DECREF(o); } Py_DECREF(mangled); return 1; @@ -2582,7 +2589,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) return 0; } if (!allows_top_level_await(st)) { - if (!_PyST_IsFunctionLike(st->st_cur)) { + if (!ste_uses_fast_locals(st->st_cur)) { PyErr_SetString(PyExc_SyntaxError, "'await' outside function"); SET_ERROR_LOCATION(st->st_filename, LOCATION(e)); @@ -2660,7 +2667,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e) } /* Special-case super: it counts as a use of __class__ */ if (e->v.Name.ctx == Load && - _PyST_IsFunctionLike(st->st_cur) && + ste_uses_fast_locals(st->st_cur) && _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) { if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e))) return 0; @@ -3103,9 +3110,16 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_comp_iter_expr++; VISIT(st, expr, outermost->iter); st->st_cur->ste_comp_iter_expr--; + + /* Non-generator comprehensions are inlined into the enclosing compilation + * unit (including generator expressions), except in annotation scopes + * that can see a class. */ + int will_inline = !is_generator && !st->st_cur->ste_can_see_class_scope; + _Py_block_ty block = will_inline ? InlinedComprehensionBlock : FunctionBlock; + /* Create comprehension scope for the rest */ if (!scope_name || - !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e, LOCATION(e))) { + !symtable_enter_block(st, scope_name, block, (void *)e, LOCATION(e))) { return 0; } switch(e->kind) { @@ -3126,8 +3140,8 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, st->st_cur->ste_coroutine = 1; } - /* Outermost iter is received as an argument */ - if (!symtable_implicit_arg(st, 0)) { + /* Outermost iter is received as an argument for non-inlined comps */ + if (!will_inline && !symtable_implicit_arg(st, 0)) { symtable_exit_block(st); return 0; } From b54ae4cdfce4b84b22889b5c5b9ed0ab9bcb3702 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Sep 2026 22:33:10 +0300 Subject: [PATCH 4/7] gh-103089: Colorize only the beginning of very long lines in IDLE (#157647) Adding a tag to a Tk text line takes time proportional to the number of tags already in the line, so a line with hundreds of thousands of tokens took hours to colorize. Now only the first 2000 characters of a line are colorized, tag positions are relative to the line start, and tags are added from the end to the start. --- Lib/idlelib/colorizer.py | 36 ++++++++++++++++--- Lib/idlelib/idle_test/test_colorizer.py | 12 +++++++ ...9-17-02-00-00.gh-issue-103089.longline.rst | 2 ++ 3 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-17-02-00-00.gh-issue-103089.longline.rst diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index 5f18a22d2f0507..dbe0723cfdbf96 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -8,6 +8,11 @@ DEBUG = False +# Adding a tag to a line takes time proportional to the number of tags +# already in the line, so only the beginning of a line is colorized; +# the rest is usually not visible anyway (gh-103089). +MAX_COLORIZED_LINE = 2000 + def any(name, alternates): "Return a named group pattern matching list of alternates." @@ -350,16 +355,39 @@ def _add_tags_in_section(self, chars, head): `chars` is a string with the text to parse and to which highlighting is to be applied. - `head` is the index in the text widget where the text is found. + `head` is the index in the text widget where the text is found. """ - for m in self.prog.finditer(chars): + # Positions are relative to the start of the current line, so that + # Tk does not resolve them through the previous lines. + line = int(head.split('.')[0]) + line_start = 0 # Offset of the current line in chars. + tags = [] + pos = 0 + while True: + m = self.prog.search(chars, pos) + if m is None: + break for name, matched_text in matched_named_groups(m): a, b = m.span(name) - self._add_tag(a, b, head, name) + tags.append((a - line_start, b - line_start, head, name)) if matched_text in ("def", "class"): if m1 := self.idprog.match(chars, b): a, b = m1.span(1) - self._add_tag(a, b, head, "DEFINITION") + tags.append((a - line_start, b - line_start, + head, "DEFINITION")) + pos = m.end() + if '\n' in m[0]: + line += m[0].count('\n') + line_start = m.start() + m[0].rindex('\n') + 1 + head = f"{line}.0" + elif pos - line_start >= MAX_COLORIZED_LINE: + # The rest of a long line is not colorized. + pos = chars.find('\n', pos) + if pos < 0: + break + # Adding a tag is faster if there are no tags after it. + for args in reversed(tags): + self._add_tag(*args) def removecolors(self): "Remove all colorizing tags." diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 5bb4f5b3ff36e6..16a8b10864ba2e 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -573,6 +573,18 @@ def test_long_multiline_string(self): e""" ''') self._assert_highlighting(source, {'STRING': [('1.0', '5.4')]}) + source = '"""a\nb""" + str\n' + self._assert_highlighting(source, {'STRING': [('1.0', '2.4')], + 'BUILTIN': [('2.7', '2.10')]}) + + def test_long_line(self): + # gh-103089: only the first MAX_COLORIZED_LINE characters of a line + # are colorized. + n = colorizer.MAX_COLORIZED_LINE + source = f"pass\n{'x' * (n - 3)}'a', 'b'\n'c'\n" + self._assert_highlighting(source, {'KEYWORD': [('1.0', '1.4')], + 'STRING': [(f'2.{n-3}', f'2.{n}'), + ('3.0', '3.3')]}) @run_in_tk_mainloop(delay=50) def test_incremental_editing(self): diff --git a/Misc/NEWS.d/next/IDLE/2026-09-17-02-00-00.gh-issue-103089.longline.rst b/Misc/NEWS.d/next/IDLE/2026-09-17-02-00-00.gh-issue-103089.longline.rst new file mode 100644 index 00000000000000..0a7037eab869ff --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-17-02-00-00.gh-issue-103089.longline.rst @@ -0,0 +1,2 @@ +IDLE no longer hangs when opening a file with very long lines. +Only the first 2,000 characters of a line are colorized. From 4bc392c13462096213777c6b2bdc470028c41536 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 23 Sep 2026 23:04:13 +0300 Subject: [PATCH 5/7] gh-69919: Catch all compile errors in the code module, pyrepl and IDLE (GH-157585) compile() can raise MemoryError or RecursionError for too deeply nested source, not only SyntaxError, OverflowError and ValueError. IDLE's Shell then lost its prompt until the input was deleted. Co-authored-by: Terry Jan Reedy Co-Authored-By: Claude Opus 5 (1M context) --- Doc/builtins/functions.rst | 18 +++++++++++++---- Doc/library/code.rst | 2 +- Lib/_pyrepl/console.py | 4 ++-- Lib/_pyrepl/simple_interact.py | 2 +- Lib/code.py | 6 +++--- Lib/idlelib/idle_test/test_runscript.py | 13 ++++++++++++ Lib/idlelib/pyshell.py | 20 +++++++++++-------- Lib/idlelib/runscript.py | 14 ++++++++----- Lib/pdb.py | 4 ++-- Lib/test/test_code_module.py | 11 ++++++++++ Lib/test/test_pdb.py | 17 ++++++++++++++++ Lib/test/test_pyrepl/test_interact.py | 17 ++++++++++++++++ ...6-09-15-16-37-19.gh-issue-69919.f6AamT.rst | 2 ++ ...6-09-15-16-37-19.gh-issue-69919.izH1zP.rst | 4 ++++ 14 files changed, 108 insertions(+), 26 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-15-16-37-19.gh-issue-69919.f6AamT.rst create mode 100644 Misc/NEWS.d/next/Library/2026-09-15-16-37-19.gh-issue-69919.izH1zP.rst diff --git a/Doc/builtins/functions.rst b/Doc/builtins/functions.rst index 5cce5e3c87628a..ece666030db981 100644 --- a/Doc/builtins/functions.rst +++ b/Doc/builtins/functions.rst @@ -384,8 +384,14 @@ are always available. They are listed here in alphabetical order. It is needed to unambiguous :ref:`filter ` syntax warnings by module name. - This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled - source is invalid. + This function raises :exc:`SyntaxError` if the compiled source is invalid, + including a *source* containing a null character or that cannot be decoded; + :exc:`ValueError` if *mode* or *flags* is invalid, + or if a string *source* contains surrogate characters; + :exc:`MemoryError` or :exc:`RecursionError` if *source* is too complex + to parse or compile, + for example an expression with many thousands of nested operators; + and :exc:`OverflowError` if *source* is too large. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -417,11 +423,15 @@ are always available. They are listed here in alphabetical order. Previously, :exc:`TypeError` was raised when null bytes were encountered in *source*. - .. versionadded:: 3.8 + .. versionchanged:: 3.8 ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable support for top-level ``await``, ``async for``, and ``async with``. - .. versionadded:: 3.15 + .. versionchanged:: 3.12 + :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes + are encountered in *source*. + + .. versionchanged:: 3.15 Added the *module* parameter. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 59c016d21501b0..71eb3e8646cd01 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -92,7 +92,7 @@ Interactive Interpreter Objects *symbol* is ``'single'``. One of several things can happen: * The input is incorrect; :func:`compile_command` raised an exception - (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be + (usually :exc:`SyntaxError`). A syntax traceback will be printed by calling the :meth:`showsyntaxerror` method. :meth:`runsource` returns ``False``. diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index dcf8ff9b083caa..0da3c36ef9ed47 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -257,7 +257,7 @@ def runsource(self, source, filename="", symbol="single"): ) self.showsyntaxerror(filename, source=source) return False - except (OverflowError, ValueError): + except Exception: self.showsyntaxerror(filename, source=source) return False if tree.body: @@ -278,7 +278,7 @@ def runsource(self, source, filename="", symbol="single"): ) self.showsyntaxerror(filename, source=source) return False - except (OverflowError, ValueError): + except Exception: self.showsyntaxerror(filename, source=source) return False diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index e6c355388a7c07..5a7c8c20202640 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -82,7 +82,7 @@ def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool: src = _strip_final_indent(unicodetext) try: code = console.compile(src, "", "single") - except (OverflowError, SyntaxError, ValueError): + except Exception: lines = src.splitlines(keepends=True) if len(lines) == 1: return False diff --git a/Lib/code.py b/Lib/code.py index df1d7199e33934..f658049197e1b7 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -44,8 +44,8 @@ def runsource(self, source, filename="", symbol="single"): One of several things can happen: 1) The input is incorrect; compile_command() raised an - exception (SyntaxError or OverflowError). A syntax traceback - will be printed by calling the showsyntaxerror() method. + exception (usually SyntaxError). A syntax traceback will be + printed by calling the showsyntaxerror() method. 2) The input is incomplete, and more input is required; compile_command() returned None. Nothing happens. @@ -62,7 +62,7 @@ def runsource(self, source, filename="", symbol="single"): """ try: code = self.compile(source, filename, symbol) - except (OverflowError, SyntaxError, ValueError): + except Exception: # Case 1 self.showsyntaxerror(filename, source=source) return False diff --git a/Lib/idlelib/idle_test/test_runscript.py b/Lib/idlelib/idle_test/test_runscript.py index 1e47f402d504f9..69d253b8f2c73f 100644 --- a/Lib/idlelib/idle_test/test_runscript.py +++ b/Lib/idlelib/idle_test/test_runscript.py @@ -29,6 +29,19 @@ def test_init(self): sb = runscript.ScriptBinding(ew) ew._close() + def test_checksyntax_compile_error(self): + # gh-69919: any error raised by compile() is reported. + ew = EditorWindow(root=self.root) + sb = runscript.ScriptBinding(ew) + sb.flist = mock.Mock() + sb.errorbox = mock.Mock() + with (mock.patch('idlelib.runscript.compile', create=True, + side_effect=MemoryError()), + mock.patch('idlelib.runscript.open', mock.mock_open(read_data=b'x\n'))): + self.assertFalse(sb.checksyntax('test.py')) + sb.errorbox.assert_called_once_with('MemoryError', '') + ew._close() + def test_run_module_event_shell_busy_no_restart(self): # gh-82183: running without restarting the busy shell aborts. ew = EditorWindow(root=self.root) diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 953394a870c8d3..7a76f89d0178f2 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -686,7 +686,7 @@ def execfile(self, filename, source=None): + source + "\ndel __file__") try: code = compile(source, filename, "exec") - except (OverflowError, SyntaxError): + except Exception: self.tkconsole.resetoutput() print('*** Error in script or command!\n' 'Traceback (most recent call last):', @@ -736,19 +736,23 @@ def showsyntaxerror(self, filename=None, **kwargs): text = tkconsole.text text.tag_remove("ERROR", "1.0", "end") type, value, tb = sys.exc_info() - msg = getattr(value, 'msg', '') or value or "" - lineno = getattr(value, 'lineno', '') or 1 - offset = getattr(value, 'offset', '') or 0 + if not issubclass(type, SyntaxError): + tkconsole.resetoutput() + InteractiveInterpreter.showsyntaxerror(self, filename, **kwargs) + tkconsole.showprompt() + return + msg = value.msg or "" + lineno = value.lineno or 1 + offset = value.offset or 0 if offset == 0: lineno += 1 #mark end of offending line if lineno == 1: - pos = "iomark + %d chars" % (offset-1) + pos = f"iomark + {offset-1} chars" else: - pos = "iomark linestart + %d lines + %d chars" % \ - (lineno-1, offset-1) + pos = f"iomark linestart + {lineno-1} lines + {offset-1} chars" tkconsole.colorize_syntax_error(text, pos) tkconsole.resetoutput() - self.write("SyntaxError: %s\n" % msg) + self.write(f"{type.__name__}: {msg}\n") tkconsole.showprompt() def showtraceback(self): diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py index cd52d206c9ca0f..af3581ce0ce773 100644 --- a/Lib/idlelib/runscript.py +++ b/Lib/idlelib/runscript.py @@ -93,15 +93,19 @@ def checksyntax(self, filename): try: # If successful, return the compiled code return compile(source, filename, "exec") - except (SyntaxError, OverflowError, ValueError) as value: - msg = getattr(value, 'msg', '') or value or "" - lineno = getattr(value, 'lineno', '') or 1 - offset = getattr(value, 'offset', '') or 0 + except SyntaxError as value: + msg = value.msg or "" + lineno = value.lineno or 1 + offset = value.offset or 0 if offset == 0: lineno += 1 #mark end of offending line pos = "0.0 + %d lines + %d chars" % (lineno-1, offset-1) editwin.colorize_syntax_error(text, pos) - self.errorbox("SyntaxError", "%-20s" % msg) + self.errorbox(type(value).__name__, msg) + return False + except Exception as value: + msg = str(value) or "" + self.errorbox(type(value).__name__, msg) return False finally: shell.set_warning_stream(saved_stream) diff --git a/Lib/pdb.py b/Lib/pdb.py index 1ef877cce4dd96..e03f4fcf62cf3f 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -156,7 +156,7 @@ def find_function(funcname, filename): if funcdef: try: code = compile(funcdef, filename, 'exec') - except SyntaxError: + except Exception: continue # We should always be able to find the code object here funccode = next(c for c in code.co_consts if @@ -2781,7 +2781,7 @@ def _compile_error_message(self, expr): """Return the error message as string if compiling `expr` fails.""" try: compile(expr, "", "eval") - except SyntaxError as exc: + except Exception as exc: return _rstr(self._format_exc(exc)) return "" diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 3642b47c2c1f03..c0c3454a97fbd8 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -140,6 +140,17 @@ def test_unicode_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + def test_compile_error(self): + # Any error raised by compile() must be reported (gh-69919). + self.infunc.side_effect = ['-' * 100_000 + '1', EOFError('Finished')] + self.console.interact() + output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) + output = output[output.index('(InteractiveConsole)'):] + output = output[output.index('\n') + 1:] + self.assertRegex(output, r'^(MemoryError|RecursionError): ') + self.assertIn(self.sysmod.last_type, (MemoryError, RecursionError)) + self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) + def test_sysexcepthook(self): self.infunc.side_effect = ["def f():", " raise ValueError('BOOM!')", diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 0aa4ceb71c866f..8876b3f5ce37a3 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3653,6 +3653,23 @@ def quux(): ('bœr', 5), ) + def test_find_function_too_complex(self): + # gh-69919: compile() can raise more than SyntaxError. + self._assert_find_function( + b"def foo():\n return " + b"-" * 100_000 + b"1\n" + b"def bar():\n pass\n", + 'bar', + ('bar', 4), + ) + + def test_compile_error_message(self): + p = pdb.Pdb() + self.assertEqual(p._compile_error_message('1 + 1'), '') + self.assertIn('SyntaxError', p._compile_error_message('1 +')) + # gh-69919: compile() can raise more than SyntaxError. + self.assertRegex(p._compile_error_message('-' * 100_000 + '1'), + r'^(MemoryError|RecursionError|SyntaxError):') + def test_print_stack_entry_uses_dynamic_line_prefix(self): """Test that pdb.line_prefix binding is dynamic (gh-141781).""" stdout = io.StringIO() diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index fd4530ebc004aa..673827608abc47 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -131,6 +131,17 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): console.runsource(source) mock_showsyntaxerror.assert_called_once() + @force_not_colorized + def test_runsource_compile_error(self): + # Any error raised by compile() is reported (gh-69919). + console = InteractiveColoredConsole() + source = '-' * 100_000 + '1' + f = io.StringIO() + with contextlib.redirect_stderr(f): + result = console.runsource(source) + self.assertFalse(result) + self.assertRegex(f.getvalue(), r'^(MemoryError|RecursionError): ') + def test_runsource_survives_null_bytes(self): console = InteractiveColoredConsole() source = "\x00\n" @@ -182,6 +193,12 @@ def test_invalid_syntax_single_line(self): console = InteractiveColoredConsole(namespace, filename="") self.assertFalse(_more_lines(console, code)) + def test_compile_error_single_line(self): + namespace = {} + code = '-' * 100_000 + '1' # MemoryError or RecursionError + console = InteractiveColoredConsole(namespace, filename="") + self.assertFalse(_more_lines(console, code)) + def test_empty_line(self): namespace = {} code = "" diff --git a/Misc/NEWS.d/next/IDLE/2026-09-15-16-37-19.gh-issue-69919.f6AamT.rst b/Misc/NEWS.d/next/IDLE/2026-09-15-16-37-19.gh-issue-69919.f6AamT.rst new file mode 100644 index 00000000000000..fc909152113077 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-15-16-37-19.gh-issue-69919.f6AamT.rst @@ -0,0 +1,2 @@ +IDLE now reports any exception raised by compiling the source, such as +:exc:`MemoryError` for too deeply nested source, instead of hanging the Shell. diff --git a/Misc/NEWS.d/next/Library/2026-09-15-16-37-19.gh-issue-69919.izH1zP.rst b/Misc/NEWS.d/next/Library/2026-09-15-16-37-19.gh-issue-69919.izH1zP.rst new file mode 100644 index 00000000000000..87fd4a7aedee84 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-15-16-37-19.gh-issue-69919.izH1zP.rst @@ -0,0 +1,4 @@ +:meth:`code.InteractiveInterpreter.runsource` and the REPL now report any +exception raised by compiling the source, such as :exc:`MemoryError` or +:exc:`RecursionError` for too deeply nested source, instead of propagating +it or printing a traceback of the REPL internals. From 09bf4c525871db3ad02c8a1359da5cf0668f27b0 Mon Sep 17 00:00:00 2001 From: Joseph Kerry Date: Wed, 23 Sep 2026 21:06:25 +0100 Subject: [PATCH 6/7] gh-157335: Fix out-of-bounds write in mmap.mmap.__setitem__ (#157438) Fix out-of-bounds write in mmap.mmap.__setitem__() that could occur when converting the index or the assigned value (via __index__() for a single item, or via the buffer protocol for a slice) resized or closed the mmap object during the assignment. Co-authored-by: Victor Stinner --- Lib/test/test_mmap.py | 45 ++++++++++++++++++- ...-09-13-15-58-28.gh-issue-157335.efaMah.rst | 4 ++ Modules/mmapmodule.c | 32 +++++++------ 3 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-09-13-15-58-28.gh-issue-157335.efaMah.rst diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 053a4ca4db53a5..59be17455a4e4e 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -75,7 +75,7 @@ def test_basic(self): # Shouldn't crash on boundary (Issue #5292) self.assertRaises(IndexError, m.__getitem__, len(m)) - self.assertRaises(IndexError, m.__setitem__, len(m), b'\0') + self.assertRaises(IndexError, m.__setitem__, len(m), 0) # Modify the file's content m[0] = b'3'[0] @@ -953,6 +953,49 @@ def test_resize_down_anonymous_mapping(self): with self.assertRaises(ValueError): m.resize(start_size) + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_setitem_resize_reentrancy(self): + """Resizing the mmap from inside __index__ while assigning to a + single item must not access memory past the new bounds (gh-157335). + """ + size = 2 * PAGESIZE + new_size = PAGESIZE + + class ResizeOnIndex: + def __init__(self, m): + self.m = m + def __index__(self): + self.m.resize(new_size) + return 0 + + with mmap.mmap(-1, size) as m: + with self.assertRaises(IndexError): + m[size - 1] = ResizeOnIndex(m) + self.assertEqual(len(m), new_size) + + @unittest.skipUnless(hasattr(mmap.mmap, 'resize'), 'requires mmap.resize') + def test_setitem_slice_resize_reentrancy(self): + """Resizing the mmap from inside a value's buffer-protocol + callback while assigning to a slice must not access memory past + the new bounds (gh-157335). + """ + size = 2 * PAGESIZE + new_size = PAGESIZE + + class ResizeOnBuffer: + def __init__(self, m, data): + self.m = m + self.data = data + def __buffer__(self, flags): + self.m.resize(new_size) + return memoryview(self.data) + + with mmap.mmap(-1, size) as m: + value = ResizeOnBuffer(m, bytes(size)) + with self.assertRaises(IndexError): + m[0:size] = value + self.assertEqual(len(m), new_size) + @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_resize_fails_if_mapping_held_elsewhere(self): """If more than one mapping is held against a named file on Windows, neither diff --git a/Misc/NEWS.d/next/Library/2026-09-13-15-58-28.gh-issue-157335.efaMah.rst b/Misc/NEWS.d/next/Library/2026-09-13-15-58-28.gh-issue-157335.efaMah.rst new file mode 100644 index 00000000000000..fb0bcad3060564 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-09-13-15-58-28.gh-issue-157335.efaMah.rst @@ -0,0 +1,4 @@ +Fix out-of-bounds write in ``mmap.mmap.__setitem__`` that could occur +when converting the index or the assigned value (via :meth:`~object.__index__` +for a single item, or via the buffer protocol for a slice) resized or closed the mmap +object during the assignment. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 58f1e3b2ddcca7..766e85a1bdba98 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1649,24 +1649,15 @@ static int mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) { mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(-1); if (!is_writable(self)) return -1; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - Py_ssize_t v; - if (i == -1 && PyErr_Occurred()) return -1; - if (i < 0) - i += self->size; - if (i < 0 || i >= self->size) { - PyErr_SetString(PyExc_IndexError, - "mmap index out of range"); - return -1; - } + if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap doesn't support item deletion"); @@ -1677,7 +1668,7 @@ mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) "mmap item value must be an int"); return -1; } - v = PyNumber_AsSsize_t(value, PyExc_TypeError); + Py_ssize_t v = PyNumber_AsSsize_t(value, PyExc_TypeError); if (v == -1 && PyErr_Occurred()) return -1; if (v < 0 || v > 255) { @@ -1686,7 +1677,18 @@ mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) "in range(0, 256)"); return -1; } + + /* Converting item or value above may have run arbitrary code + * (e.g. __index__) that resized or closed the mmap, so bounds + * are only checked now, against the current size. */ CHECK_VALID(-1); + if (i < 0) + i += self->size; + if (i < 0 || i >= self->size) { + PyErr_SetString(PyExc_IndexError, + "mmap index out of range"); + return -1; + } char v_char = (char) v; if (safe_byte_copy(self->data + i, &v_char) < 0) { @@ -1701,7 +1703,6 @@ mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (value == NULL) { PyErr_SetString(PyExc_TypeError, "mmap object doesn't support slice deletion"); @@ -1709,6 +1710,12 @@ mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) } if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) return -1; + + /* Acquiring the buffer above may have run arbitrary code (e.g. a + * __buffer__ method) that resized or closed this mmap, so the slice bounds + * are only computed now, against the current size. */ + CHECK_VALID_OR_RELEASE(-1, vbuf); + slicelen = PySlice_AdjustIndices(self->size, &start, &stop, step); if (vbuf.len != slicelen) { PyErr_SetString(PyExc_IndexError, "mmap slice assignment is wrong size"); @@ -1716,7 +1723,6 @@ mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) return -1; } - CHECK_VALID_OR_RELEASE(-1, vbuf); int result = 0; if (slicelen == 0) { } From 6893326350024d0ed3a6fa4ff59e4139b2647411 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 24 Sep 2026 00:34:39 +0300 Subject: [PATCH 7/7] gh-70331: Protect IDLE's imports from user files in the current directory (#157643) Start the user process with -P, so that the current directory is not on sys.path while idlelib.run and its dependencies are imported. sys.path is set later by transfer_path(). Protect __main__, idle, and pyshell entry points. --- Lib/idlelib/__main__.py | 7 ++++++ Lib/idlelib/idle.py | 8 ++++++- Lib/idlelib/idle_test/test_pyshell.py | 23 ++++++++++++++++++- Lib/idlelib/pyshell.py | 9 +++++++- ...6-09-16-22-00-00.gh-issue-70331.shadow.rst | 4 ++++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-16-22-00-00.gh-issue-70331.shadow.rst diff --git a/Lib/idlelib/__main__.py b/Lib/idlelib/__main__.py index ec3915b265f665..4dde0d6a5acbb5 100644 --- a/Lib/idlelib/__main__.py +++ b/Lib/idlelib/__main__.py @@ -3,5 +3,12 @@ Run IDLE as python -m idlelib """ +import sys + +if not sys.flags.safe_path: + # Remove the current directory, prepended by "python -m", so that + # user files do not shadow IDLE's imports (gh-70331). + del sys.path[0] + import idlelib.pyshell idlelib.pyshell.main() diff --git a/Lib/idlelib/idle.py b/Lib/idlelib/idle.py index 485d5a75a29c1a..d9faa12cc2ba45 100644 --- a/Lib/idlelib/idle.py +++ b/Lib/idlelib/idle.py @@ -1,6 +1,12 @@ -import os.path import sys +if __spec__ is not None and not sys.flags.safe_path: + # Remove the current directory, prepended by "python -m", so that + # user files do not shadow IDLE's imports (gh-70331). + del sys.path[0] + +import os.path + # Enable running IDLE with idlelib in a non-standard location. # This was once used to run development versions of IDLE. diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py index 4aa0ba3a90158a..907606a4b1ee49 100644 --- a/Lib/idlelib/idle_test/test_pyshell.py +++ b/Lib/idlelib/idle_test/test_pyshell.py @@ -3,8 +3,11 @@ from idlelib import pyshell import os +import sys import unittest -from test.support import requires +from unittest import mock +from test.support import os_helper, requires +from test.support.script_helper import assert_python_ok from tkinter import Tk @@ -37,6 +40,24 @@ def test_fix_user_path(self): eq(pyshell.fix_user_path(['/a', '/b']), ['/a', '/b']) eq(pyshell.fix_user_path([idlelib_dir]), []) + def test_shadowed_stdlib(self): + # gh-70331: user files in the current directory must not shadow + # the stdlib modules imported by IDLE. + with os_helper.temp_dir() as cwd: + for name in ('os', 'random', 'tkinter'): + os_helper.create_empty_file(os.path.join(cwd, f'{name}.py')) + for module in 'idlelib', 'idlelib.idle', 'idlelib.pyshell': + with self.subTest(module=module): + assert_python_ok('-m', module, '-h', + __isolated=False, __cwd=cwd) + + def test_build_subprocess_arglist(self): + interp = mock.Mock(port=1234) + args = pyshell.ModifiedInterpreter.build_subprocess_arglist(interp) + # gh-70331: -P keeps the current directory out of sys.path. + self.assertEqual(args[:2], [sys.executable, '-P']) + self.assertEqual(args[-1], '1234') + class PyShellFileListTest(unittest.TestCase): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 7a76f89d0178f2..19a6569ac61182 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -3,6 +3,10 @@ import sys if __name__ == "__main__": sys.modules['idlelib.pyshell'] = sys.modules['__main__'] + if __spec__ is not None and not sys.flags.safe_path: + # Remove the current directory, prepended by "python -m", so that + # user files do not shadow IDLE's imports (gh-70331). + del sys.path[0] try: from tkinter import * @@ -455,7 +459,10 @@ def build_subprocess_arglist(self): del_exitf = idleConf.GetOption('main', 'General', 'delete-exitfunc', default=False, type='bool') command = f"__import__('idlelib.run').run.main({del_exitf!r})" - return [sys.executable] + w + ["-c", command, str(self.port)] + # -P keeps the current directory off sys.path, so that user files + # do not shadow run's imports (gh-70331). transfer_path() sets + # sys.path later. + return [sys.executable, '-P'] + w + ["-c", command, str(self.port)] def start_subprocess(self): addr = (HOST, self.port) diff --git a/Misc/NEWS.d/next/IDLE/2026-09-16-22-00-00.gh-issue-70331.shadow.rst b/Misc/NEWS.d/next/IDLE/2026-09-16-22-00-00.gh-issue-70331.shadow.rst new file mode 100644 index 00000000000000..1a7124e76cd49f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-16-22-00-00.gh-issue-70331.shadow.rst @@ -0,0 +1,4 @@ +IDLE no longer fails to start with ``python -m idlelib``, and its user process +no longer fails to start, when the current directory contains user files with +the same names as standard library modules that IDLE imports, such as +``random.py`` or ``tkinter.py``.