[pull] main from python:main

Doc/builtins/functions.rst

@@ -384,8 +384,14 @@ are always available.  They are listed here in alphabetical order.
    It is needed to unambiguous :ref:`filter <warning-filter>` syntax warnings
    by module name.
 
-   This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled
-   source is invalid.
+   This function raises :exc:`SyntaxError` if the compiled source is invalid,
+   including a *source* containing a null character or that cannot be decoded;
+   :exc:`ValueError` if *mode* or *flags* is invalid,
+   or if a string *source* contains surrogate characters;
+   :exc:`MemoryError` or :exc:`RecursionError` if *source* is too complex
+   to parse or compile,
+   for example an expression with many thousands of nested operators;
+   and :exc:`OverflowError` if *source* is too large.
 
    If you want to parse Python code into its AST representation, see
    :func:`ast.parse`.
@@ -417,11 +423,15 @@ are always available.  They are listed here in alphabetical order.
       Previously, :exc:`TypeError` was raised when null bytes were encountered
       in *source*.
 
-   .. versionadded:: 3.8
+   .. versionchanged:: 3.8
       ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable
       support for top-level ``await``, ``async for``, and ``async with``.
 
-   .. versionadded:: 3.15
+   .. versionchanged:: 3.12
+      :exc:`SyntaxError` is raised instead of :exc:`ValueError` when null bytes
+      are encountered in *source*.
+
+   .. versionchanged:: 3.15
       Added the *module* parameter.
 
 

Doc/library/code.rst

@@ -92,7 +92,7 @@ Interactive Interpreter Objects
    *symbol* is ``'single'``.  One of several things can happen:
 
    * The input is incorrect; :func:`compile_command` raised an exception
-     (:exc:`SyntaxError` or :exc:`OverflowError`).  A syntax traceback will be
+     (usually :exc:`SyntaxError`).  A syntax traceback will be
      printed by calling the :meth:`showsyntaxerror` method.  :meth:`runsource`
      returns ``False``.
 

Doc/library/symtable.rst

@@ -57,6 +57,16 @@ Examining Symbol Tables
 
       Used for the symbol table of a class.
 
+   .. attribute:: INLINED_COMPREHENSION
+      :value: "inlined comprehension"
+
+      Used for the symbol table of a list, set or dict comprehension that
+      is inlined into the enclosing code unit (see :pep:`709`). A symbol
+      table of this type represents a sub-scope of the enclosing code unit's
+      scope, and it does not correspond to a separate compilation unit.
+
+      .. versionadded:: next
+
    The following members refer to different flavors of
    :ref:`annotation scopes <annotation-scopes>`.
 

Doc/whatsnew/3.16.rst

@@ -573,6 +573,16 @@ symtable
   like the builtin :func:`compile`.
   (Contributed by Serhiy Storchaka in :gh:`153844`.)
 
+* Inlined list, set and dict comprehensions (:pep:`709`) are now represented
+  as their own symbol table entries, of type
+  :attr:`~symtable.SymbolTableType.INLINED_COMPREHENSION`. Each such entry is
+  a lexical child of the enclosing scope and records the comprehension's own
+  locals, cells, and free names. It does not correspond to a separate
+  compilation unit. For names loaded only inside an inlined list, set, or
+  dict comprehension, :meth:`symtable.Symbol.is_referenced` can now return
+  ``True`` on the enclosing function.
+  (Contributed by Irit Katriel in :gh:`124697`.)
+
 
 tkinter
 -------

Include/internal/pycore_compile.h

@@ -69,9 +69,8 @@ typedef struct {
     PyObject *u_varnames;  /* local variables */
     PyObject *u_cellvars;  /* cell variables */
     PyObject *u_freevars;  /* free variables */
-    PyObject *u_fasthidden; /* dict; keys are names that are fast-locals only
-                               temporarily within an inlined comprehension. When
-                               value is True, treat as fast-local. */
+    PyObject *u_fasthidden; /* set of names that are fast-locals only
+                               temporarily within an inlined comprehension. */
 
     Py_ssize_t u_argcount;        /* number of arguments for block */
     Py_ssize_t u_posonlyargcount;        /* number of positional only arguments for block */
@@ -152,11 +151,10 @@ PyObject *_PyCompile_MaybeMangle(struct _PyCompiler *c, PyObject *name);
 int _PyCompile_MaybeAddStaticAttributeToClass(struct _PyCompiler *c, expr_ty e);
 int _PyCompile_GetRefType(struct _PyCompiler *c, PyObject *name);
 int _PyCompile_LookupCellvar(struct _PyCompiler *c, PyObject *name);
-int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled, int scope,
+int _PyCompile_ResolveNameop(struct _PyCompiler *c, PyObject *mangled,
                              _PyCompile_optype *optype, Py_ssize_t *arg);
 
 int _PyCompile_IsInteractiveTopLevel(struct _PyCompiler *c);
-int _PyCompile_IsInInlinedComp(struct _PyCompiler *c);
 int _PyCompile_ScopeType(struct _PyCompiler *c);
 int _PyCompile_OptimizationLevel(struct _PyCompiler *c);
 int _PyCompile_LookupArg(struct _PyCompiler *c, PyCodeObject *co, PyObject *name);
@@ -180,16 +178,15 @@ enum {
 
 typedef struct {
     PyObject *pushed_locals;
-    PyObject *temp_symbols;
-    PyObject *fast_hidden;
     _PyJumpTargetLabel cleanup;
+    PySTEntryObject *saved_ste;
 } _PyCompile_InlinedComprehensionState;
 
-int _PyCompile_TweakInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
-                                               PySTEntryObject *entry,
-                                               _PyCompile_InlinedComprehensionState *state);
-int _PyCompile_RevertInlinedComprehensionScopes(struct _PyCompiler *c, _Py_SourceLocation loc,
-                                                _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_EnterInlinedComprehensionScope(struct _PyCompiler *c,
+                                              PySTEntryObject *entry,
+                                              _PyCompile_InlinedComprehensionState *state);
+int _PyCompile_ExitInlinedComprehensionScope(struct _PyCompiler *c,
+                                             _PyCompile_InlinedComprehensionState *state);
 int _PyCompile_AddDeferredAnnotation(struct _PyCompiler *c, stmt_ty s,
                                      PyObject **conditional_annotation_index);
 void _PyCompile_EnterConditionalBlock(struct _PyCompiler *c);

Include/internal/pycore_symtable.h

@@ -33,6 +33,10 @@ typedef enum _block_type {
     // i.e., a TypeVar, a TypeVarTuple or a ParamSpec object (the latter two
     // do not support a bound or a constraint tuple).
     TypeVariableBlock,
+    // List/set/dict comprehension inlined into the enclosing compilation unit
+    // (PEP 709). Lexical child of that unit, not a separate code object.
+    // See InternalDocs/inlined_comprehensions.md.
+    InlinedComprehensionBlock,
 } _Py_block_ty;
 
 typedef enum _comprehension_type {
@@ -119,7 +123,6 @@ typedef struct _symtable_entry {
                                              should be created */
     unsigned ste_needs_classdict : 1; /* for class scopes, true if a closure
                                          over the class dict should be created */
-    unsigned ste_comp_inlined : 1; /* true if this comprehension is inlined */
     unsigned ste_comp_iter_target : 1; /* true if visiting comprehension target */
     unsigned ste_can_see_class_scope : 1; /* true if this block can see names bound in an
                                              enclosing class scope */
@@ -132,6 +135,7 @@ typedef struct _symtable_entry {
     int ste_comp_iter_expr; /* non-zero if visiting a comprehension range expression */
     _Py_SourceLocation ste_loc; /* source location of block */
     struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */
+    struct _symtable_entry *ste_parent; /* st entry for the enclosing block if this entry is a sub-scope, NULL otherwise */
     struct symtable *ste_table;
 } PySTEntryObject;
 
@@ -142,6 +146,7 @@ extern PyTypeObject PySTEntry_Type;
 extern long _PyST_GetSymbol(PySTEntryObject *, PyObject *);
 extern int _PyST_GetScope(PySTEntryObject *, PyObject *);
 extern int _PyST_IsFunctionLike(PySTEntryObject *);
+extern int _PyST_IsClassClosureName(PyObject *);
 
 extern struct symtable* _PySymtable_Build(
     struct _mod *mod,
@@ -172,7 +177,6 @@ _Py_IsPrivateName(PyObject *);
 #define DEF_ANNOT (2<<7)         /* this name is annotated */
 #define DEF_COMP_ITER (2<<8)     /* this name is a comprehension iteration variable */
 #define DEF_TYPE_PARAM (2<<9)    /* this name is a type parameter */
-#define DEF_COMP_CELL (2<<10)    /* this name is a cell in an inlined comprehension */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 

InternalDocs/README.md

@@ -23,6 +23,8 @@ Compiling Python Source Code
 
 - [Compiler Design](compiler.md)
 
+- [Inlined comprehensions](inlined_comprehensions.md)
+
 - [Changing Python's Grammar](changing_grammar.md)
 
 Runtime Objects

InternalDocs/compiler.md

@@ -354,6 +354,10 @@ AST node type).  Next, the AST tree is walked with the various code blocks that
 delineate the reach of a local variable as blocks are entered and exited using
 `symtable_enter_block()` and `symtable_exit_block()`, respectively.
 
+See [Inlined comprehensions](inlined_comprehensions.md) for how list, set,
+and dict comprehensions are represented as lexical subscopes of the
+enclosing unit.
+
 Once the symbol table is created, the `AST` is transformed by `compiler_codegen()`
 in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instructions.
 These are similar to bytecode, but in some cases they are more abstract, and are

InternalDocs/inlined_comprehensions.md

@@ -0,0 +1,132 @@
+Inlined comprehensions
+======================
+
+Since [PEP 709](https://peps.python.org/pep-0709/), list, set, and dict
+comprehensions are compiled into the enclosing compilation unit instead of
+creating a nested function and calling it. Generator expressions are not
+inlined; they still become their own code object.
+
+The resulting bytecode lives in the enclosing unit, but the comprehension
+still has its own locals: iteration variables must not leak into, or
+overwrite, names in the enclosing scope. The symbol table models that as a
+nested lexical scope; codegen then emits the comprehension inlined into its
+containing compilation unit.
+
+Which comprehensions are inlined
+--------------------------------
+
+`symtable_handle_comprehension()` in
+[`Python/symtable.c`](../Python/symtable.c) inlines a comprehension when it
+is not a generator expression and the current block cannot see class scope
+(`!ste_can_see_class_scope`). Annotation scopes that can see a class keep
+the historical nested-function compilation so class-local names are not
+treated as comprehension locals.
+
+The outermost iterator expression is always evaluated in the enclosing
+scope. The rest of the comprehension (targets, `if` clauses, inner
+generators, and the element/value expressions) is visited in the
+comprehension's own block.
+
+Symbol table
+------------
+
+An inlined comprehension gets an `InlinedComprehensionBlock` entry
+([`pycore_symtable.h`](../Include/internal/pycore_symtable.h)). That entry
+is a child of the enclosing block, with `ste_parent` pointing at the
+enclosing `PySTEntryObject`. It is a lexical subscope, not a compilation
+unit: there is no separate code object, `co_consts` entry, or compiler
+scope for it.
+
+Uses and bindings inside the comprehension are recorded on that child
+table. Because those loads are in the enclosing compilation unit, a
+`FREE` use is also marked `USE` on the parent table. For
+`def inner(): return [x for y in ()]`, both `inner.lookup("x")` and the
+inlined child's lookup are `FREE|USE`.
+
+### Analysis
+
+`analyze_block()` records this block's declarations, analyzes children,
+then classifies uses. `finalize_inlined_comprehension()` copies
+`USE` from an inlined child onto the parent before that second pass, and
+drops inlined-only frees so `analyze_cells()` does not promote those
+names to cells.
+
+A name that is `FREE` in the comprehension and bound in the parent is
+dropped from the parent's free set unless:
+
+* a real nested unit (function, lambda, or genexp) still needs it as a
+  cell, or
+* a sibling nested scope already marked it free.
+
+That keeps iteration variables as fast locals when they are only used by
+nested inlined comprehensions. A nested lambda that captures the name
+still forces a cell on the binding comprehension.
+
+Class-closure names (`__class__` and friends) loaded from an inlined
+comprehension do not require a class cell unless a nested function,
+lambda, or genexp captures them. Compile treats the inlined loads as
+implicit globals. `is_free_in_any_child()` walks through inlined children
+and only counts `FREE` on non-inlined descendants.
+
+Compiler
+--------
+
+Codegen stays in the enclosing compiler unit. Around the inlined region,
+`_PyCompile_EnterInlinedComprehensionScope()` /
+`_PyCompile_ExitInlinedComprehensionScope()` in
+[`Python/compile.c`](../Python/compile.c) swap `c->u->u_ste` so name
+lookup uses the comprehension's symbol table. The saved `u_ste` is
+restored on both success and error.
+
+`_PyCompile_ResolveNameop()` calls `compiler_resolve_inlined_free()`,
+which walks `ste_parent` while the current table is inlined and the name
+is `FREE` or missing (scope `0`). Missing names include loads synthesized
+by codegen, such as the implicit receiver for zero-argument `super()`.
+The walk stops at a class: nested scopes do not see class locals.
+Class-closure names that would otherwise be free through a class become
+`GLOBAL_IMPLICIT`.
+
+### Isolating iteration variables
+
+`codegen_push_inlined_comprehension_locals()` in
+[`Python/codegen.c`](../Python/codegen.c) isolates names bound in the
+comprehension:
+
+* `LOAD_FAST_AND_CLEAR` saves the enclosing value (possibly `NULL`) and
+  clears the slot.
+* `MAKE_CELL` runs if the name is a cell for this comprehension.
+* In module and class units the name is added to `u_fasthidden` so
+  assemble can set `CO_FAST_HIDDEN`.
+
+A `SETUP_FINALLY` / `COMPILE_FBLOCK_INLINED_COMPREHENSION` handler
+restores those slots if the comprehension raises, so an enclosing `except`
+or `finally` sees the original values. 
+
+Runtime
+-------
+
+An inlined comprehension cell can share a localsplus name with an
+enclosing free variable (for example `[lambda: x for x in x]` inside a
+nested function). `FrameLocalsProxy` keys, values, items, and `len`
+keep the first slot of each name so they agree with `getitem`.
+
+Source
+------
+
+* [`Python/symtable.c`](../Python/symtable.c):
+  `symtable_handle_comprehension()`, `analyze_block()`,
+  `finalize_inlined_comprehension()`, `is_free_in_any_child()`
+* [`Python/compile.c`](../Python/compile.c):
+  `compiler_resolve_inlined_free()`,
+  `_PyCompile_EnterInlinedComprehensionScope()`,
+  `compiler_cellvars()`
+* [`Python/codegen.c`](../Python/codegen.c):
+  `codegen_comprehension()`,
+  `push_inlined_comprehension_state()`,
+  `codegen_push_inlined_comprehension_locals()`
+* [`Include/internal/pycore_symtable.h`](../Include/internal/pycore_symtable.h):
+  `InlinedComprehensionBlock`
+* [`Include/internal/pycore_compile.h`](../Include/internal/pycore_compile.h):
+  `_PyCompile_InlinedComprehensionState`
+* [`Objects/frameobject.c`](../Objects/frameobject.c):
+  `FrameLocalsProxy` duplicate-name handling

Lib/_pyrepl/console.py

@@ -257,7 +257,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                 )
             self.showsyntaxerror(filename, source=source)
             return False
-        except (OverflowError, ValueError):
+        except Exception:
             self.showsyntaxerror(filename, source=source)
             return False
         if tree.body:
@@ -278,7 +278,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
                     )
                 self.showsyntaxerror(filename, source=source)
                 return False
-            except (OverflowError, ValueError):
+            except Exception:
                 self.showsyntaxerror(filename, source=source)
                 return False
 

Lib/_pyrepl/simple_interact.py

@@ -82,7 +82,7 @@ def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
     src = _strip_final_indent(unicodetext)
     try:
         code = console.compile(src, "<stdin>", "single")
-    except (OverflowError, SyntaxError, ValueError):
+    except Exception:
         lines = src.splitlines(keepends=True)
         if len(lines) == 1:
             return False

Lib/code.py

@@ -44,8 +44,8 @@ def runsource(self, source, filename="<input>", symbol="single"):
         One of several things can happen:
 
         1) The input is incorrect; compile_command() raised an
-        exception (SyntaxError or OverflowError).  A syntax traceback
-        will be printed by calling the showsyntaxerror() method.
+        exception (usually SyntaxError).  A syntax traceback will be
+        printed by calling the showsyntaxerror() method.
 
         2) The input is incomplete, and more input is required;
         compile_command() returned None.  Nothing happens.
@@ -62,7 +62,7 @@ def runsource(self, source, filename="<input>", symbol="single"):
         """
         try:
             code = self.compile(source, filename, symbol)
-        except (OverflowError, SyntaxError, ValueError):
+        except Exception:
             # Case 1
             self.showsyntaxerror(filename, source=source)
             return False

Lib/idlelib/__main__.py

@@ -3,5 +3,12 @@
 
 Run IDLE as python -m idlelib
 """
+import sys
+
+if not sys.flags.safe_path:
+    # Remove the current directory, prepended by "python -m", so that
+    # user files do not shadow IDLE's imports (gh-70331).
+    del sys.path[0]
+
 import idlelib.pyshell
 idlelib.pyshell.main()