gh-159465: Preserve escape errors in unfinished f-strings and t-strings

Author: pablogsal

Lib/test/test_codeop.py

@@ -176,6 +176,56 @@ def test_string_continuation(self, symbol, prefix):
         with self.assertRaises(_IncompleteInputError):
             Compile()(source, '<input>', symbol)
 
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    @subTests('quote', ('"', '"""'))
+    def test_invalid_escape_in_incomplete_string(self, symbol, prefix, quote):
+        for escape in (r'\xZZ', r'\uZZZZ', r'\U00110000', r'\x', r'\u12'):
+            for body in (escape, escape + '{0}', '{0}' + escape,
+                         '{0:' + escape):
+                source = prefix + quote + body + '\\'
+                for compiler in COMPILERS:
+                    with self.subTest(source=source, compiler=compiler):
+                        with self.assertRaises(SyntaxError) as cm:
+                            compiler(source, '<input>', symbol)
+                        self.assertNotIsInstance(cm.exception, _IncompleteInputError)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    @subTests('quote', ('"', '"""'))
+    def test_valid_escape_in_incomplete_string(self, symbol, prefix, quote):
+        for escape in (r'\x41', r'\u00e9', r'\U0001f600', r'\\xZZ'):
+            source = prefix + quote + escape + '\\'
+            with self.subTest(escape=escape):
+                self.assertIsNone(compile_command(source, symbol=symbol))
+                with self.assertRaises(_IncompleteInputError):
+                    Compile()(source, '<input>', symbol)
+                for compiler in COMPILERS:
+                    self.assertValid(source + '\n' + quote, symbol,
+                                     compiler=compiler)
+
+        source = 'r' + prefix + quote + r'\xZZ' + '\\'
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+        for compiler in COMPILERS:
+            self.assertValid(source + '\n' + quote, symbol, compiler=compiler)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    def test_unfinished_escape_in_incomplete_string(self, symbol, prefix):
+        for escape, suffix in ((r'\x', '41'), (r'\u12', '34'),
+                               (r'\U0001', 'f600'),
+                               (r'\N{LATIN', ' SMALL LETTER A}')):
+            source = prefix + '"""' + escape
+            with self.subTest(escape=escape):
+                self.assertIsNone(compile_command(source, symbol=symbol))
+                with self.assertRaises(_IncompleteInputError):
+                    Compile()(source, '<input>', symbol)
+                for compiler in COMPILERS:
+                    self.assertValid(source + suffix + '"""', symbol,
+                                     compiler=compiler)
+
     @subTests('symbol', ('single', 'exec', 'eval'))
     @subTests('newline', ('\n', '\r', '\r\n'))
     def test_continued_final_line(self, symbol, newline):

Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst

@@ -0,0 +1,2 @@
+Preserve :exc:`SyntaxError` for malformed escapes in unfinished f-strings and
+t-strings passed to :func:`codeop.compile_command`.

Parser/action_helpers.c

@@ -1507,8 +1507,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {
+        if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) {
+            _Pypegen_raise_decode_error(p);
+        }
         return NULL;
     }
+    if (tok->is_incomplete) {
+        Py_DECREF(str);
+        return _PyPegen_raise_incomplete_input(p);
+    }
     if (_PyArena_AddPyObject(p->arena, str) < 0) {
         Py_DECREF(str);
         return NULL;
@@ -1523,6 +1530,21 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
         return _PyPegen_raise_incomplete_input(p);
     }
     const char *bstr = PyBytes_AS_STRING(tok->bytes);
+    Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes);
+    if (((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) &&
+        !tok->is_raw && memchr(bstr, '\\', bsize) != NULL) {
+        // Check escapes before the string is complete, without emitting warnings.
+        PyObject *decoded = _PyPegen_decode_string(
+            p, 0, bstr, bsize, tok->is_incomplete ? tok : NULL);
+        if (decoded == NULL) {
+            _Pypegen_raise_decode_error(p);
+            return NULL;
+        }
+        Py_DECREF(decoded);
+    }
+    if (tok->is_incomplete) {
+        return _PyPegen_raise_incomplete_input(p);
+    }
     PyObject* str = PyUnicode_FromString(bstr);
     if (str == NULL) {
         return NULL;

Parser/lexer/state.c

@@ -86,6 +86,7 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTo
 {
     token->level = tok->level;
     token->is_raw = 0;
+    token->is_incomplete = 0;
     assert((start == -1 && end == -1) || (start >= 0 && end >= start));
     token->span = (_PyTok_Span){start, end};
     if (start >= 0) {

Parser/lexer/string.c

@@ -27,6 +27,7 @@ incomplete_string_token(struct tok_state *tok, struct token *token,
 {
     string_error_token(tok, token, start, location);
     tok->done = E_INCOMPLETE;
+    token->is_incomplete = 1;
     return type;
 }
 
@@ -464,8 +465,16 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
                 return MAKE_TOKEN(ERRORTOKEN);
             }
             if (c == EOF && tok->allow_incomplete_input) {
-                return incomplete_string_token(tok, token,
+                int type = incomplete_string_token(tok, token,
                     FTSTRING_MIDDLE(current), current->start, current->start_loc);
+                // Preserve the actual prefix for incremental escape decoding.
+                _PyTok_Off end = tok->cur;
+                if (end > tok->start && tok->implicit_newline) {
+                    end--;
+                }
+                token->span = (_PyTok_Span){tok->start, end};
+                token->is_raw = raw;
+                return type;
             }
 
             if (in_format_spec && c == '\n') {

Parser/pegen.c

@@ -279,6 +279,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
 
     parser_token->level = new_token->level;
     parser_token->is_raw = new_token->is_raw;
+    parser_token->is_incomplete = new_token->is_incomplete;
     parser_token->lineno = new_token->start_loc.lineno;
     parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->start_loc.byte_col

Parser/pegen.h

@@ -39,11 +39,12 @@ typedef struct _memo {
 typedef struct {
     int type;
     // Text required by grammar actions or helpers; otherwise NULL.
-    // An unfinished string token also has NULL text. Its grammar action
-    // requests more input only if the token is valid at that position.
+    // Unfinished f-string and t-string middle tokens retain their text so
+    // their grammar action can check escapes before requesting more input.
     PyObject *bytes;
     int level;
-    int is_raw;
+    unsigned char is_raw;
+    unsigned char is_incomplete;
     int lineno, col_offset, end_lineno, end_col_offset;
     Memo *memo;
     // Filter over the rule types present in `memo` (bit `type & 63` is set

Parser/string_parser.c

@@ -194,13 +194,17 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
 
     int first_invalid_escape_char;
     const char *first_invalid_escape_ptr;
-    v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, NULL,
+    // Allow a trailing unfinished escape in a partial token.
+    int incomplete = t != NULL && t->is_incomplete;
+    Py_ssize_t consumed;
+    v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL,
+                                                incomplete ? &consumed : NULL,
                                                 &first_invalid_escape_char,
                                                 &first_invalid_escape_ptr);
 
     // HACK: later we can simply pass the line no, since we don't preserve the tokens
     // when we are decoding the string but we preserve the line numbers.
-    if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) {
+    if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL && !incomplete) {
         if (warn_invalid_escape_sequence(parser, buf, first_invalid_escape_ptr, t) < 0) {
             /* We have not deallocated the buffer before because
              * first_invalid_escape_ptr points inside buf. */

Parser/tokenizer/tokenizer.h

@@ -24,7 +24,8 @@ enum {
 struct token {
     int type;
     int level;
-    int is_raw;
+    unsigned char is_raw;
+    unsigned char is_incomplete;
     _PyTok_Span span;
     _PyTok_Loc start_loc;
     _PyTok_Loc end_loc;
@@ -69,8 +70,9 @@ typedef struct {
 
 /* Get replaces the initialized token, releasing its previous metadata.
    Errors are returned as ERRORTOKEN, with or without a Python exception.
-   In incomplete-input mode, an unfinished string retains its token type
-   with an absent span and E_INCOMPLETE status. No Python exception is set. */
+   In incomplete-input mode, an unfinished string retains its token type,
+   sets is_incomplete and has E_INCOMPLETE status. Only f-string and t-string
+   middle tokens retain their text span. No Python exception is set. */
 void _PyTokenizer_Get(struct tok_state *, struct token *);
 void _PyTokenizer_Free(struct tok_state *);
 int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *);