gh-160268: Fix tokenize on a mismatched closing bracket in an f-string (#160272)

Since gh-105042, `tokenize` tolerates mismatched brackets. That also let
a mismatched closing bracket pop the brace that opened an f-string
replacement field, which the f-string state relies on. In `f'{]a'`, this
triggered an assertion failure in _PyLexer_scan_string.

Check the replacement field brace in both modes.

Author: lysnikolaou

Lib/test/test_tokenize.py

@@ -2675,15 +2675,17 @@ def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
         )
 
     def test_tolerant_fstring_closer_at_expression_entry_depth(self):
-        source = "f'1:{]]{}}r}''"
-        for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
-            with self.subTest(extra_tokens=extra_tokens):
-                with self.assertRaises(tokenize.TokenError) as caught:
-                    self._get_tokens(source, extra_tokens=extra_tokens)
-                self.assertEqual(
-                    caught.exception.args,
-                    ("f-string: unmatched ']'", position),
-                )
+        cases = [
+            ("f'1:{]]{}}r}''", ("f-string: unmatched ']'", (1, 6))),
+            ("f'{]a'", ("f-string: unmatched ']'", (1, 4))),
+            ("t'{\nx)'", ("t-string: unmatched ')'", (2, 2))),
+        ]
+        for source, error in cases:
+            for extra_tokens in (False, True):
+                with self.subTest(source=source, extra_tokens=extra_tokens):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(source, extra_tokens=extra_tokens)
+                    self.assertEqual(caught.exception.args, error)
 
     def test_int(self):
 

Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst

@@ -0,0 +1 @@
+Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser.

Parser/lexer/lexer.c

@@ -447,11 +447,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         if (tok->level > 0) {
             tok->level--;
             int opening = tok->parenstack[tok->level];
-            if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
-                                            (opening == '[' && c == ']') ||
-                                            (opening == '{' && c == '}'))) {
+            if (!((opening == '(' && c == ')') ||
+                  (opening == '[' && c == ']') ||
+                  (opening == '{' && c == '}'))) {
                 /* Do not match a closer against the brace that opened the
-                 * current replacement field. */
+                 * current replacement field.  The f-string state relies on
+                 * that brace, so this is checked even with extra tokens. */
                 if (current != NULL && opening == '{') {
                     int bracket_depth =
                         _PyLexer_FTStringBracketDepth(tok, current);
@@ -461,7 +462,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                             _PyLexer_StringPrefix(current->kind), c));
                     }
                 }
-                if (tok->parenlinenostack[tok->level] != tok->lineno) {
+                if (tok->tok_extra_tokens) {
+                    /* Allow other mismatched brackets. */
+                }
+                else if (tok->parenlinenostack[tok->level] != tok->lineno) {
                     return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                             "closing parenthesis '%c' does not match "
                             "opening parenthesis '%c' on line %d",