From 335b3757c0115436ad5fe85f6125dd2791655631 Mon Sep 17 00:00:00 2001
From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Thu, 22 Oct 2026 17:32:20 +0000
Subject: [PATCH 1/2] gh-160268: Fix tokenize on a mismatched closing bracket
 in an f-string

Since gh-105042, tokenize mode tolerates mismatched brackets. That also let
a mismatched closing bracket pop the brace that opened an f-string
replacement field, which the f-string state relies on. In f'{]a', this
triggered an assertion failure in _PyLexer_scan_string.

Check the replacement field brace in both modes.
---
 Lib/test/test_tokenize.py | 20 +++++++++++---------
 Parser/lexer/lexer.c      | 14 +++++++++-----
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 9fa294aa3618845..a7298f8913c1023 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -2675,15 +2675,17 @@ def test_tolerant_incompatible_prefix_position_after_non_ascii(self):
         )
 
     def test_tolerant_fstring_closer_at_expression_entry_depth(self):
-        source = "f'1:{]]{}}r}''"
-        for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))):
-            with self.subTest(extra_tokens=extra_tokens):
-                with self.assertRaises(tokenize.TokenError) as caught:
-                    self._get_tokens(source, extra_tokens=extra_tokens)
-                self.assertEqual(
-                    caught.exception.args,
-                    ("f-string: unmatched ']'", position),
-                )
+        cases = [
+            ("f'1:{]]{}}r}''", ("f-string: unmatched ']'", (1, 6))),
+            ("f'{]a'", ("f-string: unmatched ']'", (1, 4))),
+            ("t'{\nx)'", ("t-string: unmatched ')'", (2, 2))),
+        ]
+        for source, error in cases:
+            for extra_tokens in (False, True):
+                with self.subTest(source=source, extra_tokens=extra_tokens):
+                    with self.assertRaises(tokenize.TokenError) as caught:
+                        self._get_tokens(source, extra_tokens=extra_tokens)
+                    self.assertEqual(caught.exception.args, error)
 
     def test_int(self):
 
diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c
index 39b39b981c1fc49..9049a0487c4ac20 100644
--- a/Parser/lexer/lexer.c
+++ b/Parser/lexer/lexer.c
@@ -447,11 +447,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
         if (tok->level > 0) {
             tok->level--;
             int opening = tok->parenstack[tok->level];
-            if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') ||
-                                            (opening == '[' && c == ']') ||
-                                            (opening == '{' && c == '}'))) {
+            if (!((opening == '(' && c == ')') ||
+                  (opening == '[' && c == ']') ||
+                  (opening == '{' && c == '}'))) {
                 /* Do not match a closer against the brace that opened the
-                 * current replacement field. */
+                 * current replacement field.  The f-string state relies on
+                 * that brace, so this is checked even with extra tokens. */
                 if (current != NULL && opening == '{') {
                     int bracket_depth =
                         _PyLexer_FTStringBracketDepth(tok, current);
@@ -461,7 +462,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
                             _PyLexer_StringPrefix(current->kind), c));
                     }
                 }
-                if (tok->parenlinenostack[tok->level] != tok->lineno) {
+                if (tok->tok_extra_tokens) {
+                    /* Allow other mismatched brackets. */
+                }
+                else if (tok->parenlinenostack[tok->level] != tok->lineno) {
                     return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok,
                             "closing parenthesis '%c' does not match "
                             "opening parenthesis '%c' on line %d",

From bff1466f55d79ed22b04d7203e4a0afbdb9f5677 Mon Sep 17 00:00:00 2001
From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com>
Date: Thu, 22 Oct 2026 17:37:37 +0000
Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?=
 =?UTF-8?q?rb=5Fit.?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst               | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst
new file mode 100644
index 000000000000000..7533a25e0b57a50
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst
@@ -0,0 +1 @@
+Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser.