From 9df9bb0685d23c90e2c3141506bd54f194398613 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 22 Oct 2026 20:13:27 +0000 Subject: [PATCH 1/2] [3.15] gh-160268: Fix tokenize on a mismatched closing bracket in an f-string (GH-160272) Since gh-105042, `tokenize` tolerates mismatched brackets. That also let a mismatched closing bracket pop the brace that opened an f-string replacement field, which the f-string state relies on. In `f'{]a'`, this triggered an assertion failure in _PyLexer_scan_string. Check the replacement field brace in both modes. (cherry picked from commit 86758d1950ce2ed2626014b52d26d24c74dee55b) Co-authored-by: Lysandros Nikolaou --- .../2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst | 1 + Parser/lexer/lexer.c | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst new file mode 100644 index 000000000000000..7533a25e0b57a50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst @@ -0,0 +1 @@ +Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 52ed6f79975e0a6..63f0460e3c595bb 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1293,9 +1293,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (tok->level > 0) { tok->level--; int opening = tok->parenstack[tok->level]; - if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') || - (opening == '[' && c == ']') || - (opening == '{' && c == '}'))) { + if (!((opening == '(' && c == ')') || + (opening == '[' && c == ']') || + (opening == '{' && c == '}'))) { /* If the opening bracket belongs to an f-string's expression part (e.g. f"{)}") and the closing bracket is an arbitrary nested expression, then instead of matching a different @@ -1309,7 +1309,10 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t "%c-string: unmatched '%c'", TOK_GET_STRING_PREFIX(tok), c)); } } - if (tok->parenlinenostack[tok->level] != tok->lineno) { + if (tok->tok_extra_tokens) { + /* Allow other mismatched brackets. */ + } + else if (tok->parenlinenostack[tok->level] != tok->lineno) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "closing parenthesis '%c' does not match " "opening parenthesis '%c' on line %d", From b108c7767cc48a7f3ed2872b473496e85bd61281 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 22 Oct 2026 20:20:01 +0000 Subject: [PATCH 2/2] Line wrap news blurb and correctly indent if conditions --- .../2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst | 4 +++- Parser/lexer/lexer.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst index 7533a25e0b57a50..5132085d21258d2 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst @@ -1 +1,3 @@ -Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser. +Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the +opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. +It now raises the same "unmatched" error as the parser. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 63f0460e3c595bb..496c2df58c55175 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1294,8 +1294,8 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t tok->level--; int opening = tok->parenstack[tok->level]; if (!((opening == '(' && c == ')') || - (opening == '[' && c == ']') || - (opening == '{' && c == '}'))) { + (opening == '[' && c == ']') || + (opening == '{' && c == '}'))) { /* If the opening bracket belongs to an f-string's expression part (e.g. f"{)}") and the closing bracket is an arbitrary nested expression, then instead of matching a different