diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 9fa294aa3618845..a7298f8913c1023 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2675,15 +2675,17 @@ def test_tolerant_incompatible_prefix_position_after_non_ascii(self): ) def test_tolerant_fstring_closer_at_expression_entry_depth(self): - source = "f'1:{]]{}}r}''" - for extra_tokens, position in ((False, (1, 6)), (True, (1, 7))): - with self.subTest(extra_tokens=extra_tokens): - with self.assertRaises(tokenize.TokenError) as caught: - self._get_tokens(source, extra_tokens=extra_tokens) - self.assertEqual( - caught.exception.args, - ("f-string: unmatched ']'", position), - ) + cases = [ + ("f'1:{]]{}}r}''", ("f-string: unmatched ']'", (1, 6))), + ("f'{]a'", ("f-string: unmatched ']'", (1, 4))), + ("t'{\nx)'", ("t-string: unmatched ')'", (2, 2))), + ] + for source, error in cases: + for extra_tokens in (False, True): + with self.subTest(source=source, extra_tokens=extra_tokens): + with self.assertRaises(tokenize.TokenError) as caught: + self._get_tokens(source, extra_tokens=extra_tokens) + self.assertEqual(caught.exception.args, error) def test_int(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst new file mode 100644 index 000000000000000..7533a25e0b57a50 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-10-22-17-37-27.gh-issue-160268.gJ2iDx.rst @@ -0,0 +1 @@ +Fix a crash in :mod:`tokenize` when a mismatched closing bracket follows the opening brace of an f-string or t-string replacement field, as in ``f'{]a'``. It now raises the same "unmatched" error as the parser. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 39b39b981c1fc49..9049a0487c4ac20 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -447,11 +447,12 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token if (tok->level > 0) { tok->level--; int opening = tok->parenstack[tok->level]; - if (!tok->tok_extra_tokens && !((opening == '(' && c == ')') || - (opening == '[' && c == ']') || - (opening == '{' && c == '}'))) { + if (!((opening == '(' && c == ')') || + (opening == '[' && c == ']') || + (opening == '{' && c == '}'))) { /* Do not match a closer against the brace that opened the - * current replacement field. */ + * current replacement field. The f-string state relies on + * that brace, so this is checked even with extra tokens. */ if (current != NULL && opening == '{') { int bracket_depth = _PyLexer_FTStringBracketDepth(tok, current); @@ -461,7 +462,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token _PyLexer_StringPrefix(current->kind), c)); } } - if (tok->parenlinenostack[tok->level] != tok->lineno) { + if (tok->tok_extra_tokens) { + /* Allow other mismatched brackets. */ + } + else if (tok->parenlinenostack[tok->level] != tok->lineno) { return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "closing parenthesis '%c' does not match " "opening parenthesis '%c' on line %d",