diff --git a/mypy/nativeparse.py b/mypy/nativeparse.py index 093f50f59e315..d45d8bbb0af70 100644 --- a/mypy/nativeparse.py +++ b/mypy/nativeparse.py @@ -295,7 +295,7 @@ def parse_to_binary_ast( platform=options.platform, always_true=options.always_true, always_false=options.always_false, - cache_version=4, + cache_version=5, ) return ( ast_bytes, @@ -1348,6 +1348,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: return ce elif tag == nodes.STR_EXPR: se = StrExpr(read_str(data)) + se.has_surrogates = read_bool(data) read_loc(data, se) expect_end_tag(data) return se @@ -1437,7 +1438,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: s = StrExpr(read_str(data)) read_loc(data, s) fitems.append(s) - expr = build_fstring_join(data, fitems) + expr = build_fstring_join(data, fitems, set_has_surrogates=True) expect_end_tag(data) return expr elif tag == nodes.LIST_COMPREHENSION: @@ -1534,6 +1535,7 @@ def read_expression(state: State, data: ReadBuffer) -> Expression: read_loc(data, s) titems.append(s) expr = TemplateStrExpr(titems) + expr.has_surrogates = read_bool(data) read_loc(data, expr) state.check_min_version( "t-strings", (3, 14), expr.line, expr.column, enforce_in_stubs=True @@ -1660,16 +1662,30 @@ def read_fstring_items(state: State, data: ReadBuffer) -> Expression: return build_fstring_join(data, items) -def build_fstring_join(data: ReadBuffer, items: list[Expression]) -> Expression: +def build_fstring_join( + data: ReadBuffer, items: list[Expression], set_has_surrogates: bool = False +) -> Expression: items = collapse_consecutive_str_items(items) if len(items) == 1: expr = items[0] + if set_has_surrogates: + if isinstance(expr, StrExpr): + target = expr + else: + assert isinstance(expr, CallExpr) and isinstance(expr.callee, MemberExpr) + # It doesn't really matter where to set the surrogates flag, + # so we set it on the outermost format string. + target = expr.callee.expr + assert isinstance(target, StrExpr) + target.has_surrogates = read_bool(data) read_loc(data, expr) return expr args = ListExpr(items) str_expr = StrExpr("") member = MemberExpr(str_expr, "join") call = CallExpr(member, [args], [ARG_POS], [None]) + if set_has_surrogates: + str_expr.has_surrogates = read_bool(data) read_loc(data, call) set_line_column(args, call) set_line_column(str_expr, call) diff --git a/mypy/nodes.py b/mypy/nodes.py index 53efc1fbc86b3..e64d86f0ab150 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2275,7 +2275,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class StrExpr(Expression): """String literal""" - __slots__ = ("value", "as_type") + __slots__ = ("value", "as_type", "has_surrogates") __match_args__ = ("value",) @@ -2284,11 +2284,16 @@ class StrExpr(Expression): # represents the type denoted by the type expression. # None means "is not a type expression". as_type: NotParsed | mypy.types.Type | None + # This indicates whether original string literal contained Unicode surrogate + # codepoints. Those are not supported by Ruff parser and are replaced by + # replacement characters. Thus, we can't support them in mypyc. + has_surrogates: bool def __init__(self, value: str) -> None: super().__init__() self.value = value self.as_type = NotParsed.VALUE + self.has_surrogates = False def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_str_expr(self) @@ -2937,7 +2942,7 @@ def accept(self, visitor: ExpressionVisitor[T]) -> T: class TemplateStrExpr(Expression): """Template string expression t'...'.""" - __slots__ = ("items",) + __slots__ = ("items", "has_surrogates") __match_args__ = ("items",) # Each item is either: @@ -2946,12 +2951,14 @@ class TemplateStrExpr(Expression): # where conversion is str | None ("r", "s", "a", or None) # and format_spec_expr is Expression | None items: list[Expression | tuple[Expression, str, str | None, Expression | None]] + has_surrogates: bool def __init__( self, items: list[Expression | tuple[Expression, str, str | None, Expression | None]] ) -> None: super().__init__() self.items = items + self.has_surrogates = False def accept(self, visitor: ExpressionVisitor[T]) -> T: return visitor.visit_template_str_expr(self) diff --git a/mypy/test/test_nativeparse.py b/mypy/test/test_nativeparse.py index 1e63f4abc2991..d74ff99d5e5e4 100644 --- a/mypy/test/test_nativeparse.py +++ b/mypy/test/test_nativeparse.py @@ -247,6 +247,9 @@ def _assert_trivial_binary_data(self, b: bytes, /) -> None: def int_enc(n: int) -> int: return (n + 10) << 1 + def bool_enc(b: bool) -> int: + return int(b) + def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> list[int]: return [ LOCATION, @@ -267,6 +270,7 @@ def locs(start_line: int, start_column: int, end_line: int, end_column: int) -> + [END_TAG, LIST_GEN, 22, nodes.STR_EXPR] + [LITERAL_STR, int_enc(5)] + list(b"hello") + + [bool_enc(False)] # no unicode surrogates + locs(1, 6, 1, 13) + [END_TAG] # arg_kinds: [ARG_POS] diff --git a/mypyc/irbuild/prebuildvisitor.py b/mypyc/irbuild/prebuildvisitor.py index 0f2aa7c1c91e8..232fcc55a0a77 100644 --- a/mypyc/irbuild/prebuildvisitor.py +++ b/mypyc/irbuild/prebuildvisitor.py @@ -15,6 +15,7 @@ MypyFile, NameExpr, Node, + StrExpr, SymbolNode, Var, ) @@ -259,6 +260,17 @@ def visit_name_expr(self, expr: NameExpr) -> None: if isinstance(expr.node, (Var, FuncDef)): self.visit_symbol_node(expr.node) + def visit_str_expr(self, o: StrExpr) -> None: + # Handle surrogates before main pass to avoid conflicts with various optimizations + # like replacing `ord("")` with its integer value statically, etc. + if o.has_surrogates: + self.errors.error( + "Surrogate codepoints in string literals not supported, use chr(...) instead", + self.current_file.path, + o.line, + ) + super().visit_str_expr(o) + def visit_var(self, var: Var) -> None: self.visit_symbol_node(var) @@ -272,7 +284,7 @@ def visit_symbol_node(self, symbol: SymbolNode) -> None: orig_func = self.symbols_to_funcs[symbol] if self.is_parent(self.funcs[-1], orig_func): # The function in which the symbol was previously seen is - # nested within the function currently being visited. Thus + # nested within the function currently being visited. Thus, # the current function is a better candidate to contain the # declaration. self.symbols_to_funcs[symbol] = self.funcs[-1] diff --git a/mypyc/irbuild/visitor.py b/mypyc/irbuild/visitor.py index 594ec414fa657..bb8aa4932e397 100644 --- a/mypyc/irbuild/visitor.py +++ b/mypyc/irbuild/visitor.py @@ -313,6 +313,11 @@ def visit_dict_expr(self, expr: DictExpr) -> Value: return transform_dict_expr(self.builder, expr) def visit_template_str_expr(self, expr: TemplateStrExpr) -> Value: + if expr.has_surrogates: + self.builder.error( + "Surrogate codepoints in string literals not supported, use chr(...) instead", + expr.line, + ) self.bail("Template strings are not supported by mypyc", expr.line) def visit_set_expr(self, expr: SetExpr) -> Value: diff --git a/mypyc/test-data/irbuild-str.test b/mypyc/test-data/irbuild-str.test index b16057f645ba7..bf2773196cdec 100644 --- a/mypyc/test-data/irbuild-str.test +++ b/mypyc/test-data/irbuild-str.test @@ -1175,3 +1175,18 @@ L0: r1 = '' r2 = CPyStr_EqualLiteral(r0, r1, 0) return r2 + +[case testUnicodeSurrogate] +# flags: --native-parser + +def f() -> str: + return "\ud800" + +def test_surrogate() -> None: + assert ord(f()) == 0xd800 + assert ord("\udfff") == 0xdfff + assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'" +[out] +main:4: error: Surrogate codepoints in string literals not supported, use chr(...) instead +main:8: error: Surrogate codepoints in string literals not supported, use chr(...) instead +main:9: error: Surrogate codepoints in string literals not supported, use chr(...) instead diff --git a/mypyc/test-data/run-strings.test b/mypyc/test-data/run-strings.test index 81b85580d7a7b..4bf8fa38cf3bc 100644 --- a/mypyc/test-data/run-strings.test +++ b/mypyc/test-data/run-strings.test @@ -1099,15 +1099,6 @@ def test_encode() -> None: with assertRaises(UnicodeEncodeError): u.encode('latin1') -[case testUnicodeSurrogate] -def f() -> str: - return "\ud800" - -def test_surrogate() -> None: - assert ord(f()) == 0xd800 - assert ord("\udfff") == 0xdfff - assert repr("foobar\x00\xab\ud912\U00012345") == r"'foobar\x00«\ud912𒍅'" - [case testStrip] def test_all_strips_default() -> None: s = " a1\t" diff --git a/mypyc/test/testutil.py b/mypyc/test/testutil.py index 9d59993c03402..2d139ace7c8e9 100644 --- a/mypyc/test/testutil.py +++ b/mypyc/test/testutil.py @@ -11,6 +11,7 @@ from mypy import build from mypy.errors import CompileError +from mypy.main import process_options from mypy.nodes import Expression, MypyFile from mypy.options import Options from mypy.test.config import test_temp_dir @@ -102,10 +103,16 @@ def build_ir_for_single_file2( ) -> tuple[ModuleIR, MypyFile, dict[Expression, Type], Mapper]: program_text = "\n".join(input_lines) - # By default generate IR compatible with the earliest supported Python C API. + flags = re.search("# flags: (.*)$", program_text, flags=re.MULTILINE) + + # By default, generate IR compatible with the earliest supported Python C API. # If a test needs more recent API features, this should be overridden. compiler_options = compiler_options or CompilerOptions(capi_version=(3, 10)) - options = Options() + if flags: + flag_list = flags.group(1).split() + _, options = process_options(flag_list, require_targets=False) + else: + options = Options() options.show_traceback = True options.hide_error_codes = True options.use_builtins_fixtures = True @@ -120,7 +127,7 @@ def build_ir_for_single_file2( options.per_module_options["__main__"] = {"mypyc": True} source = build.BuildSource("main", "__main__", program_text) - # Construct input as a single single. + # Construct input as a single source. # Parse and type check the input program. result = build.build(sources=[source], options=options, alt_lib_path=test_temp_dir) result.manager.metastore.close()