[WIP] Make native parser the default

mypy/main.py

@@ -1303,11 +1303,11 @@ def add_invertible_flag(
         dest="local_partial_types",
         help=argparse.SUPPRESS,
     )
-    # --native-parser enables the native parser (experimental)
     add_invertible_flag(
-        "--native-parser",
-        default=False,
-        help="Enable faster parser that parses directly to mypy AST",
+        "--no-native-parser",
+        default=True,
+        dest="native_parser",
+        help="Do not use faster parser that parses directly to mypy AST",
     )
     # --logical-deps adds some more dependencies that are not semantically needed, but
     # may be helpful to determine relative importance of classes and functions for overall

mypy/nativeparse.py

@@ -125,6 +125,7 @@
     WithStmt,
     YieldExpr,
     YieldFromExpr,
+    check_param_names,
 )
 from mypy.options import Options
 from mypy.patterns import (
@@ -628,6 +629,11 @@ def read_parameters(state: State, data: ReadBuffer) -> tuple[list[Argument], boo
         set_line_column_range(var, arg)
         arguments.append(arg)
 
+    def fail_arg(msg: str, ctx: Argument) -> None:
+        # To match the old parser (for now).
+        state.add_error(msg, ctx.line, ctx.column, blocker=True, code="syntax")
+
+    check_param_names([arg.variable.name for arg in arguments], arguments, fail_arg)
     return arguments, has_ann
 
 
@@ -638,6 +644,22 @@ def check_type_param_defaults(
         state.check_min_version("Type parameter defaults", (3, 13), line, column)
 
 
+def check_type_param_values(
+    state: State, type_params: list[TypeParam], line: int, column: int
+) -> None:
+    for type_param in type_params:
+        if len(type_param.values) == 1:
+            state.add_error(
+                message_registry.TYPE_VAR_TOO_FEW_CONSTRAINED_TYPES.value,
+                line,
+                column,
+                blocker=False,
+                code="misc",
+            )
+            # For compatibility with old parser.
+            type_param.values = []
+
+
 def read_type_params(state: State, data: ReadBuffer) -> list[TypeParam]:
     """Read type parameters (PEP 695 generics)."""
     type_params: list[TypeParam] = []
@@ -716,6 +738,7 @@ def read_func_def(state: State, data: ReadBuffer) -> FuncDef:
             "Improved type parameter syntax", (3, 12), func_def.line, func_def.column
         )
         check_type_param_defaults(state, type_params, func_def.line, func_def.column)
+        check_type_param_values(state, type_params, func_def.line, func_def.column)
     if typ:
         typ.line = func_def.line
         typ.column = func_def.column
@@ -770,6 +793,7 @@ def read_class_def(state: State, data: ReadBuffer) -> ClassDef:
             "Improved type parameter syntax", (3, 12), class_def.line, class_def.column
         )
         check_type_param_defaults(state, type_params, class_def.line, class_def.column)
+        check_type_param_values(state, type_params, class_def.line, class_def.column)
     expect_end_tag(data)
     if state.options.include_docstrings:
         class_def.docstring = get_docstring(body)
@@ -836,6 +860,7 @@ def read_type_alias_stmt(state: State, data: ReadBuffer) -> TypeAliasStmt:
     read_loc(data, stmt)
     state.check_min_version('"type" statements', (3, 12), stmt.line, stmt.column)
     check_type_param_defaults(state, type_params, stmt.line, stmt.column)
+    check_type_param_values(state, type_params, stmt.line, stmt.column)
     expect_end_tag(data)
     return stmt
 

mypy/options.py

@@ -389,8 +389,8 @@ def __init__(self) -> None:
         self.logical_deps = False
         # If True, partial types can't span a module top level and a function
         self.local_partial_types = True
-        # If True, use the native parser (experimental)
-        self.native_parser = False
+        # If True, use the native parser
+        self.native_parser = True
         # Some behaviors are changed when using Bazel (https://bazel.build).
         self.bazel = False
         # If True, export inferred types for all expressions as BuildResult.types

mypy/test/testcheck.py

@@ -137,7 +137,6 @@ def run_case_once(
         options = parse_options(original_program_text, testcase, incremental_step)
         options.use_builtins_fixtures = True
         options.show_traceback = True
-        options.native_parser = bool(os.environ.get("TEST_NATIVE_PARSER"))
         options.reveal_verbose_types = not testcase.name.endswith("_no_verbose_reveal")
 
         if options.num_workers:
@@ -152,8 +151,9 @@ def run_case_once(
             if testcase.name.endswith("_parallel_only"):
                 raise pytest.skip("Test is only for parallel mode")
 
-        if options.native_parser and testcase.name.endswith("_no_native_parse"):
-            raise pytest.skip("Test not supported by native parser yet")
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
 
         # Enable some options automatically based on test file name.
         if "columns" in testcase.file:

mypy/test/testdeps.py

@@ -39,6 +39,9 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         options.export_types = True
         options.preserve_asts = True
         options.allow_empty_bodies = True
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
         messages, files, type_map = self.build(src, options)
         a = messages
         if files is None or type_map is None:

mypy/test/testfinegrained.py

@@ -44,6 +44,8 @@
 )
 
 # Set to True to perform (somewhat expensive) checks for duplicate AST nodes after merge
+from mypy.test.update_data import update_testcase_output
+
 CHECK_CONSISTENCY = False
 
 
@@ -130,6 +132,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         # Normalize paths in test output (for Windows).
         a = [line.replace("\\", "/") for line in a]
 
+        # This may not work perfectly, since it was designed for testcheck.py, use with care.
+        if testcase.output != a and testcase.config.getoption("--update-data", False):
+            update_testcase_output(testcase, a, incremental_step=1)
+
         assert_string_arrays_equal(
             testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})"
         )
@@ -155,6 +161,11 @@ def get_options(self, source: str, testcase: DataDrivenTestCase, build_cache: bo
         options.export_types = "inspect" in testcase.file
         # Treat empty bodies safely for these test cases.
         options.allow_empty_bodies = not testcase.name.endswith("_no_empty")
+
+        if testcase.name.endswith("_old_parser") or testcase.name.endswith("_old_parser_cached"):
+            # This test is only for the old parser.
+            options.native_parser = False
+
         options.reveal_verbose_types = True
         if re.search("flags:.*--follow-imports", source) is None:
             # Override the default for follow_imports

mypy/test/testparse.py

@@ -13,6 +13,7 @@
 from mypy.parse import parse
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.helpers import assert_string_arrays_equal, find_test_files, parse_options
+from mypy.test.update_data import update_testcase_output
 from mypy.util import get_mypy_comments
 
 
@@ -115,6 +116,11 @@ def test_parse_error(testcase: DataDrivenTestCase) -> None:
     except CompileError as e:
         if e.module_with_blocker is not None:
             assert e.module_with_blocker == "__main__"
+
+        # This may not work perfectly, since it was designed for testcheck.py, use with care.
+        if testcase.output != e.messages and testcase.config.getoption("--update-data", False):
+            update_testcase_output(testcase, e.messages, incremental_step=1)
+
         # Verify that there was a compile error and that the error messages
         # are equivalent.
         assert_string_arrays_equal(

mypy/test/testsemanal.py

@@ -21,6 +21,7 @@
 
 # Semantic analyzer test cases: dump parse tree
 # Semantic analysis test case description files.
+from mypy.test.update_data import update_testcase_output
 from mypy.types import TypeStrVisitor
 
 semanal_files = find_test_files(
@@ -63,6 +64,9 @@ def test_semanal(testcase: DataDrivenTestCase) -> None:
         src = "\n".join(testcase.input)
         options = get_semanal_options(src, testcase)
         options.python_version = testfile_pyversion(testcase.file)
+        if testcase.name.endswith("_old_parser"):
+            # This test is only for the old parser.
+            options.native_parser = False
         result = build.build(
             sources=[BuildSource("main", None, src)], options=options, alt_lib_path=test_temp_dir
         )
@@ -112,6 +116,11 @@ def test_semanal_error(testcase: DataDrivenTestCase) -> None:
         a = e.messages
     if testcase.normalize_output:
         a = normalize_error_messages(a)
+
+    # This may not work perfectly, since it was designed for testcheck.py, use with care.
+    if testcase.output != a and testcase.config.getoption("--update-data", False):
+        update_testcase_output(testcase, a, incremental_step=1)
+
     assert_string_arrays_equal(
         testcase.output, a, f"Invalid compiler output ({testcase.file}, line {testcase.line})"
     )

mypy/test/teststubtest.py

@@ -3207,7 +3207,7 @@ def test_mypy_build(self) -> None:
         output = run_stubtest(stub="+", runtime="", options=[])
         assert output == (
             "error: not checking stubs due to failed mypy compile:\n{}.pyi:1: "
-            "error: Invalid syntax  [syntax]\n".format(TEST_MODULE_NAME)
+            "error: Expected an expression  [syntax]\n".format(TEST_MODULE_NAME)
         )
 
         output = run_stubtest(stub="def f(): ...\ndef f(): ...", runtime="", options=[])

test-data/unit/check-async-await.test

@@ -192,8 +192,8 @@ async def f() -> None:
 [builtins fixtures/async_await.pyi]
 [typing fixtures/typing-async.pyi]
 
-[case testAsyncForTypeComments_no_native_parse]
-
+-- Native parser does not support type comments in `for` and `with` statements.
+[case testAsyncForTypeComments_old_parser]
 from typing import AsyncIterator, Union
 class C(AsyncIterator[int]):
     async def __anext__(self) -> int: return 0
@@ -342,8 +342,8 @@ async def f() -> None:
 [builtins fixtures/async_await.pyi]
 [typing fixtures/typing-async.pyi]
 
-[case testAsyncWithTypeComments_no_native_parse]
-
+-- Native parser does not support type comments in `for` and `with` statements.
+[case testAsyncWithTypeComments_old_parser]
 class C:
     async def __aenter__(self) -> int: pass
     async def __aexit__(self, x, y, z) -> None: pass

test-data/unit/check-basic.test

@@ -286,10 +286,11 @@ x = 1
 x in 1,  # E: Unsupported right operand type for in ("int")
 [builtins fixtures/tuple.pyi]
 
-[case testTrailingCommaInIfParsing_no_native_parse]
+[case testTrailingCommaInIfParsing]
 if x in 1, : pass
 [out]
-main:1: error: Invalid syntax
+main:1: error: Expected `:`, found `,`
+main:1: error: Expected a statement
 
 [case testInitReturnTypeError]
 class C:

test-data/unit/check-columns.test

@@ -1,10 +1,10 @@
 # Test column numbers in messages. --show-column-numbers is enabled implicitly by test runner.
 
-[case testColumnsSyntaxError_no_native_parse]
+[case testColumnsSyntaxError]
 f()
 1 +
 [out]
-main:2:5: error: Invalid syntax
+main:2:5: error: Expected an expression
 
 [case testColumnsNestedFunctions]
 import typing
@@ -146,8 +146,7 @@ if int():
     def f(a: 'A') -> None: pass
     (f(b=object())) # E:6: Unexpected keyword argument "b" for "f"
 
-[case testColumnInvalidType_no_native_parse]
-
+[case testColumnInvalidType]
 from typing import Iterable
 
 bad = 0
@@ -158,8 +157,8 @@ def f(x: bad): # E:10: Variable "__main__.bad" is not valid as a type \
            # N:8: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
 
 if int():
-    def g(x): # E:5: Variable "__main__.bad" is not valid as a type \
-              # N:5: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
+    def g(x): # E:11: Variable "__main__.bad" is not valid as a type \
+              # N:11: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
         # type: (bad) -> None
         y = 0  # type: bad  # E:9: Variable "__main__.bad" is not valid as a type \
                             # N:9: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases
@@ -342,7 +341,7 @@ if int():
 main:2:11: error: Syntax error in type annotation
 main:2:11: note: Suggestion: Is there a spurious trailing comma?
 
-[case testColumnSyntaxErrorInTypeAnnotation2_no_native_parse]
+[case testColumnSyntaxErrorInTypeAnnotation2]
 if int():
     # TODO: It would be better to point to the type comment
     xyz = 0  # type: blurbnard blarb

test-data/unit/check-errorcodes.test

@@ -101,21 +101,19 @@ class A:
 [case testErrorCodeNoteHasNoCode]
 reveal_type(1) # N: Revealed type is "Literal[1]?"
 
-[case testErrorCodeSyntaxError_no_native_parse]
+[case testErrorCodeSyntaxError]
 1 ''
 [out]
-main:1: error: Invalid syntax  [syntax]
-[out version==3.10.0]
-main:1: error: Invalid syntax. Perhaps you forgot a comma?  [syntax]
+main:1: error: Simple statements must be separated by newlines or semicolons  [syntax]
 
-[case testErrorCodeSyntaxError2_no_native_parse]
+[case testErrorCodeSyntaxError2]
 def f(): # E: Type signature has too many parameters  [syntax]
     # type: (int) -> None
     1
 
 x = 0  # type: x y  # E: Syntax error in type comment "x y"  [syntax]
 
-[case testErrorCodeSyntaxError3_no_native_parse]
+[case testErrorCodeSyntaxError3]
 # This is a bit inconsistent -- syntax error would be more logical?
 x: 'a b'  # E: Invalid type comment or annotation  [valid-type]
 for v in x:  # type: int, int  # E: Syntax error in type annotation  [syntax] \
@@ -281,7 +279,7 @@ def h(x  # type: xyz  # type: ignore[foo]  # E: Name "xyz" is not defined  [name
 import nostub  # type: ignore[import]
 from defusedxml import xyz  # type: ignore[import]
 
-[case testErrorCodeBadIgnore_no_native_parse]
+[case testErrorCodeBadIgnore]
 import nostub # type: ignore xyz  # E: Invalid "type: ignore" comment  [syntax] \
                                   # E: Cannot find implementation or library stub for module named "nostub"  [import-not-found] \
                                   # N: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
@@ -299,7 +297,7 @@ def f(x,  # type: int  # type: ignore[  # E: Invalid "type: ignore" comment  [sy
     # type: (...) -> None
     pass
 
-[case testErrorCodeBadIgnoreNoExtraComment_no_native_parse]
+[case testErrorCodeBadIgnoreNoExtraComment]
 # Omit the E: ... comments, as they affect parsing
 import nostub # type: ignore xyz
 import nostub # type: ignore[xyz
@@ -846,7 +844,7 @@ main:1: error: Name "y" is not defined  [name-defined]
 main:2: error: Name "ignored" is not defined  [name-defined]
 main:2: error: Name "y" is not defined  [name-defined]
 
-[case testErrorCodeTypeIgnoreMisspelled2_no_native_parse]
+[case testErrorCodeTypeIgnoreMisspelled2]
 x = y  # type: int  # type: ignored[foo]
 x = y  # type: int  # type: ignored [foo]
 [out]
@@ -1098,20 +1096,18 @@ def f(arg: int) -> int:
 def f(arg: str) -> str:
     ...
 
-[case testSliceInDictBuiltin_no_native_parse]
+[case testSliceInDictBuiltin]
 # flags: --show-column-numbers
 b: dict[int, x:y]
 c: dict[x:y]
 
 [builtins fixtures/dict.pyi]
 [out]
 main:2:14: error: Invalid type comment or annotation  [valid-type]
-main:2:14: note: did you mean to use ',' instead of ':' ?
 main:3:4: error: "dict" expects 2 type arguments, but 1 given  [type-arg]
 main:3:9: error: Invalid type comment or annotation  [valid-type]
-main:3:9: note: did you mean to use ',' instead of ':' ?
 
-[case testSliceInDictTyping_no_native_parse]
+[case testSliceInDictTyping]
 # flags: --show-column-numbers
 from typing import Dict
 b: Dict[int, x:y]
@@ -1120,11 +1116,8 @@ c: Dict[x:y]
 [builtins fixtures/dict.pyi]
 [out]
 main:3:14: error: Invalid type comment or annotation  [valid-type]
-main:3:14: note: did you mean to use ',' instead of ':' ?
 main:4:4: error: "dict" expects 2 type arguments, but 1 given  [type-arg]
 main:4:9: error: Invalid type comment or annotation  [valid-type]
-main:4:9: note: did you mean to use ',' instead of ':' ?
-
 
 [case testSliceInCustomTensorType]
 # syntactically mimics torchtyping.TensorType