gh-153195: Allow splitting the PEG grammar into several files (#159917)

Add an @include meta to pegen: the rules of an included grammar file are
appended to the including file's rules. The C generator emits the rules
of each included file into a separate C file next to the main output.

Use it to move the invalid_* rules from Grammar/python.gram into
Grammar/invalid.gram, which generates Parser/parser_invalid.c.

Author: lysnikolaou

.gitattributes

@@ -102,6 +102,7 @@ Modules/_testinternalcapi/test_cases.c.h            generated
 Modules/_testinternalcapi/test_targets.h            generated
 PC/python3dll.c                                     generated
 Parser/parser.c                                     generated
+Parser/parser_invalid.c                             generated
 Parser/token.c                                      generated
 Programs/test_frozenmain.h                          generated
 Python/Python-ast.c                                 generated

.github/CODEOWNERS

@@ -302,6 +302,7 @@ Python/optimizer_symbols.c    @markshannon @tomasr8 @Fidget-Spinner
 
 # Parser, Lexer, and Grammar
 Grammar/python.gram           @pablogsal @lysnikolaou
+Grammar/invalid.gram          @pablogsal @lysnikolaou
 Lib/test/test_peg_generator/  @pablogsal @lysnikolaou
 Lib/test/test_tokenize.py     @pablogsal @lysnikolaou
 Lib/tokenize.py               @pablogsal @lysnikolaou

Grammar/invalid.gram

@@ -13,10 +13,13 @@ Below is a checklist of things that may need to change.
 ## Checklist
 
 * [`Grammar/python.gram`](../Grammar/python.gram): The grammar definition,
-  with actions that build AST nodes.
-  After changing it, run ``make regen-pegen`` (or ``build.bat --regen`` on Windows),
-  to regenerate [`Parser/parser.c`](../Parser/parser.c).
-  (This runs Python's parser generator, [`Tools/peg_generator`](../Tools/peg_generator)).
+  with actions that build AST nodes. The rules for specialized syntax errors
+  (`invalid_*` rules) are in [`Grammar/invalid.gram`](../Grammar/invalid.gram),
+  included by `python.gram`. After changing either, run ``make regen-pegen``
+  (or ``build.bat --regen`` on Windows), to regenerate
+  [`Parser/parser.c`](../Parser/parser.c) and
+  [`Parser/parser_invalid.c`](../Parser/parser_invalid.c). (This runs Python's
+  parser generator, [`Tools/peg_generator`](../Tools/peg_generator)).
 
 * [`Grammar/Tokens`](../Grammar/Tokens) is a place for adding new token types.  After
   changing it, run ``make regen-token`` to regenerate

Grammar/python.gram

@@ -498,6 +498,42 @@ Pegen has some special grammatical elements and rules:
     reporting incorrect locations due to the invalid rules).
   - The order of the alternatives involving invalid rules matter
     (like any rule in PEG).
+  - These rules live in [`Grammar/invalid.gram`](../Grammar/invalid.gram),
+    which is included from [`Grammar/python.gram`](../Grammar/python.gram).
+
+A grammar can be split into several files with the `@include` meta, which
+takes a file name relative to the including file:
+
+```
+@include "invalid.gram"
+```
+
+The rules of each included file are appended, in order, to the rules of the
+including file, so the parser is the same as if the files were concatenated.
+Included files may themselves use `@include`, but may not define other metas
+(such as `@trailer`), and a rule can only be defined in one of the files.
+Includes are resolved by `pegen.build.build_parser()`, which every pegen
+command uses to load a grammar file. A grammar parsed some other way must be
+passed to `pegen.build.resolve_includes()` before generating a parser from it;
+otherwise the generator refuses it.
+
+The C generator emits the rules of each included file into a separate C file,
+with the included file's name appended: for `invalid.gram` that is
+[`Parser/parser_invalid.c`](../Parser/parser_invalid.c). An included file in a
+subdirectory of the root grammar file's directory gets its C file in the same
+subdirectory of the output's directory (`sub/extra.gram` gives
+`Parser/sub/parser_extra.c`); included files outside that directory are
+rejected. This keeps the generated files smaller, and lets them be compiled
+separately. The keyword tables and the `@trailer` code are only in the main C
+file, while the `@header` and `@subheader` code is copied into every C file,
+so it must not define non-`static` functions or variables (they would be defined
+more than once). Rules that are called from another C file, including rules of
+included files that the trailer or a C action calls, can't be `static`; the
+generated code gives them a `_PyPegen_` prefix with a `#define`, since CPython
+requires that prefix on its non-static symbols, and marks them
+`Py_LOCAL_SYMBOL` so that they aren't exported from the shared library. Helper
+rules (for groups, repetitions and so on)
+are never shared between files: each C file gets its own `static` copy.
 
 Tokenization
 ------------

InternalDocs/changing_grammar.md

@@ -1,16 +1,19 @@
 import io
+import textwrap
 import unittest
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from unittest import mock
 
 from test import test_tools
+from test.support import os_helper
 
 test_tools.skip_if_missing("peg_generator")
 with test_tools.imports_under_tool("peg_generator"):
     from pegen import grammar as grammar_module
+    from pegen.build import build_parser, included_output_files
     from pegen.c_generator import CParserGenerator
     from pegen.c_generator_file import CParserEmitter
-    from pegen.grammar import NamedItem, RuleKind
+    from pegen.grammar import GrammarError, NamedItem, RuleKind
     from pegen.grammar_parser import GeneratedParser as GrammarParser
     from pegen.testutil import ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, parse_string
 
@@ -287,5 +290,144 @@ def test_keywords_are_numbered_once(self):
         self.assertEqual(generator.soft_keywords, {"soft"})
 
 
+class TestIncludedCFiles(unittest.TestCase):
+    def setUp(self):
+        self.tmp_dir = self.enterContext(os_helper.temp_dir())
+
+    def write(self, name, source):
+        path = Path(self.tmp_dir, name)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(textwrap.dedent(source))
+        return path
+
+    def generate(self, grammar_file):
+        grammar, _, _ = build_parser(grammar_file)
+        output = io.StringIO()
+        included = {source: io.StringIO() for source in {
+            rule.source for rule in grammar.rules.values() if rule.source is not None
+        }}
+        generator = CParserGenerator(
+            grammar, ALL_TOKENS, EXACT_TOKENS, NON_EXACT_TOKENS, output,
+            included_files=included,
+        )
+        generator.generate(grammar_file)
+        return output.getvalue(), {
+            source.name: file.getvalue() for source, file in included.items()
+        }
+
+    def test_included_rules_go_to_their_own_file(self):
+        main = self.write("main.gram", """\
+            @include "extra.gram"
+            start: item+ NEWLINE
+            item: NAME ('+' | '-') | invalid_item
+            """)
+        self.write("extra.gram", """\
+            invalid_item: NUMBER ('+' | '-') item
+            """)
+        root, included = self.generate(main)
+        extra = included["extra.gram"]
+
+        self.assertTrue(root.startswith("// @generated by pegen from main.gram\n"))
+        self.assertTrue(extra.startswith("// @generated by pegen from extra.gram\n"))
+        # Each rule is defined in the C file of its grammar file, and only
+        # the root file has the keyword tables and the trailer.
+        self.assertIn("\nstart_rule(Parser *p)\n{", root)
+        self.assertNotIn("\ninvalid_item_rule(Parser *p)\n{", root)
+        self.assertIn("\ninvalid_item_rule(Parser *p)\n{", extra)
+        self.assertNotIn("\nitem_rule(Parser *p)\n{", extra)
+        self.assertIn("reserved_keywords", root)
+        self.assertNotIn("reserved_keywords", extra)
+        self.assertIn("_PyPegen_parse", root)
+        self.assertNotIn("_PyPegen_parse", extra)
+        # Rule type ids stay unique across the files.
+        self.assertIn("#define start_type 1000\n", root)
+        self.assertIn("#define item_type 1001\n", root)
+        self.assertIn("#define invalid_item_type 1002\n", extra)
+        # Rules called across files are not static and get a prefixed
+        # linker name; the others stay static.
+        for output in root, extra:
+            self.assertIn("#define item_rule _PyPegen_item_rule\n", output)
+            self.assertIn("#define invalid_item_rule _PyPegen_invalid_item_rule\n", output)
+            self.assertNotIn("_PyPegen_start_rule", output)
+        self.assertIn("static asdl_seq *_loop1_", root)
+        self.assertIn("static void *\nstart_rule(Parser *p)", root)
+        self.assertIn("\nPy_LOCAL_SYMBOL void *\nitem_rule(Parser *p)", root)
+        self.assertIn("\nPy_LOCAL_SYMBOL void *\ninvalid_item_rule(Parser *p)", extra)
+        # A helper for an identical subexpression is not shared between
+        # files: each one has its own static copy.
+        self.assertIn("static void *_tmp_", root)
+        self.assertIn("static void *_tmp_", extra)
+
+    def test_trailer_can_call_rules_of_included_files(self):
+        # The default trailer calls start_rule().
+        main = self.write("main.gram", """\
+            @include "rules.gram"
+            item: NAME
+            """)
+        self.write("rules.gram", "start: item+ NEWLINE\n")
+        root, included = self.generate(main)
+        rules = included["rules.gram"]
+        self.assertIn("return start_rule(p);", root)
+        for output in root, rules:
+            self.assertIn("#define start_rule _PyPegen_start_rule\n", output)
+            self.assertIn("\nPy_LOCAL_SYMBOL void *start_rule(Parser *p);\n", output)
+        self.assertIn("\nPy_LOCAL_SYMBOL void *\nstart_rule(Parser *p)\n{", rules)
+
+    def test_actions_can_call_rules_of_included_files(self):
+        main = self.write("main.gram", """\
+            @include "extra.gram"
+            start: NAME NEWLINE { extra_rule(p) }
+            """)
+        self.write("extra.gram", "extra: NAME\n")
+        root, included = self.generate(main)
+        extra = included["extra.gram"]
+        for output in root, extra:
+            self.assertIn("#define extra_rule _PyPegen_extra_rule\n", output)
+            self.assertIn("\nPy_LOCAL_SYMBOL void *extra_rule(Parser *p);\n", output)
+        self.assertIn("\nPy_LOCAL_SYMBOL void *\nextra_rule(Parser *p)\n{", extra)
+
+    def test_without_includes_all_rules_are_static(self):
+        main = self.write("main.gram", """\
+            start: item NEWLINE
+            item: NAME
+            """)
+        root, included = self.generate(main)
+        self.assertEqual(included, {})
+        self.assertNotIn("Rules shared", root)
+        self.assertNotIn("_PyPegen_item_rule", root)
+        self.assertIn("static void *\nitem_rule(Parser *p)", root)
+
+    def test_included_output_files(self):
+        main = self.write("main.gram", """\
+            @include "extra.gram"
+            @include "sub/more.rules.gram"
+            @include "a/same.gram"
+            @include "b/same.gram"
+            start: extra more one two NEWLINE
+            """)
+        self.write("extra.gram", "extra: NAME\n")
+        self.write("sub/more.rules.gram", "more: NAME\n")
+        self.write("a/same.gram", "one: NAME\n")
+        self.write("b/same.gram", "two: NAME\n")
+        grammar, _, _ = build_parser(main)
+        out = Path("out", "parser.c.new")
+        self.assertEqual(included_output_files(grammar, out), {
+            Path("extra.gram"): Path("out", "parser_extra.c.new"),
+            Path("sub", "more.rules.gram"): Path("out", "sub", "parser_more.rules.c.new"),
+            Path("a", "same.gram"): Path("out", "a", "parser_same.c.new"),
+            Path("b", "same.gram"): Path("out", "b", "parser_same.c.new"),
+        })
+
+    def test_included_output_files_outside_output_directory(self):
+        main = self.write("grammar/main.gram", """\
+            @include "../outside.gram"
+            start: outside NEWLINE
+            """)
+        self.write("outside.gram", "outside: NAME\n")
+        grammar, _, _ = build_parser(main)
+        with self.assertRaisesRegex(GrammarError, "would be outside the output directory"):
+            included_output_files(grammar, Path("parser.c"))
+
+
 if __name__ == "__main__":
     unittest.main()

InternalDocs/parser.md

@@ -18,6 +18,7 @@
 
 test_tools.skip_if_missing("peg_generator")
 with test_tools.imports_under_tool("peg_generator"):
+    from pegen.build import build_parser
     from pegen.grammar_parser import GeneratedParser as GrammarParser
     from pegen.testutil import (
         parse_string,
@@ -139,6 +140,9 @@ def build_extension(self, grammar_source):
 
     def run_test(self, grammar_source, test_source):
         self.build_extension(grammar_source)
+        self.run_extension_test(test_source)
+
+    def run_extension_test(self, test_source):
         test_source = textwrap.indent(textwrap.dedent(test_source), 8 * " ")
         assert_python_ok(
             "-c",
@@ -674,6 +678,36 @@ def test_soft_keywords_lookahead(self) -> None:
         """
         self.run_test(grammar_source, test_source)
 
+    def test_included_grammar_file(self) -> None:
+        # The rules of an included grammar file are compiled from a separate
+        # C file, in the same subdirectory as the grammar file; rules call
+        # each other across the two files.
+        grammar_dir = self.enterContext(os_helper.temp_dir())
+        with open(os.path.join(grammar_dir, "main.gram"), "w") as f:
+            f.write(textwrap.dedent("""\
+                @include "errors/invalid.gram"
+                start: stmt+ NEWLINE? ENDMARKER
+                stmt: invalid_stmt | NAME '=' expr
+                expr: NAME | NUMBER
+                """))
+        os.mkdir(os.path.join(grammar_dir, "errors"))
+        with open(os.path.join(grammar_dir, "errors", "invalid.gram"), "w") as f:
+            f.write(textwrap.dedent("""\
+                invalid_stmt: expr '==' expr { RAISE_SYNTAX_ERROR("use '=' to assign") }
+                """))
+        grammar, _, _ = build_parser(Path(grammar_dir, "main.gram"))
+        generate_parser_c_extension(grammar, Path('.'), library_dir=self.library_dir)
+        self.assertTrue(os.path.exists(os.path.join("errors", "parse_invalid.c")))
+        self.run_extension_test("""
+        self.check_input_strings_for_grammar(
+            valid_cases=["a = b", "a = 1"],
+            invalid_cases=["a =", "a == "],
+        )
+        with self.assertRaises(SyntaxError) as e:
+            parse.parse_string("a == b", mode=0)
+        self.assertIn("use '=' to assign", str(e.exception))
+        """)
+
     def test_forced(self) -> None:
         grammar_source = """
         start: NAME &&':' | NAME