From 4c1538f93a2e34185addddd9f6ffe437a425350a Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 24 Feb 2026 12:36:12 -0800 Subject: [PATCH 01/11] Rewrite Charset so that it uses Expression This removes the `Characters` type, and instead uses `Expression`. There was a lot of overlap with `Expression`, and this simplifies things a little bit and removes some duplication. This also helps with some future changes I am working on that do coverage analysis, and this makes it easier to track coverage data for the different character values. The downside is that without the enum it isn't clear from the code what limitations there are with `ExpressionKind::Charset`, or that `ExpressionKind::CharacterRange` should only be in a `Charset`. These are implicit based on how the parser works. I'm willing to make that tradeoff. The `Characters` map as: - `Characters::Named` to `ExpressionKind::Nt` - `Characters::Terminal` to `ExpressionKind::Terminal` - `Characters::Range` to the NEW `ExpressionKind::CharacterRange` This incidentally fixes a small oversight where `Characters::Named` was missing span with the `grammar-text` class. --- tools/grammar/src/lib.rs | 35 ++++---- tools/grammar/src/parser.rs | 42 +++++----- .../src/grammar/render_markdown.rs | 81 ++++++++----------- .../src/grammar/render_railroad.rs | 62 +++++++------- 4 files changed, 100 insertions(+), 120 deletions(-) diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index c16a07211c..a5833db5b3 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -80,7 +80,14 @@ pub enum ExpressionKind { /// `// Single line comment.` Comment(String), /// ``[`A`-`Z` `_` LF]`` - Charset(Vec), + /// + /// This should only contain expressions that are valid inside brackets + /// (`Terminal`, `Nt`, and `CharacterRange`). + Charset(Vec), + /// `` `A`-`Z` `` used in a character set. + /// + /// This should only appear inside a `Charset`. + CharacterRange(Character, Character), /// ``~[` ` LF]`` NegExpression(Box), /// `^ A B C` @@ -107,16 +114,6 @@ impl Display for RangeLimit { } } -#[derive(Clone, Debug)] -pub enum Characters { - /// `LF` - Named(String), - /// `` `_` `` - Terminal(String), - /// `` `A`-`Z` `` - Range(Character, Character), -} - #[derive(Clone, Debug)] pub enum Character { Char(char), @@ -174,11 +171,14 @@ impl Expression { | ExpressionKind::Cut(e) => { e.visit_nt(callback); } - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { + ExpressionKind::Alt(es) + | ExpressionKind::Sequence(es) + | ExpressionKind::Charset(es) => { for e in es { e.visit_nt(callback); } } + ExpressionKind::Nt(nt) => { callback(&nt); } @@ -186,15 +186,8 @@ impl Expression { | ExpressionKind::Prose(_) | ExpressionKind::Break(_) | ExpressionKind::Comment(_) - | ExpressionKind::Unicode(_) => {} - ExpressionKind::Charset(set) => { - for ch in set { - match ch { - Characters::Named(s) => callback(s), - Characters::Terminal(_) | Characters::Range(_, _) => {} - } - } - } + | ExpressionKind::Unicode(_) + | ExpressionKind::CharacterRange(..) => {} } } diff --git a/tools/grammar/src/parser.rs b/tools/grammar/src/parser.rs index a5a91a5cf0..b7a6d1a0f3 100644 --- a/tools/grammar/src/parser.rs +++ b/tools/grammar/src/parser.rs @@ -1,6 +1,6 @@ //! A parser of the ENBF-like grammar. -use super::{Character, Characters, Expression, ExpressionKind, Grammar, Production, RangeLimit}; +use super::{Character, Expression, ExpressionKind, Grammar, Production, RangeLimit}; use std::fmt; use std::fmt::Display; use std::path::Path; @@ -326,7 +326,7 @@ impl Parser<'_> { let Some(ch) = self.parse_characters()? else { break; }; - characters.push(ch); + characters.push(Expression::new_kind(ch)); } if characters.is_empty() { bail!(self, "expected at least one character in character group"); @@ -338,24 +338,24 @@ impl Parser<'_> { /// Parse an element of a character class, e.g. /// `` `a`-`b` `` | `` `term` `` | `` NonTerminal ``. - fn parse_characters(&mut self) -> Result> { + fn parse_characters(&mut self) -> Result> { if let Some(a) = self.parse_character()? { if self.take_str("-") { let Some(b) = self.parse_character()? else { bail!(self, "expected character in range"); }; - Ok(Some(Characters::Range(a, b))) + Ok(Some(ExpressionKind::CharacterRange(a, b))) } else { //~^ Parse terminal in backticks. let t = match a { Character::Char(ch) => ch.to_string(), Character::Unicode(_) => bail!(self, "unicode not supported"), }; - Ok(Some(Characters::Terminal(t))) + Ok(Some(ExpressionKind::Terminal(t))) } } else if let Some(name) = self.parse_name() { //~^ Parse nonterminal identifier. - Ok(Some(Characters::Named(name))) + Ok(Some(ExpressionKind::Nt(name))) } else { Ok(None) } @@ -600,7 +600,7 @@ fn translate_position(input: &str, index: usize) -> (&str, usize, usize) { #[cfg(test)] mod tests { use crate::parser::{parse_grammar, translate_position}; - use crate::{Character, Characters, ExpressionKind, Grammar, RangeLimit}; + use crate::{Character, ExpressionKind, Grammar, RangeLimit}; use std::path::Path; #[test] @@ -851,8 +851,8 @@ mod tests { panic!("expected Charset inside lookahead, got {:?}", inner.kind); }; assert_eq!(chars.len(), 2); - assert!(matches!(&chars[0], Characters::Terminal(t) if t == "e")); - assert!(matches!(&chars[1], Characters::Terminal(t) if t == "E")); + assert!(matches!(&chars[0].kind, ExpressionKind::Terminal(t) if t == "e")); + assert!(matches!(&chars[1].kind, ExpressionKind::Terminal(t) if t == "E")); } #[test] @@ -1004,7 +1004,7 @@ mod tests { panic!("expected Charset, got {:?}", rule.expression.kind); }; assert_eq!(chars.len(), 1); - let Characters::Range(a, b) = &chars[0] else { + let ExpressionKind::CharacterRange(a, b) = &chars[0].kind else { panic!("expected Range, got {:?}", chars[0]); }; assert!(matches!(a, Character::Unicode((ch, _)) if *ch == '\0')); @@ -1023,7 +1023,7 @@ mod tests { panic!("expected Charset, got {:?}", rule.expression.kind); }; assert_eq!(chars.len(), 1); - let Characters::Range(a, b) = &chars[0] else { + let ExpressionKind::CharacterRange(a, b) = &chars[0].kind else { panic!("expected Range, got {:?}", chars[0]); }; assert!(matches!(a, Character::Char(ch) if *ch == 'a')); @@ -1039,7 +1039,7 @@ mod tests { panic!("expected Charset, got {:?}", rule.expression.kind); }; assert_eq!(chars.len(), 1); - let Characters::Range(a, b) = &chars[0] else { + let ExpressionKind::CharacterRange(a, b) = &chars[0].kind else { panic!("expected Range, got {:?}", chars[0]); }; assert!(matches!(a, Character::Char(ch) if *ch == 'a')); @@ -1058,12 +1058,12 @@ mod tests { panic!("expected Charset, got {:?}", rule.expression.kind); }; assert_eq!(chars.len(), 2); - let Characters::Range(a1, b1) = &chars[0] else { + let ExpressionKind::CharacterRange(a1, b1) = &chars[0].kind else { panic!("expected Range, got {:?}", chars[0]); }; assert!(matches!(a1, Character::Unicode((ch, _)) if *ch == '\0')); assert!(matches!(b1, Character::Unicode((ch, _)) if *ch == '\u{D7FF}')); - let Characters::Range(a2, b2) = &chars[1] else { + let ExpressionKind::CharacterRange(a2, b2) = &chars[1].kind else { panic!("expected Range, got {:?}", chars[1]); }; assert!(matches!(a2, Character::Unicode((ch, _)) if *ch == '\u{E000}')); @@ -1079,9 +1079,9 @@ mod tests { panic!("expected Charset, got {:?}", rule.expression.kind); }; assert_eq!(chars.len(), 3); - assert!(matches!(&chars[0], Characters::Terminal(t) if t == "a")); - assert!(matches!(&chars[1], Characters::Terminal(t) if t == "b")); - assert!(matches!(&chars[2], Characters::Named(n) if n == "Foo")); + assert!(matches!(&chars[0].kind, ExpressionKind::Terminal(t) if t == "a")); + assert!(matches!(&chars[1].kind, ExpressionKind::Terminal(t) if t == "b")); + assert!(matches!(&chars[2].kind, ExpressionKind::Nt(n) if n == "Foo")); } // --- Negative lookahead combined with charset --- @@ -1103,9 +1103,9 @@ mod tests { panic!("expected Charset, got {:?}", inner.kind); }; assert_eq!(chars.len(), 3); - assert!(matches!(&chars[0], Characters::Terminal(t) if t == "x")); - assert!(matches!(&chars[1], Characters::Terminal(t) if t == "y")); - assert!(matches!(&chars[2], Characters::Named(n) if n == "LF")); + assert!(matches!(&chars[0].kind, ExpressionKind::Terminal(t) if t == "x")); + assert!(matches!(&chars[1].kind, ExpressionKind::Terminal(t) if t == "y")); + assert!(matches!(&chars[2].kind, ExpressionKind::Nt(n) if n == "LF")); } // --- Negative lookahead combined with Unicode --- @@ -1125,7 +1125,7 @@ mod tests { panic!("expected Charset, got {:?}", inner.kind); }; assert_eq!(chars.len(), 1); - let Characters::Range(a, b) = &chars[0] else { + let ExpressionKind::CharacterRange(a, b) = &chars[0].kind else { panic!("expected Range, got {:?}", chars[0]); }; assert!(matches!(a, Character::Unicode((ch, _)) if *ch == '\0')); diff --git a/tools/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs index e50f1fd180..89b0d6b35f 100644 --- a/tools/mdbook-spec/src/grammar/render_markdown.rs +++ b/tools/mdbook-spec/src/grammar/render_markdown.rs @@ -3,7 +3,7 @@ use super::RenderCtx; use crate::grammar::Grammar; use anyhow::bail; -use grammar::{Character, Characters, Expression, ExpressionKind, Production}; +use grammar::{Character, Expression, ExpressionKind, Production}; use regex::Regex; use std::borrow::Cow; use std::fmt::Write; @@ -79,6 +79,7 @@ fn last_expr(expr: &Expression) -> &ExpressionKind { | ExpressionKind::Break(_) | ExpressionKind::Comment(_) | ExpressionKind::Charset(_) + | ExpressionKind::CharacterRange(..) | ExpressionKind::NegExpression(_) | ExpressionKind::Unicode(_) => &expr.kind, } @@ -175,6 +176,20 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { write!(output, "// {s}").unwrap(); } ExpressionKind::Charset(set) => charset_render_markdown(cx, set, output), + ExpressionKind::CharacterRange(start, end) => { + let write_ch = |ch: &Character, output: &mut String| match ch { + Character::Char(ch) => write!( + output, + "{}", + markdown_escape(&ch.to_string()) + ) + .unwrap(), + Character::Unicode((_, s)) => write!(output, "U+{s}").unwrap(), + }; + write_ch(start, output); + output.push('-'); + write_ch(end, output); + } ExpressionKind::NegExpression(e) => { output.push('~'); render_expression(e, cx, output); @@ -200,11 +215,11 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { } } -fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut String) { +fn charset_render_markdown(cx: &RenderCtx, set: &[Expression], output: &mut String) { output.push_str("\\["); let mut iter = set.iter().peekable(); - while let Some(chars) = iter.next() { - render_characters(chars, cx, output); + while let Some(expr) = iter.next() { + render_expression(expr, cx, output); if iter.peek().is_some() { output.push(' '); } @@ -212,35 +227,6 @@ fn charset_render_markdown(cx: &RenderCtx, set: &[Characters], output: &mut Stri output.push(']'); } -fn render_characters(chars: &Characters, cx: &RenderCtx, output: &mut String) { - match chars { - Characters::Named(s) => { - let dest = cx.md_link_map.get(s).map_or("missing", |d| d.as_str()); - write!(output, "[{s}]({dest})").unwrap(); - } - Characters::Terminal(s) => write!( - output, - "{}", - markdown_escape(s) - ) - .unwrap(), - Characters::Range(a, b) => { - let write_ch = |ch: &Character, output: &mut String| match ch { - Character::Char(ch) => write!( - output, - "{}", - markdown_escape(&ch.to_string()) - ) - .unwrap(), - Character::Unicode((_, s)) => write!(output, "U+{s}").unwrap(), - }; - write_ch(a, output); - output.push('-'); - write_ch(b, output); - } - } -} - /// Escapes characters that markdown would otherwise interpret. fn markdown_escape(s: &str) -> Cow<'_, str> { static ESC_RE: LazyLock = @@ -302,8 +288,8 @@ mod tests { fn lookahead_charset() { let result = render(ExpressionKind::NegativeLookahead(Box::new( Expression::new_kind(ExpressionKind::Charset(vec![ - Characters::Terminal("e".to_string()), - Characters::Terminal("E".to_string()), + Expression::new_kind(ExpressionKind::Terminal("e".to_string())), + Expression::new_kind(ExpressionKind::Terminal("E".to_string())), ])), ))); assert!(result.starts_with("!"), "should start with `!`"); @@ -349,9 +335,11 @@ mod tests { #[test] fn charset_unicode_range() { - let result = render(ExpressionKind::Charset(vec![Characters::Range( - Character::Unicode(('\0', "0000".to_string())), - Character::Unicode(('\u{007F}', "007F".to_string())), + let result = render(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange( + Character::Unicode(('\0', "0000".to_string())), + Character::Unicode(('\u{007F}', "007F".to_string())), + ), )])); assert!(result.contains("\\[")); assert!(result.contains("U+0000")); @@ -361,9 +349,8 @@ mod tests { #[test] fn charset_char_range() { - let result = render(ExpressionKind::Charset(vec![Characters::Range( - Character::Char('a'), - Character::Char('z'), + let result = render(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange(Character::Char('a'), Character::Char('z')), )])); assert!(result.contains("\\[")); assert!(result.contains("grammar-literal")); @@ -373,9 +360,11 @@ mod tests { #[test] fn charset_mixed_range() { // [`a`-U+007A] - let result = render(ExpressionKind::Charset(vec![Characters::Range( - Character::Char('a'), - Character::Unicode(('\u{007A}', "007A".to_string())), + let result = render(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange( + Character::Char('a'), + Character::Unicode(('\u{007A}', "007A".to_string())), + ), )])); assert!(result.contains("grammar-literal")); assert!(result.contains("U+007A")); @@ -398,8 +387,8 @@ mod tests { #[test] fn neg_expression_rendering() { let result = render(ExpressionKind::NegExpression(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![Characters::Terminal( - "a".to_string(), + Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::Terminal("a".to_string()), )])), ))); assert!( diff --git a/tools/mdbook-spec/src/grammar/render_railroad.rs b/tools/mdbook-spec/src/grammar/render_railroad.rs index 9f4c3c5398..b0ecdd37e9 100644 --- a/tools/mdbook-spec/src/grammar/render_railroad.rs +++ b/tools/mdbook-spec/src/grammar/render_railroad.rs @@ -3,7 +3,7 @@ use super::RenderCtx; use crate::grammar::Grammar; use anyhow::bail; -use grammar::{Character, Characters, Expression, ExpressionKind, Production, RangeLimit}; +use grammar::{Character, Expression, ExpressionKind, Production, RangeLimit}; use railroad::*; use regex::Regex; use std::fmt::Write; @@ -319,8 +319,23 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, stack: bool) -> Option return None, ExpressionKind::Comment(_) => return None, ExpressionKind::Charset(set) => { - let ns: Vec<_> = set.iter().map(|c| render_characters(c, cx)).collect(); - Box::new(bounded_multichoice(ns)) + let choices: Vec<_> = set + .iter() + .map(|e| render_expression(e, cx, stack)) + .filter_map(|n| n) + .collect(); + Box::new(bounded_multichoice(choices)) + } + ExpressionKind::CharacterRange(start, end) => { + let mut s = String::new(); + let write_ch = |ch: &Character, output: &mut String| match ch { + Character::Char(ch) => output.push(*ch), + Character::Unicode((_, s)) => write!(output, "U+{s}").unwrap(), + }; + write_ch(start, &mut s); + s.push('-'); + write_ch(end, &mut s); + Box::new(Terminal::new(s)) } ExpressionKind::NegExpression(e) => { let n = render_expression(e, cx, stack)?; @@ -373,24 +388,6 @@ fn bounded_multichoice(inp: Vec>) -> MultiChoice> { MultiChoice::new(groups.collect()) } -fn render_characters(chars: &Characters, cx: &RenderCtx) -> Box { - match chars { - Characters::Named(s) => node_for_nt(cx, s), - Characters::Terminal(s) => Box::new(Terminal::new(s.clone())), - Characters::Range(a, b) => { - let mut s = String::new(); - let write_ch = |ch: &Character, output: &mut String| match ch { - Character::Char(ch) => output.push(*ch), - Character::Unicode((_, s)) => write!(output, "U+{s}").unwrap(), - }; - write_ch(a, &mut s); - s.push('-'); - write_ch(b, &mut s); - Box::new(Terminal::new(s)) - } - } -} - fn node_for_nt(cx: &RenderCtx, name: &str) -> Box { let dest = cx .rr_link_map @@ -451,7 +448,7 @@ impl Node for Except { #[cfg(test)] mod tests { use super::*; - use grammar::{Character, Characters, Expression, ExpressionKind, RangeLimit}; + use grammar::{Character, Expression, ExpressionKind, RangeLimit}; /// Render an expression to an SVG string fragment. fn render_to_svg(expr: &Expression) -> Option { @@ -586,8 +583,8 @@ mod tests { fn lookahead_charset() { let expr = Expression::new_kind(ExpressionKind::NegativeLookahead(Box::new( Expression::new_kind(ExpressionKind::Charset(vec![ - Characters::Terminal("e".to_string()), - Characters::Terminal("E".to_string()), + Expression::new_kind(ExpressionKind::Terminal("e".to_string())), + Expression::new_kind(ExpressionKind::Terminal("E".to_string())), ])), ))); let svg = render_to_svg(&expr).unwrap(); @@ -619,9 +616,11 @@ mod tests { #[test] fn charset_unicode_range() { - let expr = Expression::new_kind(ExpressionKind::Charset(vec![Characters::Range( - Character::Unicode(('\0', "0000".to_string())), - Character::Unicode(('\u{007F}', "007F".to_string())), + let expr = Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange( + Character::Unicode(('\0', "0000".to_string())), + Character::Unicode(('\u{007F}', "007F".to_string())), + ), )])); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("U+0000")); @@ -630,9 +629,8 @@ mod tests { #[test] fn charset_char_range() { - let expr = Expression::new_kind(ExpressionKind::Charset(vec![Characters::Range( - Character::Char('a'), - Character::Char('z'), + let expr = Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange(Character::Char('a'), Character::Char('z')), )])); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("a")); @@ -659,8 +657,8 @@ mod tests { #[test] fn neg_expression_rendering() { let expr = Expression::new_kind(ExpressionKind::NegExpression(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![Characters::Terminal( - "a".to_string(), + Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::Terminal("a".to_string()), )])), ))); let svg = render_to_svg(&expr).unwrap(); From f8a3bd406f75a9993a4dbc139588a99705d5791a Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Sun, 22 Feb 2026 19:19:02 -0800 Subject: [PATCH 02/11] Add last_expr and is_break helper methods --- tools/grammar/src/lib.rs | 34 ++++++++++++++++++- .../src/grammar/render_markdown.rs | 30 ++-------------- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index a5833db5b3..feca97413d 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -192,7 +192,39 @@ impl Expression { } pub fn is_break(&self) -> bool { - matches!(self.kind, ExpressionKind::Break(_)) + self.kind.is_break() + } + + /// Returns the last [`ExpressionKind`] of this expression. + pub fn last_expr(&self) -> &ExpressionKind { + match &self.kind { + ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => { + es.last().unwrap().last_expr() + } + ExpressionKind::Cut(e) => e.last_expr(), + ExpressionKind::Grouped(_) + | ExpressionKind::Optional(_) + | ExpressionKind::NegativeLookahead(_) + | ExpressionKind::Repeat(_) + | ExpressionKind::RepeatPlus(_) + | ExpressionKind::RepeatRange { .. } + | ExpressionKind::RepeatRangeNamed(_, _) + | ExpressionKind::Nt(_) + | ExpressionKind::Terminal(_) + | ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Charset(_) + | ExpressionKind::CharacterRange(_, _) + | ExpressionKind::NegExpression(_) + | ExpressionKind::Unicode(_) => &self.kind, + } + } +} + +impl ExpressionKind { + pub fn is_break(&self) -> bool { + matches!(self, ExpressionKind::Break(_)) } } diff --git a/tools/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs index 89b0d6b35f..e8a5164f24 100644 --- a/tools/mdbook-spec/src/grammar/render_markdown.rs +++ b/tools/mdbook-spec/src/grammar/render_markdown.rs @@ -61,36 +61,12 @@ fn render_production(prod: &Production, cx: &RenderCtx, output: &mut String) { output.push('\n'); } -/// Returns the last [`ExpressionKind`] of this expression. -fn last_expr(expr: &Expression) -> &ExpressionKind { - match &expr.kind { - ExpressionKind::Alt(es) | ExpressionKind::Sequence(es) => last_expr(es.last().unwrap()), - ExpressionKind::Cut(e) => last_expr(e), - ExpressionKind::Grouped(_) - | ExpressionKind::Optional(_) - | ExpressionKind::NegativeLookahead(_) - | ExpressionKind::Repeat(_) - | ExpressionKind::RepeatPlus(_) - | ExpressionKind::RepeatRange { .. } - | ExpressionKind::RepeatRangeNamed(_, _) - | ExpressionKind::Nt(_) - | ExpressionKind::Terminal(_) - | ExpressionKind::Prose(_) - | ExpressionKind::Break(_) - | ExpressionKind::Comment(_) - | ExpressionKind::Charset(_) - | ExpressionKind::CharacterRange(..) - | ExpressionKind::NegExpression(_) - | ExpressionKind::Unicode(_) => &expr.kind, - } -} - fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { match &expr.kind { ExpressionKind::Grouped(e) => { output.push_str("( "); render_expression(e, cx, output); - if !matches!(last_expr(e), ExpressionKind::Break(_)) { + if !e.last_expr().is_break() { output.push(' '); } output.push(')'); @@ -100,7 +76,7 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { while let Some(e) = iter.next() { render_expression(e, cx, output); if iter.peek().is_some() { - if !matches!(last_expr(e), ExpressionKind::Break(_)) { + if !e.last_expr().is_break() { output.push(' '); } output.push_str("| "); @@ -111,7 +87,7 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, output: &mut String) { let mut iter = es.iter().peekable(); while let Some(e) = iter.next() { render_expression(e, cx, output); - if iter.peek().is_some() && !matches!(last_expr(e), ExpressionKind::Break(_)) { + if iter.peek().is_some() && !e.last_expr().is_break() { output.push(' '); } } From 7243cade24f9203b6a559096575eea1ebcc2f577 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 15:49:08 -0700 Subject: [PATCH 03/11] Update C_STRING_LITERAL to help with parsing This removes the `_except_` notation on C_STRING_LITERAL and uses negative lookahead instead. This is easier for the parser to handle because otherwise it has to special-case the informal notation. --- src/tokens.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tokens.md b/src/tokens.md index 1d48380aad..e142cce06f 100644 --- a/src/tokens.md +++ b/src/tokens.md @@ -345,8 +345,8 @@ r[lex.token.str-c.syntax] C_STRING_LITERAL -> `c"` ^ ( ~[`"` `\` CR NUL] - | BYTE_ESCAPE _except `\0` or `\x00`_ - | UNICODE_ESCAPE _except `\u{0}`, `\u{00}`, …, `\u{000000}`_ + | !(`\0` | `\x00`) BYTE_ESCAPE + | !(`\u{` (`0` `_`*){1..=6} `}`) UNICODE_ESCAPE | STRING_CONTINUE )* `"` SUFFIX? ``` From 05de2fbf6a0093469777a18eafe2de1c11e74811 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 15:57:37 -0700 Subject: [PATCH 04/11] Change keywords list to be part of the grammar This changes the keyword list so that it is written using the grammar syntax. This helps with the grammar validation because the NON_KEYWORD_IDENTIFIER rule needs to refer to these. --- src/keywords.md | 142 +++++++++++++++++++++++++----------------------- 1 file changed, 75 insertions(+), 67 deletions(-) diff --git a/src/keywords.md b/src/keywords.md index 3c143d441d..39cc04c1c1 100644 --- a/src/keywords.md +++ b/src/keywords.md @@ -11,7 +11,7 @@ r[lex.keywords.strict] ## Strict keywords r[lex.keywords.strict.intro] -These keywords can only be used in their correct contexts. They cannot be used as the names of: +Strict keywords can only be used in their correct contexts. They cannot be used as the names of: * [Items] * [Variables] and function parameters @@ -22,48 +22,49 @@ These keywords can only be used in their correct contexts. They cannot be used a * [Macro placeholders] * [Crates] -r[lex.keywords.strict.list] -The following keywords are in all editions: - -- `_` -- `as` -- `async` -- `await` -- `break` -- `const` -- `continue` -- `crate` -- `dyn` -- `else` -- `enum` -- `extern` -- `false` -- `fn` -- `for` -- `if` -- `impl` -- `in` -- `let` -- `loop` -- `match` -- `mod` -- `move` -- `mut` -- `pub` -- `ref` -- `return` -- `self` -- `Self` -- `static` -- `struct` -- `super` -- `trait` -- `true` -- `type` -- `unsafe` -- `use` -- `where` -- `while` +r[lex.keywords.strict.syntax] +```grammar,lexer +@root STRICT_KEYWORDS -> + `_` + | `as` + | `async` + | `await` + | `break` + | `const` + | `continue` + | `crate` + | `dyn` + | `else` + | `enum` + | `extern` + | `false` + | `fn` + | `for` + | `if` + | `impl` + | `in` + | `let` + | `loop` + | `match` + | `mod` + | `move` + | `mut` + | `pub` + | `ref` + | `return` + | `self` + | `Self` + | `static` + | `struct` + | `super` + | `trait` + | `true` + | `type` + | `unsafe` + | `use` + | `where` + | `while` +``` r[lex.keywords.strict.edition2018] > [!EDITION-2018] @@ -77,23 +78,26 @@ r[lex.keywords.reserved] ## Reserved keywords r[lex.keywords.reserved.intro] -These keywords aren't used yet, but they are reserved for future use. They have the same restrictions as strict keywords. The reasoning behind this is to make current programs forward compatible with future versions of Rust by forbidding them to use these keywords. - -r[lex.keywords.reserved.list] -- `abstract` -- `become` -- `box` -- `do` -- `final` -- `gen` -- `macro` -- `override` -- `priv` -- `try` -- `typeof` -- `unsized` -- `virtual` -- `yield` +Reserved keywords aren't used yet, but they are reserved for future use. They have the same restrictions as strict keywords. The reasoning behind this is to make current programs forward compatible with future versions of Rust by forbidding them to use these keywords. + +r[lex.keywords.reserved.syntax] +```grammar,lexer +@root RESERVED_KEYWORDS -> + `abstract` + | `become` + | `box` + | `do` + | `final` + | `gen` + | `macro` + | `override` + | `priv` + | `try` + | `typeof` + | `unsized` + | `virtual` + | `yield` +``` r[lex.keywords.reserved.edition2018] > [!EDITION-2018] @@ -107,13 +111,17 @@ r[lex.keywords.weak] ## Weak keywords r[lex.keywords.weak.intro] -These keywords have special meaning only in certain contexts. For example, it is possible to declare a variable or method with the name `union`. - -- `'static` -- `macro_rules` -- `raw` -- `safe` -- `union` +Weak keywords have special meaning only in certain contexts. For example, it is possible to declare a variable or method with the name `union`. + +r[lex.keywords.weak.syntax] +```grammar,lexer +@root WEAK_KEYWORDS -> + `'static` + | `macro_rules` + | `raw` + | `safe` + | `union` +``` r[lex.keywords.weak.macro_rules] * `macro_rules` is used to create custom [macros]. From 6332b0a6de79ec343a5c875c1011bc37c6d1fa6a Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 16:03:41 -0700 Subject: [PATCH 05/11] Add a Display impl for Expression This helps with debugging to be able to easily see the Expression in the original syntax. --- tools/grammar/src/display.rs | 67 ++++++++++++++++++++++++++++++++++++ tools/grammar/src/lib.rs | 1 + 2 files changed, 68 insertions(+) create mode 100644 tools/grammar/src/display.rs diff --git a/tools/grammar/src/display.rs b/tools/grammar/src/display.rs new file mode 100644 index 0000000000..f231949141 --- /dev/null +++ b/tools/grammar/src/display.rs @@ -0,0 +1,67 @@ +use super::{Expression, ExpressionKind}; +use std::fmt::{Display, Formatter}; + +impl Display for Expression { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + match &self.kind { + ExpressionKind::Grouped(e) => write!(f, "({e})")?, + ExpressionKind::Alt(es) => { + for (i, e) in es.iter().enumerate() { + if i > 0 { + write!(f, " | ")?; + } + write!(f, "{e}")?; + } + } + ExpressionKind::Sequence(es) => { + for (i, e) in es.iter().enumerate() { + if i > 0 { + write!(f, " ")?; + } + write!(f, "{e}")?; + } + } + ExpressionKind::Optional(e) => write!(f, "{e}?")?, + ExpressionKind::NegativeLookahead(e) => write!(f, "!{e}")?, + ExpressionKind::Repeat(e) => write!(f, "{e}*")?, + ExpressionKind::RepeatPlus(e) => write!(f, "{e}+")?, + ExpressionKind::RepeatRange { + expr, + name, + min, + max, + limit, + } => write!( + f, + "{expr}{{{}{}{limit}{}}}", + name.as_ref().map(|n| format!("{n}:")).unwrap_or_default(), + min.map(|v| v.to_string()).unwrap_or_default(), + max.map(|v| v.to_string()).unwrap_or_default(), + )?, + ExpressionKind::RepeatRangeNamed(e, name) => write!(f, "{e}{{{name}}}")?, + ExpressionKind::Nt(s) => write!(f, "{s}")?, + ExpressionKind::Terminal(s) => write!(f, "`{s}`")?, + ExpressionKind::Prose(s) => write!(f, "<{s}>")?, + ExpressionKind::Break(_) => write!(f, " ")?, + ExpressionKind::Comment(_) => {} + ExpressionKind::Charset(es) => { + write!(f, "[")?; + for (i, e) in es.iter().enumerate() { + if i > 0 { + write!(f, " ")?; + } + write!(f, "{e}")?; + } + write!(f, "]")?; + } + ExpressionKind::CharacterRange(start, end) => write!(f, "{start}-{end}")?, + ExpressionKind::NegExpression(e) => write!(f, "~{e}")?, + ExpressionKind::Cut(e) => write!(f, "^ {e}")?, + ExpressionKind::Unicode((_, s)) => write!(f, "U+{s}")?, + } + if let Some(suffix) = &self.suffix { + write!(f, " _{suffix}_")?; + } + Ok(()) + } +} diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index feca97413d..d483616804 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; use std::sync::LazyLock; use walkdir::WalkDir; +mod display; mod parser; #[derive(Debug, Default)] From 6fc2917a0246e0d3c6f4d2d41e04190aa3c0c9d1 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 16:04:39 -0700 Subject: [PATCH 06/11] Clarify ExpressionKind::Unicode doc comment --- tools/grammar/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index d483616804..7947e6fba3 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -94,6 +94,8 @@ pub enum ExpressionKind { /// `^ A B C` Cut(Box), /// `U+0060` + /// + /// The `String` is the hex digits after `U+`. Unicode((char, String)), } From b438ef4457d97176f9f25f304ed19699733f7853 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 16:08:16 -0700 Subject: [PATCH 07/11] Derive Debug for grammar::parser::Error This is intended to help with debugging. --- tools/grammar/src/parser.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/grammar/src/parser.rs b/tools/grammar/src/parser.rs index b7a6d1a0f3..9702e9fc32 100644 --- a/tools/grammar/src/parser.rs +++ b/tools/grammar/src/parser.rs @@ -10,6 +10,7 @@ struct Parser<'a> { index: usize, } +#[derive(Debug)] pub struct Error { message: String, line: String, From ea0bdaffc3475399a230184fd7a69f38c6a0c82d Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 16:25:16 -0700 Subject: [PATCH 08/11] Add a unique ID to each Expression node This adds a unique ID to each Expression node so that the grammar-checker's coverage analysis can track individual expressions. --- tools/grammar/src/lib.rs | 14 +- tools/grammar/src/parser.rs | 72 +++--- .../src/grammar/render_markdown.rs | 51 +++-- .../src/grammar/render_railroad.rs | 211 +++++++++++------- 4 files changed, 212 insertions(+), 136 deletions(-) diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index 7947e6fba3..9712c7d6f6 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -16,6 +16,8 @@ pub struct Grammar { pub productions: HashMap, /// The order that the production names were discovered. pub name_order: Vec, + /// Counter for generating unique expression IDs. + pub next_id: u32, } #[derive(Debug)] @@ -40,6 +42,8 @@ pub struct Expression { pub suffix: Option, /// A footnote is a markdown footnote link. pub footnote: Option, + /// Unique ID of the expression. + pub id: u32, } #[derive(Clone, Debug)] @@ -145,6 +149,13 @@ impl Display for Character { } impl Grammar { + /// Generates a new unique expression ID. + pub fn next_id(&mut self) -> u32 { + let id = self.next_id; + self.next_id += 1; + id + } + fn visit_nt(&self, callback: &mut dyn FnMut(&str)) { for p in self.productions.values() { p.expression.visit_nt(callback); @@ -153,11 +164,12 @@ impl Grammar { } impl Expression { - pub fn new_kind(kind: ExpressionKind) -> Self { + pub fn new_kind(kind: ExpressionKind, id: u32) -> Self { Self { kind, suffix: None, footnote: None, + id, } } diff --git a/tools/grammar/src/parser.rs b/tools/grammar/src/parser.rs index 9702e9fc32..5811266d17 100644 --- a/tools/grammar/src/parser.rs +++ b/tools/grammar/src/parser.rs @@ -8,6 +8,7 @@ use std::path::Path; struct Parser<'a> { input: &'a str, index: usize, + grammar: &'a mut Grammar, } #[derive(Debug)] @@ -63,11 +64,15 @@ pub fn parse_grammar( category: &str, path: &Path, ) -> Result<()> { - let mut parser = Parser { input, index: 0 }; + let mut parser = Parser { + input, + index: 0, + grammar, + }; loop { let p = parser.parse_production(category, path)?; - grammar.name_order.push(p.name.clone()); - if let Some(dupe) = grammar.productions.insert(p.name.clone(), p) { + parser.grammar.name_order.push(p.name.clone()); + if let Some(dupe) = parser.grammar.productions.insert(p.name.clone(), p) { bail!(parser, "duplicate production {} in grammar", dupe.name); } parser.take_while(&|ch| ch == '\n'); @@ -79,6 +84,12 @@ pub fn parse_grammar( } impl Parser<'_> { + /// Helper to create a new expression with a unique ID. + fn new_expr(&mut self, kind: ExpressionKind) -> Expression { + let id = self.grammar.next_id(); + Expression::new_kind(kind, id) + } + fn take_while(&mut self, f: &dyn Fn(char) -> bool) -> &str { let mut upper = 0; let i = self.index; @@ -144,8 +155,8 @@ impl Parser<'_> { let mut comments = Vec::new(); while let Ok(comment) = self.parse_comment() { self.expect("\n", "expected newline")?; - comments.push(Expression::new_kind(comment)); - comments.push(Expression::new_kind(ExpressionKind::Break(0))); + comments.push(self.new_expr(comment)); + comments.push(self.new_expr(ExpressionKind::Break(0))); } let is_root = self.parse_is_root(); self.space0(); @@ -191,7 +202,7 @@ impl Parser<'_> { match es.len() { 0 => Ok(None), 1 => Ok(Some(es.pop().unwrap())), - _ => Ok(Some(Expression::new_kind(ExpressionKind::Alt(es)))), + _ => Ok(Some(self.new_expr(ExpressionKind::Alt(es)))), } } @@ -212,11 +223,7 @@ impl Parser<'_> { match es.len() { 0 => Ok(None), 1 => Ok(Some(es.pop().unwrap())), - _ => Ok(Some(Expression { - kind: ExpressionKind::Sequence(es), - suffix: None, - footnote: None, - })), + _ => Ok(Some(self.new_expr(ExpressionKind::Sequence(es)))), } } @@ -226,11 +233,7 @@ impl Parser<'_> { let Some(rhs) = self.parse_seq()? else { bail!(self, "expected expression after cut operator"); }; - Ok(Expression { - kind: ExpressionKind::Cut(Box::new(rhs)), - suffix: None, - footnote: None, - }) + Ok(self.new_expr(ExpressionKind::Cut(Box::new(rhs)))) } fn parse_expr1(&mut self) -> Result> { @@ -284,11 +287,10 @@ impl Parser<'_> { let suffix = self.parse_suffix()?; let footnote = self.parse_footnote()?; - Ok(Some(Expression { - kind, - suffix, - footnote, - })) + let mut expr = self.new_expr(kind); + expr.suffix = suffix; + expr.footnote = footnote; + Ok(Some(expr)) } fn parse_nonterminal(&mut self) -> Option { @@ -327,7 +329,7 @@ impl Parser<'_> { let Some(ch) = self.parse_characters()? else { break; }; - characters.push(Expression::new_kind(ch)); + characters.push(self.new_expr(ch)); } if characters.is_empty() { bail!(self, "expected at least one character in character group"); @@ -413,7 +415,8 @@ impl Parser<'_> { self.error("expected a charset, terminal, or name after ~ negation".to_string()) })?, }; - Ok(ExpressionKind::NegExpression(box_kind(kind))) + let inner_expr = self.new_expr(kind); + Ok(ExpressionKind::NegExpression(Box::new(inner_expr))) } fn parse_negative_lookahead(&mut self) -> Result { @@ -454,19 +457,22 @@ impl Parser<'_> { /// Parse `?` after expression. fn parse_optional(&mut self, kind: ExpressionKind) -> Result { self.expect("?", "expected `?`")?; - Ok(ExpressionKind::Optional(box_kind(kind))) + let inner_expr = self.new_expr(kind); + Ok(ExpressionKind::Optional(Box::new(inner_expr))) } /// Parse `*` after expression. fn parse_repeat(&mut self, kind: ExpressionKind) -> Result { self.expect("*", "expected `*`")?; - Ok(ExpressionKind::Repeat(box_kind(kind))) + let inner_expr = self.new_expr(kind); + Ok(ExpressionKind::Repeat(Box::new(inner_expr))) } /// Parse `+` after expression. fn parse_repeat_plus(&mut self, kind: ExpressionKind) -> Result { self.expect("+", "expected `+`")?; - Ok(ExpressionKind::RepeatPlus(box_kind(kind))) + let inner_expr = self.new_expr(kind); + Ok(ExpressionKind::RepeatPlus(Box::new(inner_expr))) } /// Parse `{a..b}` | `{a..=b}` | `{name:a..=b}` | `{name}` after expression. @@ -482,7 +488,8 @@ impl Parser<'_> { } (Some(name), Some(b'}')) => { self.index += 1; - return Ok(ExpressionKind::RepeatRangeNamed(box_kind(kind), name)); + let inner_expr = self.new_expr(kind); + return Ok(ExpressionKind::RepeatRangeNamed(Box::new(inner_expr), name)); } _ => { self.index = start; @@ -517,8 +524,9 @@ impl Parser<'_> { _ => {} } self.expect("}", "expected `}`")?; + let inner_expr = self.new_expr(kind); Ok(ExpressionKind::RepeatRange { - expr: box_kind(kind), + expr: Box::new(inner_expr), name, min, max, @@ -569,14 +577,6 @@ impl Parser<'_> { } } -fn box_kind(kind: ExpressionKind) -> Box { - Box::new(Expression { - kind, - suffix: None, - footnote: None, - }) -} - /// Helper to translate a byte index to a `(line, line_no, col_no)` (1-based). fn translate_position(input: &str, index: usize) -> (&str, usize, usize) { if input.is_empty() { diff --git a/tools/mdbook-spec/src/grammar/render_markdown.rs b/tools/mdbook-spec/src/grammar/render_markdown.rs index e8a5164f24..458b992de5 100644 --- a/tools/mdbook-spec/src/grammar/render_markdown.rs +++ b/tools/mdbook-spec/src/grammar/render_markdown.rs @@ -228,7 +228,7 @@ mod tests { /// Renders a single expression to a markdown string. fn render(kind: ExpressionKind) -> String { let cx = test_cx(); - let expr = Expression::new_kind(kind); + let expr = Expression::new_kind(kind, 0); let mut output = String::new(); render_expression(&expr, &cx, &mut output); output @@ -239,7 +239,7 @@ mod tests { #[test] fn lookahead_nonterminal() { let result = render(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Nt("CHAR".to_string())), + Expression::new_kind(ExpressionKind::Nt("CHAR".to_string()), 0), ))); assert!(result.contains("!"), "should contain `!` prefix"); assert!( @@ -251,7 +251,7 @@ mod tests { #[test] fn lookahead_terminal() { let result = render(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Terminal("'".to_string())), + Expression::new_kind(ExpressionKind::Terminal("'".to_string()), 0), ))); assert!(result.starts_with("!"), "should start with `!`"); assert!( @@ -263,10 +263,13 @@ mod tests { #[test] fn lookahead_charset() { let result = render(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![ - Expression::new_kind(ExpressionKind::Terminal("e".to_string())), - Expression::new_kind(ExpressionKind::Terminal("E".to_string())), - ])), + Expression::new_kind( + ExpressionKind::Charset(vec![ + Expression::new_kind(ExpressionKind::Terminal("e".to_string()), 0), + Expression::new_kind(ExpressionKind::Terminal("E".to_string()), 0), + ]), + 0, + ), ))); assert!(result.starts_with("!"), "should start with `!`"); assert!( @@ -278,13 +281,15 @@ mod tests { #[test] fn lookahead_grouped() { // !( `.` | `_` ) - let inner = - ExpressionKind::Grouped(Box::new(Expression::new_kind(ExpressionKind::Alt(vec![ - Expression::new_kind(ExpressionKind::Terminal(".".to_string())), - Expression::new_kind(ExpressionKind::Terminal("_".to_string())), - ])))); + let inner = ExpressionKind::Grouped(Box::new(Expression::new_kind( + ExpressionKind::Alt(vec![ + Expression::new_kind(ExpressionKind::Terminal(".".to_string()), 0), + Expression::new_kind(ExpressionKind::Terminal("_".to_string()), 0), + ]), + 0, + ))); let result = render(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(inner), + Expression::new_kind(inner, 0), ))); assert!(result.starts_with("!(")); assert!(result.contains("|")); @@ -316,6 +321,7 @@ mod tests { Character::Unicode(('\0', "0000".to_string())), Character::Unicode(('\u{007F}', "007F".to_string())), ), + 0, )])); assert!(result.contains("\\[")); assert!(result.contains("U+0000")); @@ -327,6 +333,7 @@ mod tests { fn charset_char_range() { let result = render(ExpressionKind::Charset(vec![Expression::new_kind( ExpressionKind::CharacterRange(Character::Char('a'), Character::Char('z')), + 0, )])); assert!(result.contains("\\[")); assert!(result.contains("grammar-literal")); @@ -341,6 +348,7 @@ mod tests { Character::Char('a'), Character::Unicode(('\u{007A}', "007A".to_string())), ), + 0, )])); assert!(result.contains("grammar-literal")); assert!(result.contains("U+007A")); @@ -353,6 +361,7 @@ mod tests { fn cut_rendering() { let result = render(ExpressionKind::Cut(Box::new(Expression::new_kind( ExpressionKind::Nt("Foo".to_string()), + 0, )))); assert!(result.starts_with("^ "), "cut should render as `^ ` prefix"); assert!(result.contains("Foo")); @@ -363,9 +372,13 @@ mod tests { #[test] fn neg_expression_rendering() { let result = render(ExpressionKind::NegExpression(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( - ExpressionKind::Terminal("a".to_string()), - )])), + Expression::new_kind( + ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::Terminal("a".to_string()), + 0, + )]), + 0, + ), ))); assert!( result.starts_with("~"), @@ -397,7 +410,7 @@ mod tests { fn repeat_range_with_name() { // A RepeatRange with a name renders as `n:1..=255`. let result = render(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()))), + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()), 0)), name: Some("n".to_string()), min: Some(1), max: Some(255), @@ -414,7 +427,7 @@ mod tests { // A RepeatRange without a name renders with no spurious // colon -- just `2..5`. let result = render(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()))), + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()), 0)), name: None, min: Some(2), max: Some(5), @@ -434,7 +447,7 @@ mod tests { fn repeat_range_named_reference() { // A RepeatRangeNamed renders as `n`. let result = render(ExpressionKind::RepeatRangeNamed( - Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()))), + Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()), 0)), "n".to_string(), )); assert!( diff --git a/tools/mdbook-spec/src/grammar/render_railroad.rs b/tools/mdbook-spec/src/grammar/render_railroad.rs index b0ecdd37e9..405e7e2910 100644 --- a/tools/mdbook-spec/src/grammar/render_railroad.rs +++ b/tools/mdbook-spec/src/grammar/render_railroad.rs @@ -240,6 +240,7 @@ fn render_expression(expr: &Expression, cx: &RenderCtx, stack: bool) -> Option Option, max: Option, limit: RangeLimit) -> Expression { - Expression::new_kind(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()))), - name: None, - min, - max, - limit, - }) + Expression::new_kind( + ExpressionKind::RepeatRange { + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()), 0)), + name: None, + min, + max, + limit, + }, + 0, + ) } #[test] @@ -558,9 +565,13 @@ mod tests { #[test] fn lookahead_nonterminal() { - let expr = Expression::new_kind(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Nt("CHAR".to_string())), - ))); + let expr = Expression::new_kind( + ExpressionKind::NegativeLookahead(Box::new(Expression::new_kind( + ExpressionKind::Nt("CHAR".to_string()), + 0, + ))), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("not followed by"), @@ -571,9 +582,13 @@ mod tests { #[test] fn lookahead_terminal() { - let expr = Expression::new_kind(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Terminal("CR".to_string())), - ))); + let expr = Expression::new_kind( + ExpressionKind::NegativeLookahead(Box::new(Expression::new_kind( + ExpressionKind::Terminal("CR".to_string()), + 0, + ))), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("not followed by")); assert!(svg.contains("CR")); @@ -581,12 +596,16 @@ mod tests { #[test] fn lookahead_charset() { - let expr = Expression::new_kind(ExpressionKind::NegativeLookahead(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![ - Expression::new_kind(ExpressionKind::Terminal("e".to_string())), - Expression::new_kind(ExpressionKind::Terminal("E".to_string())), - ])), - ))); + let expr = Expression::new_kind( + ExpressionKind::NegativeLookahead(Box::new(Expression::new_kind( + ExpressionKind::Charset(vec![ + Expression::new_kind(ExpressionKind::Terminal("e".to_string()), 0), + Expression::new_kind(ExpressionKind::Terminal("E".to_string()), 0), + ]), + 0, + ))), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("not followed by")); assert!(svg.contains("e")); @@ -597,17 +616,17 @@ mod tests { #[test] fn unicode_4_digit() { - let expr = Expression::new_kind(ExpressionKind::Unicode(('\t', "0009".to_string()))); + let expr = Expression::new_kind(ExpressionKind::Unicode(('\t', "0009".to_string())), 0); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("U+0009"), "should render Unicode code point"); } #[test] fn unicode_6_digit() { - let expr = Expression::new_kind(ExpressionKind::Unicode(( - '\u{10FFFF}', - "10FFFF".to_string(), - ))); + let expr = Expression::new_kind( + ExpressionKind::Unicode(('\u{10FFFF}', "10FFFF".to_string())), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("U+10FFFF")); } @@ -616,12 +635,16 @@ mod tests { #[test] fn charset_unicode_range() { - let expr = Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( - ExpressionKind::CharacterRange( - Character::Unicode(('\0', "0000".to_string())), - Character::Unicode(('\u{007F}', "007F".to_string())), - ), - )])); + let expr = Expression::new_kind( + ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange( + Character::Unicode(('\0', "0000".to_string())), + Character::Unicode(('\u{007F}', "007F".to_string())), + ), + 0, + )]), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("U+0000")); assert!(svg.contains("U+007F")); @@ -629,9 +652,13 @@ mod tests { #[test] fn charset_char_range() { - let expr = Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( - ExpressionKind::CharacterRange(Character::Char('a'), Character::Char('z')), - )])); + let expr = Expression::new_kind( + ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::CharacterRange(Character::Char('a'), Character::Char('z')), + 0, + )]), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!(svg.contains("a")); assert!(svg.contains("z")); @@ -641,9 +668,13 @@ mod tests { #[test] fn cut_rendering() { - let expr = Expression::new_kind(ExpressionKind::Cut(Box::new(Expression::new_kind( - ExpressionKind::Nt("Foo".to_string()), - )))); + let expr = Expression::new_kind( + ExpressionKind::Cut(Box::new(Expression::new_kind( + ExpressionKind::Nt("Foo".to_string()), + 0, + ))), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("no backtracking"), @@ -656,11 +687,16 @@ mod tests { #[test] fn neg_expression_rendering() { - let expr = Expression::new_kind(ExpressionKind::NegExpression(Box::new( - Expression::new_kind(ExpressionKind::Charset(vec![Expression::new_kind( - ExpressionKind::Terminal("a".to_string()), - )])), - ))); + let expr = Expression::new_kind( + ExpressionKind::NegExpression(Box::new(Expression::new_kind( + ExpressionKind::Charset(vec![Expression::new_kind( + ExpressionKind::Terminal("a".to_string()), + 0, + )]), + 0, + ))), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("with the exception of"), @@ -674,10 +710,13 @@ mod tests { fn repeat_range_named_reference() { // RepeatRangeNamed renders with a "repeat exactly n times" // label. - let expr = Expression::new_kind(ExpressionKind::RepeatRangeNamed( - Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()))), - "n".to_string(), - )); + let expr = Expression::new_kind( + ExpressionKind::RepeatRangeNamed( + Box::new(Expression::new_kind(ExpressionKind::Nt("x".to_string()), 0)), + "n".to_string(), + ), + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("repeat exactly n times"), @@ -688,13 +727,16 @@ mod tests { #[test] fn repeat_range_with_name_renders() { // A named RepeatRange should display the name as a label. - let expr = Expression::new_kind(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()))), - name: Some("n".to_string()), - min: Some(2), - max: Some(5), - limit: RangeLimit::Closed, - }); + let expr = Expression::new_kind( + ExpressionKind::RepeatRange { + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()), 0)), + name: Some("n".to_string()), + min: Some(2), + max: Some(5), + limit: RangeLimit::Closed, + }, + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("repeat count n"), @@ -706,13 +748,16 @@ mod tests { fn repeat_range_with_name_optional() { // `e{k:0..=5}` decomposes to Optional(RepeatRange). The // name label should still appear on the outermost node. - let expr = Expression::new_kind(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()))), - name: Some("k".to_string()), - min: Some(0), - max: Some(5), - limit: RangeLimit::Closed, - }); + let expr = Expression::new_kind( + ExpressionKind::RepeatRange { + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()), 0)), + name: Some("k".to_string()), + min: Some(0), + max: Some(5), + limit: RangeLimit::Closed, + }, + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("repeat count k"), @@ -724,13 +769,16 @@ mod tests { fn repeat_range_without_name_no_label() { // An unnamed RepeatRange should not have a "repeat count" // label. - let expr = Expression::new_kind(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()))), - name: None, - min: Some(2), - max: Some(5), - limit: RangeLimit::Closed, - }); + let expr = Expression::new_kind( + ExpressionKind::RepeatRange { + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()), 0)), + name: None, + min: Some(2), + max: Some(5), + limit: RangeLimit::Closed, + }, + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( !svg.contains("repeat count"), @@ -742,13 +790,16 @@ mod tests { fn repeat_range_with_name_identity() { // `e{n:1..=1}` renders as plain `e` but should still // display the name label. - let expr = Expression::new_kind(ExpressionKind::RepeatRange { - expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()))), - name: Some("n".to_string()), - min: Some(1), - max: Some(1), - limit: RangeLimit::Closed, - }); + let expr = Expression::new_kind( + ExpressionKind::RepeatRange { + expr: Box::new(Expression::new_kind(ExpressionKind::Nt("e".to_string()), 0)), + name: Some("n".to_string()), + min: Some(1), + max: Some(1), + limit: RangeLimit::Closed, + }, + 0, + ); let svg = render_to_svg(&expr).unwrap(); assert!( svg.contains("repeat count n"), From a1de0dc1cda076e700f9caa64ef894613bbd1401 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 16:30:52 -0700 Subject: [PATCH 09/11] Minor code cleanup in grammar Some basic cleanup as suggested by clippy. --- tools/grammar/src/lib.rs | 5 +++-- tools/grammar/src/parser.rs | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index 9712c7d6f6..e2736e218a 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -195,7 +195,7 @@ impl Expression { } ExpressionKind::Nt(nt) => { - callback(&nt); + callback(nt); } ExpressionKind::Terminal(_) | ExpressionKind::Prose(_) @@ -314,7 +314,8 @@ fn check_unexpected_roots(grammar: &Grammar, diag: &mut Diagnostics) { let expected: HashSet<_> = grammar .productions .values() - .filter_map(|p| p.is_root.then(|| p.name.as_str())) + .filter(|&p| p.is_root) + .map(|p| p.name.as_str()) .collect(); if set != expected { let new: Vec<_> = set.difference(&expected).collect(); diff --git a/tools/grammar/src/parser.rs b/tools/grammar/src/parser.rs index 5811266d17..d7a2ccdddb 100644 --- a/tools/grammar/src/parser.rs +++ b/tools/grammar/src/parser.rs @@ -93,8 +93,7 @@ impl Parser<'_> { fn take_while(&mut self, f: &dyn Fn(char) -> bool) -> &str { let mut upper = 0; let i = self.index; - let mut ci = self.input[i..].chars(); - while let Some(ch) = ci.next() { + for ch in self.input[i..].chars() { if !f(ch) { break; } @@ -191,8 +190,7 @@ impl Parser<'_> { fn parse_expression(&mut self) -> Result> { let mut es = Vec::new(); - loop { - let Some(e) = self.parse_seq()? else { break }; + while let Some(e) = self.parse_seq()? { es.push(e); _ = self.space0(); if !self.take_str("|") { From 1621a640c4495026155bf35a37c6f288d0f513c4 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Tue, 11 Aug 2026 20:10:53 -0700 Subject: [PATCH 10/11] Add grammar validation tool This adds a tool that validates the grammar by testing it against other parsers such as rustc and proc-macro2. There are two new crates: - `parser` is an interpreter that takes a source input and generates a tree of nodes based on the Reference grammar. - `grammar-check` is a CLI program for checking the grammar against other parsers. It includes several different subcommands and utilities. This is not entirely complete, but should be sufficient for validating the lexer. --- .github/workflows/daily-grammar-check.yml | 70 ++ .github/workflows/main.yml | 17 + Cargo.lock | 653 +++++++++++--- tools/grammar-check/Cargo.toml | 24 + tools/grammar-check/README.md | 57 ++ .../grammar-check/src/commands/lex_compare.rs | 378 ++++++++ .../src/commands/print_grammar.rs | 29 + .../grammar-check/src/commands/split_check.rs | 441 ++++++++++ tools/grammar-check/src/commands/tokenize.rs | 73 ++ tools/grammar-check/src/commands/tree.rs | 73 ++ tools/grammar-check/src/main.rs | 427 +++++++++ tools/grammar-check/src/permute.rs | 831 ++++++++++++++++++ tools/grammar-check/src/test_cases.rs | 84 ++ tools/grammar-check/src/tools/pm2.rs | 455 ++++++++++ tools/grammar-check/src/tools/rustc.rs | 267 ++++++ tools/grammar-check/src/tools/rustc_lexer.rs | 30 + tools/grammar/src/frontmatter.rs | 55 ++ tools/grammar/src/lib.rs | 9 + tools/parser/Cargo.toml | 12 + tools/parser/README.md | 25 + tools/parser/src/coverage.rs | 601 +++++++++++++ tools/parser/src/lexer.rs | 281 ++++++ tools/parser/src/lib.rs | 119 +++ tools/parser/src/main.rs | 41 + tools/parser/src/parser.rs | 630 +++++++++++++ tools/parser/src/tree.rs | 89 ++ 26 files changed, 5671 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/daily-grammar-check.yml create mode 100644 tools/grammar-check/Cargo.toml create mode 100644 tools/grammar-check/README.md create mode 100644 tools/grammar-check/src/commands/lex_compare.rs create mode 100644 tools/grammar-check/src/commands/print_grammar.rs create mode 100644 tools/grammar-check/src/commands/split_check.rs create mode 100644 tools/grammar-check/src/commands/tokenize.rs create mode 100644 tools/grammar-check/src/commands/tree.rs create mode 100644 tools/grammar-check/src/main.rs create mode 100644 tools/grammar-check/src/permute.rs create mode 100644 tools/grammar-check/src/test_cases.rs create mode 100644 tools/grammar-check/src/tools/pm2.rs create mode 100644 tools/grammar-check/src/tools/rustc.rs create mode 100644 tools/grammar-check/src/tools/rustc_lexer.rs create mode 100644 tools/grammar/src/frontmatter.rs create mode 100644 tools/parser/Cargo.toml create mode 100644 tools/parser/README.md create mode 100644 tools/parser/src/coverage.rs create mode 100644 tools/parser/src/lexer.rs create mode 100644 tools/parser/src/lib.rs create mode 100644 tools/parser/src/main.rs create mode 100644 tools/parser/src/parser.rs create mode 100644 tools/parser/src/tree.rs diff --git a/.github/workflows/daily-grammar-check.yml b/.github/workflows/daily-grammar-check.yml new file mode 100644 index 0000000000..f5e5305d4d --- /dev/null +++ b/.github/workflows/daily-grammar-check.yml @@ -0,0 +1,70 @@ +name: Daily Grammar Check +on: + schedule: + # Run at 4am UTC every day + - cron: '0 4 * * *' + workflow_dispatch: + +jobs: + grammar-check: + if: github.repository == 'rust-lang/reference' + runs-on: ubuntu-latest + steps: + - name: Checkout reference repository + uses: actions/checkout@v7 + + - name: Checkout rust-lang/rust (shallow clone) + uses: actions/checkout@v7 + with: + repository: rust-lang/rust + path: rust + fetch-depth: 1 + + - name: Update rustup + run: rustup self update + + - name: Install Rust nightly + run: | + rustup set profile minimal + rustup toolchain install nightly -c rustc-dev -c llvm-tools + rustup default nightly + + - name: Report versions + run: | + rustup --version + rustc -Vv + + - name: Run grammar check + id: grammar-check + continue-on-error: true + run: | + cargo run --release -p grammar-check -- lex-compare --path rust + cargo run --release -p grammar-check -- lex-compare --permute Token + + - name: Check for existing open issues + if: steps.grammar-check.outcome == 'failure' + id: check-issues + env: + GH_TOKEN: ${{ github.token }} + run: | + # Check if there's already an open issue with the label 'daily-grammar-check' + ISSUE_COUNT=$(gh issue list --label "daily-grammar-check" --state open --json number --jq 'length') + echo "open_issues=$ISSUE_COUNT" >> $GITHUB_OUTPUT + + - name: Create issue on failure + if: steps.grammar-check.outcome == 'failure' && steps.check-issues.outputs.open_issues == '0' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue create \ + --title "Daily Grammar Check Failed - $(date +%Y-%m-%d)" \ + --label "daily-grammar-check" \ + --body "The daily grammar check failed on $(date +%Y-%m-%d). + + **Workflow Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + + **Command:** \`cargo run --release -p grammar-check -- lex-compare --path rust/tests\` + + Please investigate the failure and update the reference as needed. + + This issue was automatically created by the daily grammar check workflow." diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5cde022bad..bed033f0ab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -97,6 +97,22 @@ jobs: run: | rustup --version rustc -Vv + + tools: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Update rustup + run: rustup self update + - name: Install Rust nightly + run: | + rustup set profile minimal + rustup toolchain install nightly -c rustc-dev -c llvm-tools + rustup default nightly + - name: Report versions + run: | + rustup --version + rustc -Vv - name: Verify tools workspace lockfile is current run: cargo update -p mdbook-spec --locked - name: Test tools @@ -165,6 +181,7 @@ jobs: - code-tests - style-tests - mdbook-spec + - tools - dev-guide # preview is explicitly excluded here since it doesn't run on merge runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 6f5b8e5e45..47e2054d73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,24 +4,89 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "bitflags" -version = "2.10.0" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "cfg-if" @@ -29,10 +94,90 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "console" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +dependencies = [ + "encode_unicode", + "libc", + "unicode-width", + "windows-sys", +] + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + [[package]] name = "diagnostics" version = "0.0.0" +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.2" @@ -51,9 +196,33 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.3.0" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] [[package]] name = "getopts" @@ -66,14 +235,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi", - "wasip2", ] [[package]] @@ -86,45 +254,117 @@ dependencies = [ "walkdir", ] +[[package]] +name = "grammar-check" +version = "0.0.0" +dependencies = [ + "clap", + "ctrlc", + "diagnostics", + "grammar", + "indicatif", + "parser", + "proc-macro2", + "regex", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", + "tracing-tree", + "unicode-ident", + "walkdir", +] + [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "indexmap" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown", ] +[[package]] +name = "indicatif" +version = "0.18.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" +dependencies = [ + "console", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linux-raw-sys" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] [[package]] name = "mdbook-core" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39a3873d4afac65583f1acb56ff058df989d5b4a2464bb02c785549727d307ee" +checksum = "8725b7f8e94a5c40a00c907e4006301ba2fc06722de489a0cb19db1823fdf200" dependencies = [ "anyhow", "regex", @@ -136,20 +376,20 @@ dependencies = [ [[package]] name = "mdbook-markdown" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c41bf35212f5d8b83e543aa6a4887dc5709c8489c5fb9ed00f1b51ce1a2cc6" +checksum = "9ca553aa4330b15fa2c706aef373bc714cc719513d1da73c17be3dba208b9aab" dependencies = [ - "pulldown-cmark 0.13.0", + "pulldown-cmark 0.13.4", "regex", "tracing", ] [[package]] name = "mdbook-preprocessor" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d87bf40be0597f26f0822f939a64f02bf92c4655ba04490aadbf83601a013bb" +checksum = "f8e75b08763e31982701d5b2680124bd4bd9bed4a4e1b5a499381320ccae199f" dependencies = [ "anyhow", "mdbook-core", @@ -178,15 +418,69 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.6" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parser" +version = "0.0.0" +dependencies = [ + "diagnostics", + "grammar", + "tracing", + "tracing-subscriber", + "tracing-tree", + "unicode-ident", +] [[package]] name = "pathdiff" @@ -196,15 +490,21 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pin-project-lite" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -224,9 +524,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.0" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" dependencies = [ "bitflags", "memchr", @@ -248,18 +548,18 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "quote" -version = "1.0.42" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] [[package]] name = "r-efi" -version = "5.3.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "railroad" @@ -272,9 +572,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -284,9 +584,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.13" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -295,15 +595,15 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ "bitflags", "errno", @@ -313,10 +613,10 @@ dependencies = [ ] [[package]] -name = "ryu" -version = "1.0.20" +name = "rustversion" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "same-file" @@ -329,15 +629,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -345,46 +645,73 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ "serde_core", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "style-check" version = "0.0.0" @@ -394,9 +721,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -405,9 +743,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.23.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom", @@ -416,11 +754,20 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "toml" -version = "0.9.8" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap", "serde_core", @@ -433,33 +780,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.3" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -474,29 +821,71 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "tracing-tree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac87aa03b6a4d5a7e4810d1a80c19601dbe0f8a837e9177f23af721c7ba7beec" +dependencies = [ + "nu-ansi-term", + "tracing-core", + "tracing-log", + "tracing-subscriber", ] [[package]] name = "unicase" -version = "2.8.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" [[package]] name = "unicode-ident" -version = "1.0.22" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-width" @@ -504,6 +893,24 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "walkdir" version = "2.5.0" @@ -515,12 +922,58 @@ dependencies = [ ] [[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" +name = "wasm-bindgen" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ - "wit-bindgen", + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", ] [[package]] @@ -549,16 +1002,16 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" - -[[package]] -name = "wit-bindgen" -version = "0.46.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "xtask" version = "0.0.0" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/grammar-check/Cargo.toml b/tools/grammar-check/Cargo.toml new file mode 100644 index 0000000000..aafdc49f3f --- /dev/null +++ b/tools/grammar-check/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "grammar-check" +edition = "2024" +license = "MIT OR Apache-2.0" + +[dependencies] +clap = "4.5.53" +ctrlc = "3.5.1" +diagnostics = { path = "../diagnostics" } +grammar = { path = "../grammar" } +indicatif = "0.18.3" +parser = { path = "../parser" } +proc-macro2 = { version = "1.0.103", features = ["span-locations"] } +regex = "1.12.2" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.145" +tracing = "0.1.43" +tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } +tracing-tree = "0.4.1" +unicode-ident = "1.0.22" +walkdir = "2.5.0" + +[package.metadata.rust-analyzer] +rustc_private = true diff --git a/tools/grammar-check/README.md b/tools/grammar-check/README.md new file mode 100644 index 0000000000..80ae093942 --- /dev/null +++ b/tools/grammar-check/README.md @@ -0,0 +1,57 @@ +# Reference grammar checker + +This is a CLI tool for validating the Reference grammar against other parsers (called *tools*). + +## Commands + +There are several different subcommands: + +- `grammar-check lex-compare` — Compare tokenization between implementations. +- `grammar-check tokenize` — Convert source to tokens. +- `grammar-check tree` — Convert source to a tree. + +Pass `--help` for more information. + +It is recommended to run this in the release profile, especially when testing against a large corpus. + +```shell +cargo r -r -- lex-compare --path /path/to/rust/tests +``` + +Some subcommands like `lex-compare` can parse multiple different kinds of sources, like stdin or auto-generated permutations. See the help output for more. + +## Tools + +This tool supports various parsers which are called *tools*. They are: + +- `reference` — The Reference interpreter using the grammar from the Reference. +- `rustc_parse` — The AST parser from `rustc`. +- `rustc_lexer` — The low-level lexer from `rustc`. This generally isn't useful other than doing deeper analysis on rustc. +- `proc-macro2` — The `proc-macro2` crate. + +## Coverage analysis + +The tool can emit an HTML coverage report of the Reference grammar. Run a command like this: + +```shell +cargo r -r -- lex-compare --coverage --permute Token +``` + +Then open `coverage.html` and look at the token rules to see how well they were covered. Green means it was fully covered, yellow was partially covered, and red is not covered at all. You can mouse-over to get a popup that shows more details about each sub-expression. + +Ideally this should have full coverage, but it's not quite there. + +## Edition support + +There are the beginnings of edition support here, but generally it is incomplete. The Reference grammar itself is not Edition-aware. This will take some significant more work to support properly. Ideally the path-based input could parse the compiletest-based headers to figure out which edition to use for each file. + +## AST parsing + +The tree-based parsing is incomplete and needs some work. It can parse a simple individual item (like `struct S;`), but otherwise can't parse general Rust source. It needs work on both the parser itself and the Reference grammar itself. Example command: + +```shell +cargo r -r -- tree --string 'struct S {x: i32}' +``` + +Comparison against other parsers is not implemented. A new `tree-compare` subcommand needs to be added. It will need to somehow be able to compare the trees between the Reference and the tool (either by normalizing, or having a large `match` that would compare every expression kind). + diff --git a/tools/grammar-check/src/commands/lex_compare.rs b/tools/grammar-check/src/commands/lex_compare.rs new file mode 100644 index 0000000000..a9a52580e3 --- /dev/null +++ b/tools/grammar-check/src/commands/lex_compare.rs @@ -0,0 +1,378 @@ +//! Subcommand that compares lexer tokenization between tools. +//! +//! To compare against the Reference grammar, this needs to do some +//! normalization because different tools have different ideas of exactly what +//! is a token, or the exact span of bytes of a token. +//! +//! Unfortunately this does a poor job of handling when both the Reference and +//! the tool fails to parse some input. Ideally it should compare the exact +//! error (or the span of the error), but that would be extremely difficult. + +use crate::CommonOptions; +use crate::tools::{pm2, rustc}; +use crate::{Message, Tool, display_line}; +use clap::ArgMatches; +use diagnostics::Diagnostics; +use grammar::Grammar; +use parser::Edition; +use parser::ParseError; +use parser::coverage::Coverage; +use parser::lexer::Tokens; +use std::cell::RefCell; +use std::ops::Range; +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +const DEFAULT_COMPARE_TOOLS: [Tool; 2] = [Tool::RustcParse, Tool::ProcMacro2]; + +thread_local! { + static PANIC_OUTPUT: RefCell> = const { RefCell::new(None) }; +} + +pub fn compare_parallel(matches: &ArgMatches) { + let start = Instant::now(); + let (opts, receiver) = CommonOptions::new(matches, &DEFAULT_COMPARE_TOOLS); + if opts.tools.iter().any(|t| *t == Tool::Reference) { + panic!("can't compare reference to itself"); + } + if let Some(t) = opts + .tools + .iter() + .find(|t| !DEFAULT_COMPARE_TOOLS.contains(t)) + { + panic!("tool {t} is not supported for comparison"); + } + + std::panic::set_hook(Box::new(|info| { + let payload = info.payload(); + let msg = if let Some(s) = payload.downcast_ref::<&str>() { + s + } else if let Some(s) = payload.downcast_ref::() { + s.as_str() + } else { + "Box" + }; + let location = info + .location() + .map(|l| format!("{}:{}:{}", l.file(), l.line(), l.column())) + .unwrap_or_else(|| "unknown".to_string()); + let thread = std::thread::current(); + let name = thread.name().unwrap_or(""); + let output = format!("thread '{name}' panicked at {location}:\n{msg}"); + + // We print it here as well to ensure it is seen if the thread dies unexpectedly. + // eprintln!("{}", output); + + PANIC_OUTPUT.with(|c| { + *c.borrow_mut() = Some(output); + }); + })); + + let mut diag = Diagnostics::new(); + let grammar = Arc::new(grammar::load_grammar_with_frontmatter(&mut diag)); + let coverage = Arc::new(Mutex::new(Coverage::default())); + + // Spawn threads to run the tests. + let sender = opts.channel.clone(); + let mut thread_count = opts.thread_count; + let opts = Arc::new(Mutex::new(opts)); + for _ in 0..thread_count { + let opts_c = opts.clone(); + let grammar = grammar.clone(); + let coverage = coverage.clone(); + std::thread::spawn(move || { + compare_loop(opts_c, grammar, coverage); + }); + } + ctrlc::set_handler(move || { + sender.send(Message::CtrlC).unwrap(); + }) + .unwrap(); + // Receive results from the threads. + loop { + match receiver.recv().unwrap() { + Message::ThreadComplete => { + thread_count -= 1; + if thread_count == 0 { + break; + } + } + Message::CtrlC => { + break; + } + } + } + + if opts.lock().unwrap().coverage { + coverage.lock().unwrap().save(&grammar); + } + print_final_summary(&opts, start); +} + +fn compare_loop( + opts: Arc>, + grammar: Arc, + final_coverage: Arc>, +) { + let mut coverage = Coverage::default(); + let channel = opts.lock().unwrap().channel.clone(); + let edition = opts.lock().unwrap().edition(); + loop { + let mut opts_l = opts.lock().unwrap(); + let Some((name, src)) = opts_l.next() else { + break; + }; + let tools = opts_l.tools.clone(); + drop(opts_l); + let lexer_result = match std::panic::catch_unwind(AssertUnwindSafe(|| { + parser::lexer::tokenize(&grammar, &mut coverage, &src) + })) { + Ok(r) => r, + Err(_) => { + let panic_msg = PANIC_OUTPUT.with(|c| { + c.borrow_mut() + .take() + .unwrap_or_else(|| "unknown panic".to_string()) + }); + let mut opts_l = opts.lock().unwrap(); + opts_l.errors.push(format!( + "test {name} for reference lexer panicked:\n{panic_msg}" + )); + opts_l.set_progress_err_msg(); + break; + } + }; + + for tool in &*tools { + match std::panic::catch_unwind(|| { + compare_src(lexer_result.clone(), &name, &src, *tool, edition) + }) { + Ok(Ok(())) => {} + Ok(Err(e)) => { + let mut opts_l = opts.lock().unwrap(); + opts_l.errors.push(e); + opts_l.set_progress_err_msg(); + } + Err(_) => { + let panic_msg = PANIC_OUTPUT.with(|c| { + c.borrow_mut() + .take() + .unwrap_or_else(|| "unknown panic".to_string()) + }); + let mut opts_l = opts.lock().unwrap(); + opts_l.errors.push(format!( + "test {name} for tool {tool} panicked:\n{panic_msg}" + )); + opts_l.set_progress_err_msg(); + } + } + let opts_l = opts.lock().unwrap(); + opts_l.progress.inc(1); + } + } + final_coverage.lock().unwrap().merge(coverage); + channel.send(Message::ThreadComplete).unwrap(); +} + +fn compare_src( + lexer_result: Result, + name: &str, + src: &str, + tool: Tool, + edition: Edition, +) -> Result<(), String> { + let (tool_result, mut lexer_result) = match tool { + Tool::RustcParse => { + let lexer_result = lexer_result.and_then(|ts| rustc::normalize(&ts.tokens)); + (rustc::tokenize(src, edition), lexer_result) + } + Tool::ProcMacro2 => { + // Unfortunately proc-macro2 does not handle shebang or + // frontmatter. In order to handle files with that, this replaces + // those with whitespace in order to retain the original byte + // positions. + if let Err(ParseError { message, .. }) = &lexer_result + && message.contains("invalid frontmatter") + { + return Ok(()); + } + let mut stripped_src = String::from(src); + let mut replace = |range: &Range| { + let replacement = "\n".repeat(range.end - range.start); + stripped_src.replace_range(range.clone(), &replacement); + }; + if let Ok(Tokens { + shebang: Some(shebang), + .. + }) = &lexer_result + { + replace(&shebang.range); + } + if let Ok(Tokens { + frontmatter: Some(frontmatter), + .. + }) = &lexer_result + { + replace(&frontmatter.range); + } + let pm2_result = pm2::tokenize(&stripped_src); + pm2::normalize(pm2_result, lexer_result, src) + } + _ => unreachable!(), + }; + if let Ok(tokens) = &lexer_result + && let Some(invalid) = tokens + .iter() + .find(|token| token.name == "RESERVED_TOKEN" || token.name.starts_with("INVALID_")) + { + lexer_result = Err(ParseError { + byte_offset: invalid.range.start, + message: format!("invalid token {}", invalid.name), + }); + } + + match (lexer_result, tool_result) { + (Ok(lex_tokens), Ok(tool_tokens)) => { + let mut lex_iter = lex_tokens.iter(); + let mut tool_iter = tool_tokens.iter(); + loop { + let lex_token = lex_iter.next(); + let tool_token = tool_iter.next(); + match (lex_token, tool_token) { + (Some(lex_token), Some(tool_token)) => { + let lex_text = &src[lex_token.range.clone()]; + let tool_text = &src[tool_token.range.clone()]; + if lex_text != tool_text || lex_token.name != tool_token.name { + return Err(format!( + "error: token mismatch\n\ + test: {name}\n\ + reference token: {:?} {:?} {:?}\n\ + {}\n\ + {tool} token: {:?} {:?} {:?}\n\ + {}", + lex_token.name, + lex_text, + lex_token.range, + display_line(src, &lex_token.range), + tool_token.name, + tool_text, + tool_token.range, + display_line(src, &tool_token.range), + )); + } + } + (None, None) => break, + (Some(lex_token), None) => { + return Err(format!( + "error: reference has more tokens (compared to {tool})\n\ + test: {name}\n\ + reference token: {:?} {:?}\n\ + {}", + lex_token.name, + &src[lex_token.range.clone()], + display_line(src, &lex_token.range), + )); + } + (None, Some(tool_token)) => { + return Err(format!( + "error: {tool} has more tokens (compared to reference grammar)\n\ + test: {name}\n\ + {tool} token: {:?} {:?}\n\ + {}", + tool_token.name, + &src[tool_token.range.clone()], + display_line(src, &tool_token.range), + )); + } + } + } + return Ok(()); + } + (Err(e), Ok(_)) => { + return Err(format!( + "error: reference failed, {tool} passed\n\ + test: {name}\n\ + reference error: {}\n\ + {}", + e.display(src), + display_line( + src, + &Range { + start: e.byte_offset, + end: e.byte_offset + 1 + } + ) + )); + } + (Ok(_), Err(e)) => { + return Err(format!( + "error: {tool} failed, reference passed\n\ + test: {name}\n\ + {tool} error: {}\n\ + {}", + e.display(src), + display_line( + src, + &Range { + start: e.byte_offset, + end: e.byte_offset + 1 + } + ) + )); + } + (Err(_), Err(_)) => { + // Unfortunately getting the error byte offsets to match between + // the reference lexer and the tools is probably just too much + // effort. This means that they could be reporting errors for + // different reasons, but we wouldn't know. + // + // There are some substantial challenges here: + // + // - Cut errors can supersede previous RESERVED_ tokens, making the offset wildly different. + // - Recovery is a problem, for tests that have several errors. + // Example is /rust/tests/ui/rust-2021/reserved-prefixes.rs. + // - rustc ParseError only includes the byte offset of the first error. + // - reference has no recovery. + return Ok(()); + } + } +} + +fn print_final_summary(opts: &Arc>, start: Instant) { + let opts_l = opts.lock().unwrap(); + // Get the actual count of tests run from progress position. + let actual_test_count = opts_l.progress.position() as u32; + opts_l.progress.finish_and_clear(); + if !opts_l.errors.is_empty() { + eprintln!("------------------------------------------------------------"); + for error in &opts_l.errors { + eprintln!( + "{error}\n\ + ------------------------------------------------------------" + ); + } + } + let n_errs = opts_l.errors.len() as u32; + // Use actual test count (from progress) when test_count is 0 (spinner mode). + let total = if opts_l.test_count == 0 { + actual_test_count + } else { + opts_l.test_count + }; + eprintln!("passed: {}", total.saturating_sub(n_errs)); + eprintln!("failed: {n_errs}"); + let elapsed = start.elapsed(); + if elapsed.as_secs() < 60 { + eprintln!("finished in {:.1} seconds", elapsed.as_secs_f64()); + } else { + eprintln!( + "finished in {} minutes {} seconds", + elapsed.as_secs() / 60, + elapsed.as_secs() % 60 + ); + } + if !opts_l.errors.is_empty() { + std::process::exit(1); + } +} diff --git a/tools/grammar-check/src/commands/print_grammar.rs b/tools/grammar-check/src/commands/print_grammar.rs new file mode 100644 index 0000000000..f17d0fcf8b --- /dev/null +++ b/tools/grammar-check/src/commands/print_grammar.rs @@ -0,0 +1,29 @@ +//! Simple subcommand that just spits out the grammar. +//! +//! This is helpful for getting a consolidated capture of all the grammar +//! rules in a plain text format for doing manual analysis and other +//! debugging. + +use clap::ArgMatches; +use diagnostics::Diagnostics; + +pub fn print_grammar(matches: &ArgMatches) { + let debug = matches.get_flag("debug"); + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar(&mut diag); + + if debug { + for name in &grammar.name_order { + let production = grammar.productions.get(name).unwrap(); + println!("{} ->", name); + println!("{:#?}", production.expression); + println!(); + } + } else { + for name in &grammar.name_order { + let production = grammar.productions.get(name).unwrap(); + println!("{} -> {}", name, production.expression); + println!(); + } + } +} diff --git a/tools/grammar-check/src/commands/split_check.rs b/tools/grammar-check/src/commands/split_check.rs new file mode 100644 index 0000000000..8041d71ac6 --- /dev/null +++ b/tools/grammar-check/src/commands/split_check.rs @@ -0,0 +1,441 @@ +//! Experimental subcommand to identify token-splitting locations. +//! +//! This tries to find where multi-character tokens might be candidates to be +//! split into smaller tokens. This has a fairly high false-positive rate, so +//! it can take some manual effort to analyze. +//! +//! The current analysis of places where tokens are split are exhaustively +//! listed in https://github.com/rust-lang/rust/issues/152398. That issue also +//! highlights situations where rustc fails to split tokens (since it has to +//! do it manually). This highlights a situation where it will be difficult to +//! align the reference grammar with rustc, particularly when doing +//! permutation tests. + +use clap::ArgMatches; +use diagnostics::Diagnostics; +use grammar::{Expression, ExpressionKind, Grammar}; +use std::collections::{HashMap, HashSet}; + +// Multi-character tokens that may need to be split +const MULTI_CHAR_TOKENS: &[&str] = &[ + "...", "..=", "<<=", ">>=", "!=", "%=", "&&", "&=", "*=", "+=", "-=", "->", "..", "/=", "::", + "<-", "<<", "<=", "==", "=>", ">=", ">>", "^=", "|=", "||", +]; + +pub fn split_check(_matches: &ArgMatches) { + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar(&mut diag); + + println!("Checking grammar for potential token splitting locations...\n"); + + // Map to store all locations where token splitting may be necessary + // Key: token, Value: list of (production_name, context) + let mut split_locations: HashMap<&str, Vec<(String, String)>> = HashMap::new(); + + // Check each production + for (prod_name, production) in &grammar.productions { + let mut locations_in_prod = Vec::new(); + let mut visited = HashSet::new(); + find_split_locations( + &grammar, + &production.expression, + &mut locations_in_prod, + prod_name, + &mut visited, + ); + + for (token, context) in locations_in_prod { + split_locations + .entry(token) + .or_insert_with(Vec::new) + .push((prod_name.clone(), context)); + } + } + + // Print results grouped by token + if split_locations.is_empty() { + println!("No potential token splitting locations found."); + } else { + for token in MULTI_CHAR_TOKENS { + if let Some(locations) = split_locations.get(token) { + println!("Token: `{}`", token); + println!(" Locations: {}", locations.len()); + for (prod_name, context) in locations { + println!(" - {}: {}", prod_name, context); + } + println!(); + } + } + } +} + +fn find_split_locations<'a>( + grammar: &'a Grammar, + expr: &'a Expression, + locations: &mut Vec<(&'a str, String)>, + current_production: &str, + visited: &mut HashSet, +) { + match &expr.kind { + ExpressionKind::Grouped(e) => { + find_split_locations(grammar, e, locations, current_production, visited); + } + ExpressionKind::Alt(es) => { + for e in es { + find_split_locations(grammar, e, locations, current_production, visited); + } + } + ExpressionKind::Sequence(es) => { + // Check for adjacent elements that might require token splitting + for (i, e) in es.iter().enumerate() { + find_split_locations(grammar, e, locations, current_production, visited); + + // Check if this element could combine with following elements + // Skip non-token-producing elements (Break, Comment) when looking for the next element + if !matches!( + e.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) { + // Find the next token-producing element + for j in (i + 1)..es.len() { + let next = &es[j]; + if !matches!( + next.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) { + check_adjacent_for_splits( + grammar, + e, + next, + locations, + current_production, + ); + break; // Only check the immediate next token-producing element + } + } + } + } + } + ExpressionKind::Optional(e) + | ExpressionKind::NegativeLookahead(e) + | ExpressionKind::NegExpression(e) + | ExpressionKind::Cut(e) => { + find_split_locations(grammar, e, locations, current_production, visited); + } + ExpressionKind::Repeat(e) | ExpressionKind::RepeatPlus(e) => { + find_split_locations(grammar, e, locations, current_production, visited); + // Check if repeating this element could create a multi-char token + check_repeat_for_splits(grammar, e, locations, current_production, "repeat"); + } + ExpressionKind::RepeatRange { expr: e, .. } | ExpressionKind::RepeatRangeNamed(e, _) => { + find_split_locations(grammar, e, locations, current_production, visited); + check_repeat_for_splits(grammar, e, locations, current_production, "repeat range"); + } + ExpressionKind::Nt(_nt) => { + // Don't recurse into nonterminals - we only want to find direct uses + // and adjacent elements within the current production level. + // The main loop in split_check already visits each production. + } + ExpressionKind::Terminal(term) => { + // Check if this terminal is a multi-char token + for &multi_token in MULTI_CHAR_TOKENS { + if term == multi_token { + locations.push(( + multi_token, + format!("direct use of terminal `{}`", multi_token), + )); + } + } + } + ExpressionKind::Prose(_) + | ExpressionKind::Break(_) + | ExpressionKind::Comment(_) + | ExpressionKind::Charset(_) + | ExpressionKind::CharacterRange(..) + | ExpressionKind::Unicode(_) => { + // These don't contribute to token splitting + } + } +} + +fn describe_expression(expr: &Expression) -> String { + match &expr.kind { + ExpressionKind::Nt(nt) => nt.clone(), + ExpressionKind::Terminal(t) => format!("terminal `{}`", t), + ExpressionKind::Optional(e) => format!("optional {}", describe_expression(e)), + ExpressionKind::Grouped(e) => format!("grouped {}", describe_expression(e)), + ExpressionKind::Repeat(e) => format!("{} repeated", describe_expression(e)), + ExpressionKind::RepeatPlus(e) => format!("{} repeated (+)", describe_expression(e)), + ExpressionKind::Alt(_) => "alternative".to_string(), + ExpressionKind::Sequence(_) => "sequence".to_string(), + ExpressionKind::Prose(p) => format!("<{}>", p), + _ => "expression".to_string(), + } +} + +fn check_adjacent_for_splits<'a>( + grammar: &'a Grammar, + left: &'a Expression, + right: &'a Expression, + locations: &mut Vec<(&'a str, String)>, + _current_production: &str, +) { + // Get the possible ending tokens from the left expression + let left_endings = get_possible_endings(grammar, left); + // Get the possible starting tokens from the right expression + let right_starts = get_possible_starts(grammar, right); + + // Get descriptions of the left and right elements + let left_desc = describe_expression(left); + let right_desc = describe_expression(right); + + // Check if any combination could form a multi-char token + for left_end in &left_endings { + for right_start in &right_starts { + let combined = format!("{}{}", left_end, right_start); + for &multi_token in MULTI_CHAR_TOKENS { + if combined == multi_token { + // Exact match - the two elements combine to form the token + locations.push(( + multi_token, + format!( + "{} ends with `{}` and can be immediately followed by {} which can start with `{}`, forming `{}`", + left_desc, left_end, right_desc, right_start, multi_token + ), + )); + } else if combined.starts_with(multi_token) { + // Combined is longer and starts with the token (e.g., "+=" in "+==" for token "+=") + locations.push(( + multi_token, + format!( + "{} ends with `{}` followed by {} starting with `{}` could form `{}`", + left_desc, left_end, right_desc, right_start, multi_token + ), + )); + } else if multi_token.starts_with(&combined) { + // Token is longer than combined (e.g., "+" and "=" is partial for "+=") + // This shouldn't happen since combined should be complete, but keep for completeness + locations.push(( + multi_token, + format!( + "{} ends with `{}` followed by {} starting with `{}` (partial match for `{}`)", + left_desc, left_end, right_desc, right_start, multi_token + ), + )); + } + } + } + } +} + +fn check_repeat_for_splits<'a>( + grammar: &'a Grammar, + expr: &'a Expression, + locations: &mut Vec<(&'a str, String)>, + _current_production: &str, + repeat_type: &str, +) { + // Get the possible endings and starts from the expression + let endings = get_possible_endings(grammar, expr); + let starts = get_possible_starts(grammar, expr); + + let expr_desc = describe_expression(expr); + + // Check if repeating could form a multi-char token + for ending in &endings { + for start in &starts { + let combined = format!("{}{}", ending, start); + for &multi_token in MULTI_CHAR_TOKENS { + if combined == multi_token { + locations.push(( + multi_token, + format!( + "{} (in {}) ends with `{}` and can be immediately followed by another {} which can start with `{}`, forming `{}`", + expr_desc, repeat_type, ending, expr_desc, start, multi_token + ), + )); + } else if combined.starts_with(multi_token) || multi_token.starts_with(&combined) { + locations.push(( + multi_token, + format!( + "{} (in {}) ending with `{}` followed by start `{}` could form `{}`", + expr_desc, repeat_type, ending, start, multi_token + ), + )); + } + } + } + } +} + +fn get_possible_endings(grammar: &Grammar, expr: &Expression) -> HashSet { + let mut endings = HashSet::new(); + get_possible_endings_impl(grammar, expr, &mut endings, &mut HashSet::new()); + endings +} + +fn get_possible_endings_impl( + grammar: &Grammar, + expr: &Expression, + endings: &mut HashSet, + visited: &mut HashSet, +) { + match &expr.kind { + ExpressionKind::Terminal(term) => { + // Extract the last character from the terminal + if let Some(last_ch) = term.chars().last() { + endings.insert(last_ch.to_string()); + } + } + ExpressionKind::Grouped(e) + | ExpressionKind::Optional(e) + | ExpressionKind::NegativeLookahead(e) + | ExpressionKind::Repeat(e) + | ExpressionKind::RepeatPlus(e) + | ExpressionKind::RepeatRange { expr: e, .. } + | ExpressionKind::RepeatRangeNamed(e, _) + | ExpressionKind::NegExpression(e) + | ExpressionKind::Cut(e) => { + get_possible_endings_impl(grammar, e, endings, visited); + } + ExpressionKind::Alt(es) => { + for e in es { + get_possible_endings_impl(grammar, e, endings, visited); + } + } + ExpressionKind::Sequence(es) => { + // The ending comes from the last element in the sequence that produces tokens + // Skip trailing Breaks and Comments + for e in es.iter().rev() { + if !matches!( + e.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) { + get_possible_endings_impl(grammar, e, endings, visited); + break; + } + } + } + ExpressionKind::Nt(nt) => { + if visited.insert(nt.clone()) { + if let Some(prod) = grammar.productions.get(nt) { + get_possible_endings_impl(grammar, &prod.expression, endings, visited); + } + } + } + ExpressionKind::Charset(chars) => { + for ch in chars { + get_possible_endings_impl(grammar, ch, endings, visited); + } + } + ExpressionKind::CharacterRange(a, b) => { + // For ranges, we'll just add the boundary characters + endings.insert(a.get_ch().to_string()); + endings.insert(b.get_ch().to_string()); + } + ExpressionKind::Unicode((ch, _)) => { + endings.insert(ch.to_string()); + } + ExpressionKind::Prose(text) => { + // Handle "Token" prose - it can be any token + if text.to_lowercase().contains("token") { + // Add all characters that could be part of multi-char tokens + for &token in MULTI_CHAR_TOKENS { + for ch in token.chars() { + endings.insert(ch.to_string()); + } + } + } + } + ExpressionKind::Break(_) | ExpressionKind::Comment(_) => { + // These don't produce tokens + } + } +} + +fn get_possible_starts(grammar: &Grammar, expr: &Expression) -> HashSet { + let mut starts = HashSet::new(); + get_possible_starts_impl(grammar, expr, &mut starts, &mut HashSet::new()); + starts +} + +fn get_possible_starts_impl( + grammar: &Grammar, + expr: &Expression, + starts: &mut HashSet, + visited: &mut HashSet, +) { + match &expr.kind { + ExpressionKind::Terminal(term) => { + // Extract the first character from the terminal + if let Some(first_ch) = term.chars().next() { + starts.insert(first_ch.to_string()); + } + } + ExpressionKind::Grouped(e) + | ExpressionKind::NegativeLookahead(e) + | ExpressionKind::Repeat(e) + | ExpressionKind::RepeatPlus(e) + | ExpressionKind::RepeatRange { expr: e, .. } + | ExpressionKind::RepeatRangeNamed(e, _) + | ExpressionKind::NegExpression(e) + | ExpressionKind::Cut(e) => { + get_possible_starts_impl(grammar, e, starts, visited); + } + ExpressionKind::Optional(e) => { + get_possible_starts_impl(grammar, e, starts, visited); + // Optional also means the next element could be the start + } + ExpressionKind::Alt(es) => { + for e in es { + get_possible_starts_impl(grammar, e, starts, visited); + } + } + ExpressionKind::Sequence(es) => { + // The start comes from the first element in the sequence that produces tokens + // Skip leading Breaks and Comments + for e in es.iter() { + if !matches!( + e.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) { + get_possible_starts_impl(grammar, e, starts, visited); + break; + } + } + } + ExpressionKind::Nt(nt) => { + if visited.insert(nt.clone()) { + if let Some(prod) = grammar.productions.get(nt) { + get_possible_starts_impl(grammar, &prod.expression, starts, visited); + } + } + } + ExpressionKind::Charset(chars) => { + for ch in chars { + get_possible_starts_impl(grammar, ch, starts, visited); + } + } + ExpressionKind::CharacterRange(a, b) => { + starts.insert(a.get_ch().to_string()); + starts.insert(b.get_ch().to_string()); + } + ExpressionKind::Unicode((ch, _)) => { + starts.insert(ch.to_string()); + } + ExpressionKind::Prose(text) => { + // Handle "Token" prose - it can be any token + if text.to_lowercase().contains("token") { + // Add all characters that could be part of multi-char tokens + for &token in MULTI_CHAR_TOKENS { + for ch in token.chars() { + starts.insert(ch.to_string()); + } + } + } + } + ExpressionKind::Break(_) | ExpressionKind::Comment(_) => { + // These don't produce tokens + } + } +} diff --git a/tools/grammar-check/src/commands/tokenize.rs b/tools/grammar-check/src/commands/tokenize.rs new file mode 100644 index 0000000000..55785e31dc --- /dev/null +++ b/tools/grammar-check/src/commands/tokenize.rs @@ -0,0 +1,73 @@ +//! A subcommand that converts input to human-readable sequence of tokens. + +use crate::tools::{pm2, rustc, rustc_lexer}; +use crate::{CommonOptions, Tool, display_line}; +use clap::ArgMatches; +use diagnostics::Diagnostics; +use parser::Edition; +use parser::coverage::Coverage; +use parser::lexer::Tokens; +use std::ops::Range; + +pub fn tokenize(matches: &ArgMatches) { + let (mut opts, _) = CommonOptions::new(matches, &[Tool::Reference]); + opts.progress.finish_and_clear(); + for tool in &*opts.tools.clone() { + while let Some((name, src)) = opts.next() { + println!("------------------------------------------------------------"); + println!("tool `{tool}` token results for `{name}`:"); + tokenize_src(&src, *tool, opts.edition()); + println!("------------------------------------------------------------"); + } + } +} + +fn tokenize_src(src: &str, tool: Tool, edition: Edition) { + let tokens = match tool { + Tool::Reference => { + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar_with_frontmatter(&mut diag); + let mut coverage = Coverage::default(); + let tokens = parser::lexer::tokenize(&grammar, &mut coverage, src); + if let Ok(Tokens { + shebang: Some(shebang), + .. + }) = &tokens + { + println!("Shebang in range: {:?}", shebang); + } + if let Ok(Tokens { + frontmatter: Some(frontmatter), + .. + }) = &tokens + { + println!("Frontmatter in range: {:?}", frontmatter); + } + tokens.map(|ts| ts.tokens) + } + Tool::RustcParse => rustc::tokenize(src, edition), + Tool::ProcMacro2 => pm2::tokenize(src), + Tool::RustcLexer => rustc_lexer::tokenize(src), + }; + let tokens = match tokens { + Ok(tokens) => tokens, + Err(e) => { + eprintln!( + "error: {}\n\ + {}", + e.message, + display_line( + src, + &Range { + start: e.byte_offset, + end: e.byte_offset + 1 + } + ) + ); + return; + } + }; + for token in tokens { + println!("{:?}: {}", &src[token.range], token.name); + } +} diff --git a/tools/grammar-check/src/commands/tree.rs b/tools/grammar-check/src/commands/tree.rs new file mode 100644 index 0000000000..e2e6e7ab2f --- /dev/null +++ b/tools/grammar-check/src/commands/tree.rs @@ -0,0 +1,73 @@ +//! A subcommand that converts input to a human-readable tree. +//! +//! The output here is pretty hard to read. It could definitely be improved, +//! or maybe even use an HTML-based output. + +use crate::{CommonOptions, Tool, display_line}; +use clap::ArgMatches; +use diagnostics::Diagnostics; +use std::ops::Range; + +pub fn tree(matches: &ArgMatches) { + let (mut opts, _) = CommonOptions::new(matches, &[Tool::Reference]); + opts.progress.finish_and_clear(); + let production = matches.get_one::("production").unwrap(); + for tool in &*opts.tools.clone() { + while let Some((name, src)) = opts.next() { + println!("------------------------------------------------------------"); + println!("tool `{tool}` tree results for `{name}`:"); + display_tree(&src, *tool, production); + println!("------------------------------------------------------------"); + } + } +} + +fn display_tree(src: &str, tool: Tool, production: &str) { + match tool { + Tool::Reference => display_reference_tree(src, production), + _ => unimplemented!("{tool} not implemented yet"), + } +} + +fn display_reference_tree(src: &str, production: &str) { + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar_with_frontmatter(&mut diag); + let node = match parser::tree::parse(&grammar, src, production) { + Ok(node) => node, + Err(e) => { + eprintln!( + "error: {}\n\ + {}", + e.message, + display_line( + src, + &Range { + start: e.byte_offset, + end: e.byte_offset + 1 + } + ) + ); + std::process::exit(1); + } + }; + display_tree_node(src, &node, 0); +} + +fn display_tree_node(src: &str, node: &parser::Node, indent: usize) { + let node_text = &src[node.range.clone()]; + let display_text = if node_text.len() > 20 { + format!("{}…", &node_text[..20]) + } else { + node_text.to_string() + }; + println!( + "{}{} {:?} {:?}", + " ".repeat(indent), + node.name, + node.range, + display_text + ); + for child in &node.children.0 { + display_tree_node(src, child, indent + 2); + } +} diff --git a/tools/grammar-check/src/main.rs b/tools/grammar-check/src/main.rs new file mode 100644 index 0000000000..12512d26dd --- /dev/null +++ b/tools/grammar-check/src/main.rs @@ -0,0 +1,427 @@ +#![feature(rustc_private)] + +extern crate rustc_interface; +extern crate rustc_span; + +use clap::{Command, arg}; +use diagnostics::Diagnostics; +use indicatif::{ProgressBar, ProgressStyle}; +use parser::Edition; +use std::cmp::min; +use std::fmt::Display; +use std::io::{IsTerminal, Read}; +use std::ops::Range; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::mpsc::{Receiver, Sender, channel}; +use std::time::Duration; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use walkdir::WalkDir; + +mod permute; +mod test_cases; +mod commands { + pub mod lex_compare; + pub mod print_grammar; + pub mod split_check; + pub mod tokenize; + pub mod tree; +} +mod tools { + pub mod pm2; + pub mod rustc; + pub mod rustc_lexer; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Tool { + Reference, + RustcParse, + ProcMacro2, + RustcLexer, +} + +impl FromStr for Tool { + type Err = String; + fn from_str(s: &str) -> Result { + match s { + "reference" => Ok(Tool::Reference), + "rustc_parse" => Ok(Tool::RustcParse), + "proc-macro2" => Ok(Tool::ProcMacro2), + "rustc_lexer" => Ok(Tool::RustcLexer), + _ => Err(format!("invalid tool: {s}")), + } + } +} + +impl Display for Tool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + match self { + Tool::Reference => write!(f, "reference"), + Tool::RustcParse => write!(f, "rustc_parse"), + Tool::ProcMacro2 => write!(f, "proc-macro2"), + Tool::RustcLexer => write!(f, "rustc_lexer"), + } + } +} + +enum Message { + ThreadComplete, + CtrlC, +} + +struct CommonOptions { + strings: Vec<(String, String)>, + paths: Vec, + permute_iter: Option + Send>>>, + tools: Arc>, + edition: Option, + coverage: bool, + test_count: u32, + thread_count: u32, + errors: Vec, + progress: ProgressBar, + channel: Sender, + use_spinner: bool, +} + +impl CommonOptions { + fn new( + matches: &clap::ArgMatches, + default_tools: &[Tool], + ) -> (CommonOptions, Receiver) { + fn map_case(case: &String) -> Vec<(String, String)> { + match case.as_ref() { + "all" => test_cases::LEX_CASES + .iter() + .flat_map(|(name, cases)| { + cases.iter().map(|c| (name.to_string(), c.to_string())) + }) + .collect(), + case_pattern => { + let cs: Vec<_> = test_cases::LEX_CASES + .iter() + .filter(|(name, _)| { + let name_parts: Vec<_> = name.split("::").collect(); + let pattern_parts: Vec<_> = case_pattern.split("::").collect(); + if pattern_parts.len() > name_parts.len() { + return false; + } + name_parts + .iter() + .zip(pattern_parts.iter()) + .all(|(n, p)| n == p) + }) + .flat_map(|(name, cases)| { + cases.iter().map(|c| (name.to_string(), c.to_string())) + }) + .collect(); + if cs.is_empty() { + eprintln!( + "error: case pattern `{case_pattern}` did not match any test cases" + ); + std::process::exit(1); + } + cs + } + } + } + fn map_path(path: &String) -> Result, walkdir::Error> { + WalkDir::new(path) + .into_iter() + .collect::, _>>() + .map(|entries| { + entries + .into_iter() + .filter(|e| e.file_type().is_file()) + .filter(|e| e.path().extension().map(|ext| ext == "rs").unwrap_or(false)) + .map(|e| e.into_path()) + .collect() + }) + } + let mut strings: Vec<_> = matches + .get_many("string") + .map(|ss| { + ss.map(|s: &String| ("CLI string".to_string(), s.to_string())) + .collect() + }) + .unwrap_or_default(); + let cases: Vec<_> = matches + .get_many("case") + .map(|ps| ps.flat_map(map_case).collect()) + .unwrap_or_default(); + strings.extend(cases); + if matches.get_flag("stdin") { + let mut buffer = String::new(); + if std::io::stdin().is_terminal() { + println!("Enter source text:"); + } + std::io::stdin().read_to_string(&mut buffer).unwrap(); + strings.push(("stdin".to_string(), buffer)); + } + let paths: Vec<_> = matches + .get_many("path") + .map(|ps| { + ps.map(map_path) + .collect::, _>>() + .unwrap_or_else(|e| { + eprintln!("error: failed to read path: {}", e); + std::process::exit(1); + }) + .into_iter() + .flatten() + .collect() + }) + .unwrap_or_default(); + + // Handle --permute flag to generate test cases from grammar productions. + let permute_iter = matches.get_one::("permute").map(|permute_name| { + if permute_name == "three" { + return Mutex::new(Box::new(permute::ThreeIterator::new()) + as Box + Send>); + } + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar(&mut diag); + + // Leak the grammar to get a 'static reference for the iterator + let grammar_ref: &'static grammar::Grammar = Box::leak(Box::new(grammar)); + let production = &grammar_ref + .productions + .get(permute_name) + .unwrap_or_else(|| panic!("production `{permute_name}` not found")) + .expression; + + Mutex::new( + Box::new(permute::PermutationIterator::new(grammar_ref, production)) + as Box + Send>, + ) + }); + + let use_spinner = permute_iter.is_some(); + + if strings.is_empty() && paths.is_empty() && permute_iter.is_none() { + strings.extend(map_case(&"all".to_string())); + } + let tools: Vec<_> = matches + .get_many("tool") + .map(|ts| ts.cloned().collect()) + .unwrap_or_else(|| default_tools.to_vec()); + let tools = Arc::new(tools); + let edition = matches + .get_one::("edition") + .map(|e| e.parse::().unwrap()); + for tool in &*tools { + match (tool, edition) { + (Tool::RustcParse, _) => {} + (Tool::Reference, Some(_)) => panic!("reference does not yet support editions"), + (Tool::ProcMacro2, Some(_)) => panic!("proc-macro2 does not support editions"), + (Tool::RustcLexer, Some(_)) => panic!("rustc_lexer is edition agnostic"), + (_, None) => {} + } + } + let coverage = matches.get_flag("coverage"); + // When using permute, we don't know the total count upfront. + let test_count = if use_spinner { + 0 + } else { + ((strings.len() + paths.len()) as u32) * tools.len() as u32 + }; + let available_parallelism = std::thread::available_parallelism().unwrap().get() as u32; + let thread_count = if use_spinner { + available_parallelism + } else { + min(test_count.max(1), available_parallelism) + }; + let progress = if use_spinner { + let p = ProgressBar::new_spinner(); + p.enable_steady_tick(Duration::from_millis(100)); + p + } else { + let p = ProgressBar::new(test_count as u64); + p.enable_steady_tick(Duration::from_millis(200)); + p + }; + progress.set_message("0"); + let (channel, receiver) = channel(); + let opts = CommonOptions { + strings, + paths, + permute_iter, + tools, + edition, + coverage, + test_count, + thread_count, + errors: Vec::new(), + progress, + channel, + use_spinner, + }; + opts.set_progress_style(); + (opts, receiver) + } + + fn next(&mut self) -> Option<(String, String)> { + if let Some((name, src)) = self.strings.pop() { + return Some((name, src)); + } + if let Some(path) = self.paths.pop() { + // TODO: Switch path to a string, not needed as PathBuf anymore. + let contents = std::fs::read_to_string(&path).unwrap(); + let display = format!("{}", path.display()); + return Some((display, contents)); + } + if let Some(ref iter) = self.permute_iter { + if let Ok(mut iter) = iter.lock() { + if let Some(content) = iter.next() { + // println!("{:?}", content); + return Some(("permutation".to_string(), content)); + } + } + } + None + } + + fn set_progress_style(&self) { + let color = if self.errors.is_empty() { + "green" + } else { + "red" + }; + let tick_chars = "🌑🌒🌓🌔🌕🌖🌗🌘"; + if self.use_spinner { + self.progress.set_style( + ProgressStyle::with_template(&format!( + "{{spinner:.green}} [{{elapsed_precise}}] {{pos}} tests — {{msg:.{color}}} failures" + )) + .unwrap() + .tick_chars(tick_chars), + ); + } else { + self.progress.set_style(ProgressStyle::with_template(&format!("{{spinner:.green}} [{{elapsed_precise}}] [{{wide_bar:.blue}}] {{pos}}/{{len}} — {{msg:.{color}}} failures")).unwrap() + .progress_chars("█▉▊▋▌▍▎▏ ") + .tick_chars(tick_chars)); + } + } + + fn set_progress_err_msg(&self) { + self.progress.set_message(format!("{}", self.errors.len())); + self.set_progress_style(); + } + + fn edition(&self) -> Edition { + self.edition.unwrap_or(Edition::Edition2024) + } +} + +fn common_args() -> Vec { + vec![ + arg!(--case ... "internal test cases to compare"), + arg!(--string ... "source string to tokenize"), + arg!(--path ... "path of rust files to compare"), + arg!(--permute "grammar production to generate permutations for"), + arg!(--tool ... "tool to compare").value_parser(clap::value_parser!(Tool)), + arg!(--edition "edition to use"), + arg!(--coverage "record coverage data"), + arg!(--stdin "read input from stdin"), + ] +} + +fn main() { + let filter = tracing_subscriber::EnvFilter::builder() + .with_env_var("GRAMMAR_LOG") + .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into()) + .from_env_lossy(); + + tracing_subscriber::registry() + .with(filter) + .with( + tracing_tree::HierarchicalLayer::new(2) + .with_writer(std::io::stderr) + .with_ansi(std::io::IsTerminal::is_terminal(&std::io::stderr())), + ) + .init(); + + let matches = Command::new("grammar-check") + .subcommand_required(true) + .arg_required_else_help(true) + .subcommand( + Command::new("lex-compare") + .about("Compare tokenization between implementations") + .args(common_args()), + ) + .subcommand( + Command::new("tokenize") + .about("Convert source to tokens") + .args(common_args()), + ) + .subcommand( + Command::new("tree") + .about("Convert source to a tree") + .arg( + arg!(--production "the production name to parse").default_value("Crate"), + ) + .args(common_args()), + ) + .subcommand( + Command::new("split-check") + .about("Check for potential token splitting locations in the grammar"), + ) + .subcommand( + Command::new("print-grammar") + .about("Print the grammar to stdout") + .arg(arg!(--debug "Print using Debug format")), + ) + .get_matches(); + match matches.subcommand() { + Some(("lex-compare", sub_matches)) => { + commands::lex_compare::compare_parallel(sub_matches); + } + Some(("tokenize", sub_matches)) => { + commands::tokenize::tokenize(sub_matches); + } + Some(("tree", sub_matches)) => { + commands::tree::tree(sub_matches); + } + Some(("split-check", sub_matches)) => { + commands::split_check::split_check(sub_matches); + } + Some(("print-grammar", sub_matches)) => { + commands::print_grammar::print_grammar(sub_matches); + } + _ => unreachable!(), + } +} + +/// Helper to translate a byte index to a `(line, line_no, col_no)` (1-based). +fn translate_position(input: &str, index: usize) -> (&str, usize, usize) { + if input.is_empty() { + return ("", 0, 0); + } + let index = index.min(input.len()); + + let mut line_start = 0; + let mut line_number = 0; + for line in input.lines() { + let line_end = line_start + line.len(); + if index >= line_start && index <= line_end { + let column_number = index - line_start + 1; + return (line, line_number + 1, column_number); + } + line_start = line_end + 1; + line_number += 1; + } + ("", line_number + 1, 0) +} + +fn display_line(src: &str, range: &Range) -> String { + let (line, line_no, col_no) = translate_position(src, range.start); + let prefix = format!("{line_no}: "); + let indent = col_no.saturating_sub(1); + let len = (range.end - range.start).min(line.len().saturating_sub(indent)); + let underline = format!("{}{}", " ".repeat(prefix.len() + indent), "━".repeat(len)); + format!("{prefix}{line}\n{underline}\n") +} diff --git a/tools/grammar-check/src/permute.rs b/tools/grammar-check/src/permute.rs new file mode 100644 index 0000000000..c8c3cebfbb --- /dev/null +++ b/tools/grammar-check/src/permute.rs @@ -0,0 +1,831 @@ +//! Permutation-based tests. +//! +//! This attempts to generate exhaustive coverage of the grammar by using +//! permutations of all of the possible inputs to the grammar. This includes +//! both valid and invalid inputs (particularly those that are truncated). +//! +//! It generates representative inputs for some of the expressions. For +//! example, a a repetition generates an output that includes 0, 1, or 2 +//! repetitions of the expression. Or something like "Identifier" just does a +//! few representative values like "a", "ab", and "abb" (with the assumption +//! that the Identifier grammar is already correct). +//! +//! This uses a state machine and is driven using the `Iterator` API to fetch +//! each new input to test. +//! +//! This is incomplete, and I'm not entirely happy with the design. This +//! misses some inputs, particularly invalid ones with unexpected inputs. I +//! intended to spend more time reading +//! https://www.fuzzingbook.org/html/Grammars.html to think of better +//! strategies to stress the parser. +//! +//! However, a coverage-based fuzzer wouldn't necessarily give all the input +//! that I would want because the point of this tool is to compare against +//! rustc. The fuzzer would be fuzzing the Reference grammar, not the rustc +//! parser. We want to get full coverage of *both* those parsers. A fuzzer +//! based on just the Reference coverage wouldn't ensure that the Reference +//! isn't missing something. +//! +//! Known issues: +//! +//! - Permute didn't find that pm2 doesn't error on `prefix'x'` because it was +//! only generating `prefix'`. Any ideas on how to generate tests that +//! exercise this? + +use grammar::{Expression, ExpressionKind, Grammar, RangeLimit}; +use std::collections::HashMap; + +pub struct PermutationIterator<'g> { + pub grammar: &'g Grammar, + name_context: HashMap, + state: IteratorState<'g>, +} + +enum IteratorState<'g> { + Terminal { + value: String, + done: bool, + }, + Seq { + exprs: Vec<&'g Expression>, + /// Current active length; counts down from exprs.len() to 1 to emit truncated sequences. + truncated_len: usize, + iterators: Vec>, + current_values: Vec, + initialized: bool, + exhausted: bool, + }, + SeqWithNamedRanges { + exprs: Vec<&'g Expression>, + named_range_indices: Vec<(usize, String, usize, usize)>, // (index, name, min, max) + current_named_values: HashMap, + iterators: Vec>, + current_values: Vec, + exhausted: bool, + }, + Alt { + iterators: Vec>, + current_index: usize, + }, + Optional { + iterator: Box>, + emitted_empty: bool, + }, + Repeat { + expr: &'g Expression, + include_empty: bool, + current_stage: usize, // 0 = empty (if include_empty), 1 = single, 2 = double + iterator: Option>>, + }, + RepeatRange { + expr: &'g Expression, + max: usize, + current_count: usize, + iterator: Option>>, + pending_repeat: Option<(String, usize)>, // (value, times_left_to_emit) + }, +} + +impl<'g> PermutationIterator<'g> { + pub fn new(grammar: &'g Grammar, expression: &'g Expression) -> PermutationIterator<'g> { + Self::new_with_context(grammar, expression, HashMap::new()) + } + + fn new_with_context( + grammar: &'g Grammar, + expression: &'g Expression, + name_context: HashMap, + ) -> PermutationIterator<'g> { + let state = match &expression.kind { + ExpressionKind::Alt(exprs) => { + let iterators: Vec<_> = exprs + .iter() + .map(|e| Self::new_with_context(grammar, e, name_context.clone())) + .collect(); + IteratorState::Alt { + iterators, + current_index: 0, + } + } + ExpressionKind::Grouped(expr) => { + return Self::new_with_context(grammar, expr, name_context); + } + ExpressionKind::Sequence(exprs) => { + if exprs.is_empty() { + IteratorState::Terminal { + value: String::new(), + done: false, + } + } else { + let filtered_exprs: Vec<&Expression> = exprs + .iter() + .filter(|e| { + !matches!( + e.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) + }) + .collect(); + + // Check if any expressions are named repeat ranges + let mut named_range_indices = Vec::new(); + for (idx, expr) in filtered_exprs.iter().enumerate() { + if let ExpressionKind::RepeatRange { + name: Some(name), + min, + max, + limit, + .. + } = &expr.kind + { + let min_count = min.unwrap_or(0) as usize; + let max_count = match max { + Some(m) => match limit { + RangeLimit::HalfOpen => *m as usize, + RangeLimit::Closed => (*m + 1) as usize, + }, + None => min_count + 3, + }; + named_range_indices.push((idx, name.clone(), min_count, max_count)); + } + } + + if named_range_indices.is_empty() { + // No named ranges, use regular Seq + let n = filtered_exprs.len(); + + IteratorState::Seq { + exprs: filtered_exprs, + truncated_len: n, + iterators: Vec::new(), + current_values: Vec::new(), + initialized: false, + exhausted: false, + } + } else { + // Has named ranges, use special handling + let current_named_values: HashMap = named_range_indices + .iter() + .map(|(_, name, min, _)| (name.clone(), *min)) + .collect(); + + IteratorState::SeqWithNamedRanges { + exprs: filtered_exprs, + named_range_indices, + current_named_values, + iterators: Vec::new(), + current_values: Vec::new(), + exhausted: false, + } + } + } + } + ExpressionKind::Optional(expr) => { + let iterator = + Box::new(Self::new_with_context(grammar, expr, name_context.clone())); + IteratorState::Optional { + iterator, + emitted_empty: false, + } + } + ExpressionKind::NegativeLookahead(expr) => { + let iterator = + Box::new(Self::new_with_context(grammar, expr, name_context.clone())); + IteratorState::Optional { + iterator, + emitted_empty: false, + } + } + ExpressionKind::Repeat(expr) | ExpressionKind::RepeatPlus(expr) => { + IteratorState::Repeat { + expr, + include_empty: true, + current_stage: 0, + iterator: None, + } + } + ExpressionKind::RepeatRange { + expr, + name, + min, + max, + limit, + } => { + // If this has a name and it's in the context, use that specific count + if let Some(name) = name { + if let Some(&count) = name_context.get(name) { + // Use the specified count from context + IteratorState::RepeatRange { + expr, + max: count + 1, + current_count: count, + iterator: None, + pending_repeat: None, + } + } else { + // Name not in context yet, this shouldn't happen in SeqWithNamedRanges + // but handle it anyway + let min_count = min.unwrap_or(0) as usize; + let max_count = match max { + Some(m) => match limit { + RangeLimit::HalfOpen => *m as usize, + RangeLimit::Closed => (*m + 1) as usize, + }, + None => min_count + 3, + }; + let start_count = if min_count == 0 { 0 } else { min_count }; + IteratorState::RepeatRange { + expr, + max: max_count, + current_count: start_count, + iterator: None, + pending_repeat: None, + } + } + } else { + // No name, normal behavior + let min_count = min.unwrap_or(0) as usize; + let max_count = match max { + Some(m) => match limit { + RangeLimit::HalfOpen => *m as usize, + RangeLimit::Closed => (*m + 1) as usize, + }, + None => min_count + 3, + }; + let start_count = if min_count == 0 { 0 } else { min_count }; + IteratorState::RepeatRange { + expr, + max: max_count, + current_count: start_count, + iterator: None, + pending_repeat: None, + } + } + } + ExpressionKind::RepeatRangeNamed(expr, name) => { + // Look up the count from the context + let count = name_context.get(name).copied().unwrap_or(1); + IteratorState::RepeatRange { + expr, + max: count + 1, + current_count: count, + iterator: None, + pending_repeat: None, + } + } + ExpressionKind::Nt(name) => { + let prod = grammar.productions.get(name).unwrap(); + return Self::new_with_context(grammar, &prod.expression, name_context); + } + ExpressionKind::Terminal(s) => IteratorState::Terminal { + value: s.clone(), + done: false, + }, + ExpressionKind::Prose(prose) => match prose.as_str() { + "`XID_Start` defined by Unicode" => IteratorState::Terminal { + value: "a".to_string(), + done: false, + }, + "`XID_Continue` defined by Unicode" => IteratorState::Terminal { + value: "b".to_string(), + done: false, + }, + _ => panic!("prose {prose} not supported"), + }, + ExpressionKind::Break(_) => unreachable!(), + ExpressionKind::Comment(_) => unreachable!(), + ExpressionKind::Charset(chars) => { + let iterators: Vec<_> = chars + .iter() + .map(|e| Self::new_with_context(grammar, e, name_context.clone())) + .collect(); + IteratorState::Alt { + iterators, + current_index: 0, + } + } + ExpressionKind::CharacterRange(start, end) => { + // Behave like Alt of start and end characters + let mut iterators = Vec::new(); + let start_ch = start.get_ch(); + let end_ch = end.get_ch(); + iterators.push(PermutationIterator { + grammar, + name_context: name_context.clone(), + state: IteratorState::Terminal { + value: start_ch.to_string(), + done: false, + }, + }); + iterators.push(PermutationIterator { + grammar, + name_context: name_context.clone(), + state: IteratorState::Terminal { + value: end_ch.to_string(), + done: false, + }, + }); + IteratorState::Alt { + iterators, + current_index: 0, + } + } + ExpressionKind::NegExpression(_expr) => IteratorState::Terminal { + value: String::from("a"), // TODO: Comment here why this choice. + done: false, + }, + ExpressionKind::Cut(expr) => { + return Self::new_with_context(grammar, expr, name_context); + } + ExpressionKind::Unicode((ch, _)) => IteratorState::Terminal { + value: ch.to_string(), + done: false, + }, + }; + PermutationIterator { + grammar, + name_context, + state, + } + } +} + +impl<'g> Iterator for PermutationIterator<'g> { + type Item = String; + + fn next(&mut self) -> Option { + // Capture grammar reference before mutably borrowing state + let grammar = self.grammar; + + match &mut self.state { + IteratorState::Terminal { value, done } => { + if *done { + None + } else { + *done = true; + Some(value.clone()) + } + } + IteratorState::Alt { + iterators, + current_index, + } => { + while *current_index < iterators.len() { + if let Some(val) = iterators[*current_index].next() { + return Some(val); + } + *current_index += 1; + } + None + } + IteratorState::Optional { + iterator, + emitted_empty, + } => { + if !*emitted_empty { + *emitted_empty = true; + return Some(String::new()); + } + iterator.next() + } + IteratorState::Repeat { + expr, + include_empty, + current_stage, + iterator, + } => { + // Stage 0: emit empty string (only for Repeat, not RepeatPlus) + if *current_stage == 0 && *include_empty { + *current_stage = 1; + return Some(String::new()); + } + + // Stage 1: emit single permutations + if *current_stage == 1 { + if iterator.is_none() { + *iterator = Some(Box::new(Self::new_with_context( + grammar, + expr, + self.name_context.clone(), + ))); + } + + if let Some(iter) = iterator { + if let Some(result) = iter.next() { + return Some(result); + } + } + + // Stage 1 complete, move to stage 2 + *current_stage = 2; + *iterator = Some(Box::new(Self::new_with_context( + grammar, + expr, + self.name_context.clone(), + ))); + } + + // Stage 2: emit double permutations (each element repeated twice) + if *current_stage == 2 { + // Get next single value and prepare to emit it twice + if let Some(iter) = iterator { + if let Some(val) = iter.next() { + let doubled = format!("{val}{val}"); + return Some(doubled); + } + } + } + + None + } + IteratorState::RepeatRange { + expr, + max, + current_count, + iterator, + pending_repeat, + } => { + // If we're at count 0 (min was 0), emit empty string + if *current_count == 0 { + *current_count = 1; + if *current_count >= *max { + return None; + } + return Some(String::new()); + } + + loop { + // If we haven't reached max count yet + if *current_count >= *max { + return None; + } + + // Check if we have a pending repeat to emit + if let Some((val, times_left)) = pending_repeat { + if *times_left > 1 { + *times_left -= 1; + return Some(val.clone()); + } else { + // Emit last repetition and clear pending + let result = val.clone(); + *pending_repeat = None; + return Some(result); + } + } + + // Initialize iterator for current count if needed + if iterator.is_none() { + *iterator = Some(Box::new(Self::new_with_context( + grammar, + expr, + self.name_context.clone(), + ))); + } + + // Try to get next value from iterator + if let Some(iter) = iterator { + if let Some(val) = iter.next() { + let result = val.repeat(*current_count); + return Some(result); + } + } + + // Current count exhausted, move to next + *current_count += 1; + *iterator = None; + } + } + IteratorState::Seq { + exprs, + truncated_len, + iterators, + current_values, + initialized, + exhausted, + } => { + if *exhausted { + return None; + } + + loop { + // (Re)initialize iterators for the current truncated_len. + if !*initialized { + let tlen = *truncated_len; + *iterators = exprs[..tlen] + .iter() + .map(|e| Self::new_with_context(grammar, e, self.name_context.clone())) + .collect(); + *current_values = vec![String::new(); tlen]; + + // Get first value from each iterator. + let mut ok = true; + for (i, iter) in iterators.iter_mut().enumerate() { + if let Some(val) = iter.next() { + current_values[i] = val; + } else { + ok = false; + break; + } + } + + if ok { + *initialized = true; + return Some(current_values.concat()); + } else { + // Empty iterator at this length; try shorter. + if *truncated_len > 1 { + *truncated_len -= 1; + continue; + } else { + *exhausted = true; + return None; + } + } + } + + // Try to advance the rightmost iterator. + let mut pos = iterators.len() - 1; + let mut advanced = false; + loop { + if let Some(val) = iterators[pos].next() { + current_values[pos] = val; + advanced = true; + break; + } else { + // This iterator is exhausted; reset it and move left. + if pos == 0 { + // All iterators for this truncated_len are exhausted. + break; + } + iterators[pos] = Self::new_with_context( + grammar, + &exprs[pos], + self.name_context.clone(), + ); + if let Some(val) = iterators[pos].next() { + current_values[pos] = val; + } + pos -= 1; + } + } + + if advanced { + return Some(current_values.concat()); + } + + // Current length exhausted; move to the next shorter truncation. + if *truncated_len > 1 { + *truncated_len -= 1; + *initialized = false; + } else { + *exhausted = true; + return None; + } + } + } + IteratorState::SeqWithNamedRanges { + exprs, + named_range_indices, + current_named_values, + iterators, + current_values, + exhausted, + } => { + if *exhausted { + return None; + } + + loop { + // Initialize iterators if needed + if iterators.is_empty() { + for expr in exprs.iter() { + iterators.push(Self::new_with_context( + grammar, + expr, + current_named_values.clone(), + )); + } + *current_values = iterators.iter().map(|_| String::new()).collect(); + + // Get first value from each iterator + for (i, iter) in iterators.iter_mut().enumerate() { + if let Some(val) = iter.next() { + current_values[i] = val; + } else { + // Empty iterator, try next name values + break; + } + } + + if current_values.iter().all(|v| !v.is_empty()) { + return Some(current_values.concat()); + } + } + + // Try to advance rightmost iterator + let mut pos = iterators.len().checked_sub(1)?; + loop { + if let Some(val) = iterators[pos].next() { + current_values[pos] = val; + return Some(current_values.concat()); + } else { + // This iterator exhausted + if pos == 0 { + // All iterators for this name combo exhausted + // Try to increment named values (only min and max, not values in between) + let mut incremented = false; + for (_idx, name, min, max) in named_range_indices.iter().rev() { + let current_val = + current_named_values.get(name).copied().unwrap_or(*min); + // Only generate for min and max values + if current_val == *min && *min + 1 < *max { + // Jump from min to max-1 (which is the actual max value since max is exclusive) + current_named_values.insert(name.clone(), *max - 1); + incremented = true; + break; + } else { + // Reset to min + current_named_values.insert(name.clone(), *min); + } + } + + if !incremented { + *exhausted = true; + return None; + } + + // Reset all iterators with new named values + iterators.clear(); + current_values.clear(); + break; // Go back to initialization + } + + // Reset this iterator and move left + iterators[pos] = Self::new_with_context( + grammar, + &exprs[pos], + current_named_values.clone(), + ); + if let Some(val) = iterators[pos].next() { + current_values[pos] = val; + } + pos -= 1; + } + } + } + } + } + } +} + +/// Generates all permutations of one, two, or three character long strings. +pub struct ThreeIterator { + /// Current string length being generated (1, 2, or 3). + len: u8, + /// Character indices for each position (values 0..=0x7F). + indices: [u8; 3], + /// Set when all lengths are exhausted. + done: bool, +} + +impl ThreeIterator { + pub fn new() -> ThreeIterator { + ThreeIterator { + len: 1, + indices: [0; 3], + done: false, + } + } +} + +impl Iterator for ThreeIterator { + type Item = String; + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + let len = self.len as usize; + + // Build the current string from the active indices. + let result: String = self.indices[..len] + .iter() + .map(|&i| char::from_u32(i as u32).unwrap()) + .collect(); + + // Advance indices right-to-left, carrying into higher positions. + let mut carry = true; + for i in (0..len).rev() { + if carry { + if self.indices[i] < 0x7F { + self.indices[i] += 1; + carry = false; + } else { + self.indices[i] = 0; + // carry remains true; propagate left + } + } + } + + if carry { + // All positions overflowed — this length is exhausted. + if self.len < 3 { + self.len += 1; + self.indices = [0; 3]; + } else { + self.done = true; + } + } + + Some(result) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn assert_permutations(grammar: &str, expected: &[&str]) { + let g = Grammar::grammar_from_str(grammar, "cat").unwrap(); + let e = &g.productions.get("P").unwrap().expression; + let ps: Vec<_> = PermutationIterator::new(&g, e).collect(); + assert_eq!(ps, expected); + } + + #[test] + fn seq_and_alt() { + // Full sequence, then truncations (length 2, then length 1). + assert_permutations( + "P -> `A` (`B` | (`C1` | `C2`) | `D`) `E`", + &["ABE", "AC1E", "AC2E", "ADE", "AB", "AC1", "AC2", "AD", "A"], + ); + } + + #[test] + fn optional() { + // Full sequence, then truncations. + assert_permutations( + "P -> `A` (`B` | `C`)? `D`", + &["AD", "ABD", "ACD", "A", "AB", "AC", "A"], + ); + } + + #[test] + fn seq_truncated() { + // A single sequence with no alternatives: ABC, then AB, then A. + assert_permutations("P -> `A` `B` `C`", &["ABC", "AB", "A"]); + } + + #[test] + fn seq_truncated_with_alts() { + // Each position has alternatives; verify all combos per length, then shorter lengths. + assert_permutations( + "P -> (`A` | `X`) (`B` | `Y`)", + &["AB", "AY", "XB", "XY", "A", "X"], + ); + } + + #[test] + fn repeat() { + assert_permutations("P -> (`A` | `B`)*", &["", "A", "B", "AA", "BB"]); + } + + #[test] + fn repeat_plus() { + assert_permutations("P -> (`A` | `B`)+", &["", "A", "B", "AA", "BB"]); + } + + #[test] + fn repeat_range() { + assert_permutations("P -> (`A` | `B`){0..}", &["", "A", "B", "AA", "BB"]); + + assert_permutations("P -> (`A` | `B`){1..3}", &["A", "B", "AA", "BB"]); + + assert_permutations("P -> (`A` | `B`){2..=3}", &["AA", "BB", "AAA", "BBB"]); + } + + #[test] + fn charset() { + // Test with Terminal and Range + assert_permutations("P -> [`A` `X`-`Z`]", &["A", "X", "Z"]); + + // Test with just Range + assert_permutations("P -> [`a`-`c`]", &["a", "c"]); + } + + #[test] + fn named_repeat_range() { + // Test named repeat ranges are synchronized (only min and max values) + assert_permutations("P -> `A`{n:1..=5} `B` `C`{n}", &["ABC", "AAAAABCCCCC"]); + } + + #[test] + fn negative_lookahead() { + // NegativeLookahead emits empty string first, then all permutations of the expression. + assert_permutations("P -> !`A`", &["", "A"]); + assert_permutations("P -> !(`A` | `B`)", &["", "A", "B"]); + // In a sequence: empty lookahead plus the rest, then lookahead expr plus the rest, + // then truncated-length permutations (just the lookahead expression alone). + assert_permutations("P -> !`X` `Y`", &["Y", "XY", "", "X"]); + } +} diff --git a/tools/grammar-check/src/test_cases.rs b/tools/grammar-check/src/test_cases.rs new file mode 100644 index 0000000000..da3a13e19c --- /dev/null +++ b/tools/grammar-check/src/test_cases.rs @@ -0,0 +1,84 @@ +//! Built-in test cases. +//! +//! The initial idea with this was to collect a broad set of tests that +//! exercise all of the grammar rules. However, that's a tall order as it +//! would end up being quite large. I instead started leaning on the +//! permutation-based tests to more exhaustively cover the grammar. However, +//! there's less certainty using that mechanism, and it also makes it harder +//! to iterate on a single grammar rule. This may still be useful, but would +//! take some considerable work to make it useful. + +macro_rules! cases { + ($($name:path => $($s:literal)+)+) => { + pub static LEX_CASES: &[(&str, &[&str])] = &[ + $( + (stringify!($name), &[ $($s),* ]), + )+ + ]; + }; +} + +cases! { + empty => + "" + + comment::line_comment => + "// line comment" + "////" + "//// this is a comment" + "//\n" + comment::block_comment => + "/* block comment */" + comment::inner_line_doc => + "//! inner line doc" + comment::inner_block_doc => + "/*! inner block doc */" + comment::outer_line_doc => + "/// outer line doc" + "///" + "///\n" + "///abc\n" + "/// ☃" + comment::outer_block_doc => + "/** outer block doc */" + + reserved::pounds => + "##" + "###" + "####" + "#####" + + raw_identifier => + "r#fn" + char => + "'x'" + string => + "\"string\"" + raw_string => + "r\"raw string\"" + "r#\"raw string\"#" + "r#\"\"\"#" + byte => + "b'x'" + byte_string => + "b\"byte\"" + raw_byte_string => + "br\"raw byte\"" + "br#\"raw byte\"#" + c_string => + "c\"c str\"" + raw_c_string => + "cr\"raw c str\"" + "cr#\"raw c str\"#" + float => + "1.2" + integer => + "123" + lifetime => + "'a" + punctuation => + "!" + identifier => + "ident" + "fn" +} diff --git a/tools/grammar-check/src/tools/pm2.rs b/tools/grammar-check/src/tools/pm2.rs new file mode 100644 index 0000000000..f471daae23 --- /dev/null +++ b/tools/grammar-check/src/tools/pm2.rs @@ -0,0 +1,455 @@ +//! The proc-macro2 tool. + +use parser::lexer::Tokens; +use parser::{Node, ParseError}; +use proc_macro2::{Spacing, TokenStream, TokenTree}; +use regex::Regex; +use std::ops::Range; +use std::str::FromStr; +use std::sync::LazyLock; + +pub fn tokenize(src: &str) -> Result, ParseError> { + let mut tokens = Vec::new(); + let stream = TokenStream::from_str(src).map_err(|e| ParseError { + byte_offset: 0, + message: e.to_string(), + })?; + tokens_from_ts(src, stream, &mut tokens)?; + Ok(tokens) +} + +// proc-macro2 does not reject literals starting with E. +// We'll need to do that to match behavior. +// https://github.com/dtolnay/proc-macro2/issues/506 +static SUFFIX_NO_E: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?x) + ^( + ([0-9][0-9_]*[eE]) # DEC_LITERAL + | (0b([01]|_)*?[01]([01]|_)*[eE]) # BIN_LITERL + | (0o([0-7]|_)*?[0-7]([0-7]|_)*[eE]) # OCT_LITERAL + | ([0-9]([0-9]|_)*\.[0-9]([0-9]|_)*[eE]) # FLOAT_LITERAL + ) + ", + ) + .unwrap() +}); + +static FLOAT_EXPONENT: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?x) + ^ + [0-9]([0-9]|_)* + (\. [0-9]([0-9]|_)*)? + [eE] [+-]? ([0-9]|_)*? [0-9] ([0-9]|_)* + ", + ) + .unwrap() +}); + +static NUM_DOT: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?x) + ^( + (0b([01]|_)*?[01]([01]|_)*) # BIN_LITERL + | (0o([0-7]|_)*?[0-7]([0-7]|_)*) # OCT_LITERAL + | (0x([0-9a-fA-F]|_)*?[0-9a-fA-F]([0-9a-fA-F]|_)*) # HEX_LITERAL + ) + \. + ", + ) + .unwrap() +}); + +fn tokens_from_ts(src: &str, ts: TokenStream, output: &mut Vec) -> Result<(), ParseError> { + let trees: Vec = ts.into_iter().collect(); + let mut i = 0; + while i < trees.len() { + let tt = &trees[i]; + let span = tt.span(); + let mut range = span.byte_range(); + + // For OUTER_LINE_DOC and CRLF input, the range ends up pointing at + // the CR. Adjust this to match the other tools. + let s_range = &src[range.clone()]; + if (s_range.starts_with("///") || s_range.starts_with("//!")) && s_range.ends_with('\r') { + range.end -= 1; + } + + match tt { + TokenTree::Ident(_) => { + // proc-macro2 does not reject Edition 2021 reserved prefixes. + // https://github.com/dtolnay/proc-macro2/issues/534 + if src[range.end..].chars().next() == Some('"') + && !matches!(&src[range.clone()], "b" | "c" | "r" | "br" | "cr") + { + return Err(ParseError { + message: "RESERVED_TOKEN_DOUBLE_QUOTE".to_string(), + byte_offset: range.end, + }); + } + if src[range.end..].chars().next() == Some('\'') + && !matches!(&src[range.clone()], "b") + { + return Err(ParseError { + message: "RESERVED_TOKEN_SINGLE_QUOTE".to_string(), + byte_offset: range.end, + }); + } + + if src[range.end..].chars().next() == Some('#') { + return Err(ParseError { + message: "RESERVED_TOKEN_POUND".to_string(), + byte_offset: range.start, + }); + } + + i += 1; + let name = if src[range.start..].starts_with("r#") { + "RAW_IDENTIFIER" + } else { + "IDENTIFIER_OR_KEYWORD" + } + .to_string(); + output.push(Node::new(name, range)); + } + TokenTree::Punct(p) => { + // In order to be consistent with rustc which uses joined tokens, + // this looks to join multiple punctuation tokens. + + // s accumulates the punctuation string. + let mut s = p.as_char().to_string(); + i += 1; + let mut current_spacing = p.spacing(); + + // Try to consume subsequent punctuation if it is joint and + // forms a valid operator. + while current_spacing == Spacing::Joint && i < trees.len() { + match &trees[i] { + TokenTree::Punct(next_p) => { + s.push(next_p.as_char()); + if is_valid_punctuation(&s) { + range.end = next_p.span().byte_range().end; + current_spacing = next_p.spacing(); + i += 1; + } else { + s.pop(); + break; + } + } + TokenTree::Ident(ident) => { + // lifetime + s.push_str(&ident.to_string()); + range.end = ident.span().byte_range().end; + + // For some reason, proc-macro2 doesn't seem to fail + // when it sees IDENT'IDENT. + if i >= 2 { + let prev_tt = &trees[i - 2]; + let prev_range = prev_tt.span().byte_range(); + if let TokenTree::Ident(_) = prev_tt + && prev_range.end == range.start + { + return Err(ParseError { + message: "RESERVED_TOKEN_SINGLE_QUOTE".to_string(), + byte_offset: prev_range.start, + }); + } + } + i += 1; + break; + } + _ => break, + } + } + + // https://github.com/dtolnay/proc-macro2/issues/535 + if s == "#" + && i < trees.len() + && let TokenTree::Literal(lit) = &trees[i] + && lit.to_string().starts_with('"') + && trees[i].span().byte_range().start == range.start + 1 + { + return Err(ParseError { + message: "RESERVED_GUARDED_STRING_LITERAL".to_string(), + byte_offset: range.start, + }); + } + + let name = if s.starts_with('\'') && s.len() > 1 { + "LIFETIME_TOKEN" + } else { + "PUNCTUATION" + } + .to_string(); + output.push(Node::new(name, range)); + } + TokenTree::Literal(lit) => { + let s = lit.to_string(); + if SUFFIX_NO_E.is_match(&s) && !FLOAT_EXPONENT.is_match(&s) { + return Err(ParseError { + message: "bad E suffix".to_string(), + byte_offset: range.start, + }); + } + + // https://github.com/dtolnay/proc-macro2/issues/531 + if [ + "'''", "'\r'", "'\n'", "'\t'", "b'''", "b'\r'", "b'\n'", "b'\t'", + ] + .iter() + .any(|p| s.starts_with(p)) + { + return Err(ParseError { + message: "invalid byte or char literal".to_string(), + byte_offset: range.start, + }); + } + + // https://github.com/dtolnay/proc-macro2/issues/532 + if matches!(s.as_bytes().last_chunk::<2>(), Some(b"'_" | b"\"_" | b"#_")) { + return Err(ParseError { + message: "underscore suffix not allowed".to_string(), + byte_offset: range.start, + }); + } + + // https://github.com/dtolnay/proc-macro2/issues/533 + let s_rest = &src[range.start..]; + if let Some(m) = NUM_DOT.find(s_rest) { + let next = src[range.start + m.len()..].chars().next(); + if !matches!(next, Some('.' | '_')) + && !next + .map(|ch| unicode_ident::is_xid_start(ch)) + .unwrap_or(false) + { + return Err(ParseError { + message: "reserved bin/oct/hex literal followed by .".to_string(), + byte_offset: range.start, + }); + } + } + + output.push(Node::new(lit_to_reference(&s), range)); + i += 1; + } + TokenTree::Group(group) => { + let delim = group.delimiter(); + let delim_str = match &delim { + proc_macro2::Delimiter::Parenthesis => "(", + proc_macro2::Delimiter::Brace => "{", + proc_macro2::Delimiter::Bracket => "[", + proc_macro2::Delimiter::None => "", + }; + if !delim_str.is_empty() { + output.push(Node::new( + "PUNCTUATION".to_string(), + group.span_open().byte_range(), + )); + } + tokens_from_ts(src, group.stream(), output)?; + if !delim_str.is_empty() { + let close_delim = match delim_str { + "(" => ")", + "{" => "}", + "[" => "]", + _ => "", + }; + let mut range = group.span_close().byte_range(); + // proc-macro2's CRLF handling ends up with a range pointing + // at the CR instead of the byte before. + if &src[range.clone()] == "\r" && close_delim == "]" { + range.start -= 1; + range.end -= 1; + // After shifting back one byte we may now be inside a + // multi-byte UTF-8 character. Walk start back further + // until we're on a char boundary, then set end to the + // end of that character. + while range.start > 0 && !src.is_char_boundary(range.start) { + range.start -= 1; + } + range.end = range.start + + src[range.start..] + .chars() + .next() + .map_or(1, |c| c.len_utf8()); + } + output.push(Node::new("PUNCTUATION".to_string(), range)); + } + i += 1; + } + } + } + Ok(()) +} + +fn lit_to_reference(lit: &str) -> String { + if lit.starts_with("cr") { + "RAW_C_STRING_LITERAL".to_string() + } else if lit.starts_with('c') { + "C_STRING_LITERAL".to_string() + } else if lit.starts_with("br") { + "RAW_BYTE_STRING_LITERAL".to_string() + } else if lit.starts_with("b'") { + "BYTE_LITERAL".to_string() + } else if lit.starts_with("b\"") { + "BYTE_STRING_LITERAL".to_string() + } else if lit.starts_with("r\"") || lit.starts_with("r#") { + "RAW_STRING_LITERAL".to_string() + } else if lit.starts_with('\'') { + "CHAR_LITERAL".to_string() + } else if lit.starts_with('"') { + "STRING_LITERAL".to_string() + } else if lit.starts_with("0x") || lit.starts_with("0o") || lit.starts_with("0b") { + "INTEGER_LITERAL".to_string() + } else if lit.contains('.') { + "FLOAT_LITERAL".to_string() + } else if lit.contains('e') || lit.contains('E') { + // Could be float with exponent or integer with suffix containing 'e'/'E' + // Check if there's a valid float exponent pattern + if lit + .bytes() + .position(|b| b == b'e' || b == b'E') + .map(|pos| { + lit.as_bytes() + .get(pos - 1) + .map_or(false, |&ch| ch.is_ascii_digit() || ch == b'_') + }) + .unwrap_or(false) + { + "FLOAT_LITERAL".to_string() + } else { + "INTEGER_LITERAL".to_string() + } + } else { + "INTEGER_LITERAL".to_string() + } +} + +fn is_valid_punctuation(s: &str) -> bool { + matches!( + s, + "..." + | "..=" + | "<<=" + | ">>=" + | "!=" + | "%=" + | "&&" + | "&=" + | "*=" + | "+=" + | "-=" + | "->" + | ".." + | "/=" + | "::" + | "<-" + | "<<" + | "<=" + | "==" + | "=>" + | ">=" + | ">>" + | "^=" + | "|=" + | "||" + ) +} + +pub fn normalize( + pm2_result: Result, ParseError>, + reference_result: Result, + src: &str, +) -> (Result, ParseError>, Result, ParseError>) { + let reference_result = + reference_result.map(|tokens| normalize_reference_tokens(tokens.tokens, src)); + let pm2_result = match (&pm2_result, &reference_result) { + (Ok(_), Err(e)) => { + // For some reason, proc-macro2 treats NBSP as whitespace. + if src[e.byte_offset..].chars().next() == Some('\u{a0}') { + Err(ParseError { + message: "unexpected NBSP whitespace".to_string(), + byte_offset: e.byte_offset, + }) + } else { + pm2_result + } + } + _ => pm2_result, + }; + (pm2_result, reference_result) +} + +fn normalize_reference_tokens(tokens: Vec, src: &str) -> Vec { + let len = tokens.len(); + tokens + .into_iter() + .filter(|token| !matches!(token.name.as_str(), "LINE_COMMENT" | "BLOCK_COMMENT")) + .fold(Vec::with_capacity(len), |mut acc, token| { + // proc-macro2 does not handle ## reserved tokens (treats them as individual punctuation) + // https://github.com/dtolnay/proc-macro2/issues/535 + if token.name == "RESERVED_TOKEN" && src[token.range.clone()].chars().all(|c| c == '#') + { + let count = token.range.len(); + for i in 0..count { + acc.push(Node::new( + String::from("PUNCTUATION"), // # + Range { + start: token.range.start + i, + end: token.range.start + i + 1, + }, + )); + } + return acc; + } + // proc-macro2 converts doc comments into doc attributes. + match &*token.name { + "OUTER_LINE_DOC" | "INNER_LINE_DOC" | "OUTER_BLOCK_DOC" | "INNER_BLOCK_DOC" => { + acc.push(Node::new( + String::from("PUNCTUATION"), // # + token.range.clone(), + )); + if token.name.starts_with("INNER") { + acc.push(Node::new( + String::from("PUNCTUATION"), // ! + token.range.clone(), + )); + } + acc.push(Node::new( + String::from("PUNCTUATION"), // [ + Range { + start: token.range.start, + end: token.range.start + 1, + }, + )); + acc.push(Node::new( + String::from("IDENTIFIER_OR_KEYWORD"), + token.range.clone(), + )); + acc.push(Node::new( + String::from("PUNCTUATION"), // = + token.range.clone(), + )); + acc.push(Node::new( + String::from("STRING_LITERAL"), + token.range.clone(), + )); + acc.push(Node::new( + String::from("PUNCTUATION"), // ] + Range { + start: token.range.end + - src[..token.range.end] + .chars() + .next_back() + .unwrap() + .len_utf8(), + end: token.range.end, + }, + )); + } + _ => acc.push(token), + } + acc + }) +} diff --git a/tools/grammar-check/src/tools/rustc.rs b/tools/grammar-check/src/tools/rustc.rs new file mode 100644 index 0000000000..d374c93c6d --- /dev/null +++ b/tools/grammar-check/src/tools/rustc.rs @@ -0,0 +1,267 @@ +//! The `rustc_parse`-based tool. + +use std::fmt::Write as _; +extern crate rustc_ast; +extern crate rustc_driver; +extern crate rustc_errors; +extern crate rustc_lexer; +extern crate rustc_parse; +extern crate rustc_session; +extern crate rustc_span; + +use parser::{Edition, Node, ParseError}; +use rustc_ast::ast::AttrStyle; +use rustc_ast::token::{CommentKind, IdentIsRaw, TokenKind}; +use rustc_errors::emitter::HumanReadableErrorType; +use rustc_errors::json::JsonEmitter; +use rustc_errors::{ColorConfig, DiagCtxt}; +use rustc_parse::lexer::StripTokens; +use rustc_session::parse::ParseSess; +use rustc_span::FileName; +use rustc_span::fatal_error::FatalError; +use rustc_span::source_map::{FilePathMapping, SourceMap}; +use std::io; +use std::io::Write; +use std::ops::Range; +use std::sync::{Arc, Mutex}; + +struct Shared { + data: Arc>, +} + +impl Write for Shared { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.data.lock().unwrap().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.data.lock().unwrap().flush() + } +} + +fn to_rustc_edition(edition: Edition) -> rustc_span::edition::Edition { + match edition { + Edition::Edition2015 => rustc_span::edition::Edition::Edition2015, + Edition::Edition2018 => rustc_span::edition::Edition::Edition2018, + Edition::Edition2021 => rustc_span::edition::Edition::Edition2021, + Edition::Edition2024 => rustc_span::edition::Edition::Edition2024, + } +} + +pub fn tokenize(src: &str, edition: Edition) -> Result, ParseError> { + rustc_span::create_session_globals_then(to_rustc_edition(edition), &[], None, || { + let source_map = Arc::new(SourceMap::new(FilePathMapping::empty())); + // TODO: probably not needed? + // source_map.new_source_file(Path::new("test.rs").to_owned().into(), "".to_owned()); + let output = Arc::new(Mutex::new(Vec::new())); + let je = JsonEmitter::new( + Box::new(Shared { + data: output.clone(), + }), + Some(source_map.clone()), + false, // pretty + HumanReadableErrorType { + short: true, + unicode: true, + }, + ColorConfig::Never, + ); + + let dcx = DiagCtxt::new(Box::new(je)); + let psess = ParseSess::with_dcx(dcx, source_map); + // TODO: Use StripTokens::Nothing before frontmatter is + // stabilized. Use StripTokens::ShebangAndFrontmatter after it is + // stabilized. + let strip_tokens = StripTokens::ShebangAndFrontmatter; + let source = String::from(src); + let filename = FileName::Custom("internal".into()); + rustc_driver::catch_fatal_errors(|| { + let mut parser = match rustc_parse::new_parser_from_source_str( + &psess, + filename, + source, + strip_tokens, + ) { + Ok(parser) => parser, + Err(e) => { + for diag in e { + diag.emit(); + } + FatalError.raise(); + } + }; + let mut tokens = Vec::new(); + while parser.token.kind != TokenKind::Eof { + let source_file = psess + .source_map() + .lookup_source_file(parser.token.span.lo()); + let start = source_file + .original_relative_byte_pos(parser.token.span.lo()) + .0 as usize; + let end = source_file + .original_relative_byte_pos(parser.token.span.hi()) + .0 as usize; + + let token = Node::new(to_reference_name(&parser.token.kind), Range { start, end }); + tokens.push(token); + parser.bump(); + } + // Unfortunately this is handled outside of normal lexing. + psess.bad_unicode_identifiers.with_lock(|idents| { + for (ident, spans) in idents.drain(..) { + psess + .dcx() + .struct_span_err( + spans, + format!("identifiers cannot contain emoji: {ident}"), + ) + .emit(); + } + }); + psess.dcx().emit_stashed_diagnostics(); + let diags = diagnostics(&output.lock().unwrap()); + if diags.iter().any(|diag| diag.level.starts_with("error")) { + FatalError.raise(); + } + tokens + }) + .map_err(|_| { + let mut message = String::new(); + let out = &output.lock().unwrap(); + let diags = diagnostics(out); + let mut byte_offset = 0; + for diag in diags { + write!(message, "error: {}", diag.rendered).unwrap(); + if byte_offset == 0 { + byte_offset = diag + .spans + .iter() + .find(|sp| sp.is_primary) + .map(|sp| sp.byte_start) + .unwrap_or_default(); + } + } + ParseError { + byte_offset: byte_offset as usize, + message, + } + }) + }) +} + +fn to_reference_name(kind: &TokenKind) -> String { + match kind { + TokenKind::Eq + | TokenKind::Lt + | TokenKind::Le + | TokenKind::EqEq + | TokenKind::Ne + | TokenKind::Ge + | TokenKind::Gt + | TokenKind::AndAnd + | TokenKind::OrOr + | TokenKind::Bang + | TokenKind::Tilde + | TokenKind::Plus + | TokenKind::Minus + | TokenKind::Star + | TokenKind::Slash + | TokenKind::Percent + | TokenKind::Caret + | TokenKind::And + | TokenKind::Or + | TokenKind::Shl + | TokenKind::Shr + | TokenKind::PlusEq + | TokenKind::MinusEq + | TokenKind::StarEq + | TokenKind::SlashEq + | TokenKind::PercentEq + | TokenKind::CaretEq + | TokenKind::AndEq + | TokenKind::OrEq + | TokenKind::ShlEq + | TokenKind::ShrEq + | TokenKind::At + | TokenKind::Dot + | TokenKind::DotDot + | TokenKind::DotDotDot + | TokenKind::DotDotEq + | TokenKind::Comma + | TokenKind::Semi + | TokenKind::Colon + | TokenKind::PathSep + | TokenKind::RArrow + | TokenKind::LArrow + | TokenKind::FatArrow + | TokenKind::Pound + | TokenKind::Dollar + | TokenKind::Question + | TokenKind::SingleQuote + | TokenKind::OpenParen + | TokenKind::CloseParen + | TokenKind::OpenBrace + | TokenKind::CloseBrace + | TokenKind::OpenBracket + | TokenKind::CloseBracket => "PUNCTUATION", + TokenKind::OpenInvisible(_) | TokenKind::CloseInvisible(_) => { + panic!("unexpected invisible token") + } + TokenKind::Literal(lit) => match lit.kind { + rustc_ast::token::LitKind::Bool => "IDENTIFIER_OR_KEYWORD", + rustc_ast::token::LitKind::Byte => "BYTE_LITERAL", + rustc_ast::token::LitKind::Char => "CHAR_LITERAL", + rustc_ast::token::LitKind::Integer => "INTEGER_LITERAL", + rustc_ast::token::LitKind::Float => "FLOAT_LITERAL", + rustc_ast::token::LitKind::Str => "STRING_LITERAL", + rustc_ast::token::LitKind::StrRaw(_) => "RAW_STRING_LITERAL", + rustc_ast::token::LitKind::ByteStr => "BYTE_STRING_LITERAL", + rustc_ast::token::LitKind::ByteStrRaw(_) => "RAW_BYTE_STRING_LITERAL", + rustc_ast::token::LitKind::CStr => "C_STRING_LITERAL", + rustc_ast::token::LitKind::CStrRaw(_) => "RAW_C_STRING_LITERAL", + // Diagnostics handle this below. + rustc_ast::token::LitKind::Err(_) => "Literal Error", + }, + TokenKind::Ident(_, IdentIsRaw::No) => "IDENTIFIER_OR_KEYWORD", + TokenKind::Ident(_, IdentIsRaw::Yes) => "RAW_IDENTIFIER", + TokenKind::NtIdent(..) => panic!("unexpected NtIdent"), + TokenKind::Lifetime(..) => "LIFETIME_TOKEN", + TokenKind::NtLifetime(..) => panic!("unexpected NtLifetime"), + TokenKind::DocComment(CommentKind::Line, AttrStyle::Inner, ..) => "INNER_LINE_DOC", + TokenKind::DocComment(CommentKind::Line, AttrStyle::Outer, ..) => "OUTER_LINE_DOC", + TokenKind::DocComment(CommentKind::Block, AttrStyle::Inner, ..) => "INNER_BLOCK_DOC", + TokenKind::DocComment(CommentKind::Block, AttrStyle::Outer, ..) => "OUTER_BLOCK_DOC", + TokenKind::Eof => panic!("unexpected EOF"), + } + .to_string() +} + +fn diagnostics(output: &[u8]) -> Vec { + let json = std::str::from_utf8(output).unwrap(); + json.lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} + +#[derive(serde::Deserialize)] +struct Diagnostic { + rendered: String, + level: String, + spans: Vec, +} + +#[derive(serde::Deserialize)] +struct DiagSpan { + is_primary: bool, + byte_start: u32, +} + +pub fn normalize(tokens: &[Node]) -> Result, ParseError> { + let new_ts = tokens + .iter() + // rustc_parse does not retain comments. + .filter(|token| !matches!(token.name.as_str(), "LINE_COMMENT" | "BLOCK_COMMENT")) + .cloned() + .collect(); + Ok(new_ts) +} diff --git a/tools/grammar-check/src/tools/rustc_lexer.rs b/tools/grammar-check/src/tools/rustc_lexer.rs new file mode 100644 index 0000000000..16aad33b96 --- /dev/null +++ b/tools/grammar-check/src/tools/rustc_lexer.rs @@ -0,0 +1,30 @@ +//! The `rustc_lexer`-based tool. +//! +//! This is generally not useful, but is used for internal investigation. The +//! `rustc_parse` module does a lot of translations to the lower-level +//! `rustc_lexer`. The Reference is based on the tokens as used and seen by +//! macros, and those are not the same as the ones as generated by +//! `rustc_lexer`. + +extern crate rustc_lexer; + +use parser::{Node, ParseError}; +use rustc_lexer::{FrontmatterAllowed, TokenKind}; +use std::ops::Range; + +pub fn tokenize(src: &str) -> Result, ParseError> { + let mut pos = 0; + let ts: Vec<_> = rustc_lexer::tokenize(src, FrontmatterAllowed::Yes) + .filter_map(|token| { + let start = pos; + let end = pos + token.len as usize; + pos += token.len as usize; + if matches!(token.kind, TokenKind::Whitespace) { + return None; + } + let t = Node::new(format!("{:?}", token.kind), Range { start, end }); + Some(t) + }) + .collect(); + Ok(ts) +} diff --git a/tools/grammar/src/frontmatter.rs b/tools/grammar/src/frontmatter.rs new file mode 100644 index 0000000000..36fe5b4643 --- /dev/null +++ b/tools/grammar/src/frontmatter.rs @@ -0,0 +1,55 @@ +//! This is a temporary hack to include the frontmatter grammar until it is +//! stabilized. +//! +//! This should be removed once FRONTMATTER is added to the Reference. + +use crate::{Grammar, parser}; +use diagnostics::Diagnostics; +use std::path::Path; + +pub fn load_grammar_with_frontmatter(diag: &mut Diagnostics) -> Grammar { + let mut grammar = super::load_grammar(diag); + + parser::parse_grammar(FRONTMATTER, &mut grammar, "lexer", Path::new("")).unwrap(); + + grammar +} + +static FRONTMATTER: &str = "⊥ -> CHAR* CHAR + +error -> ^ ⊥ // Should be a hard error. + +@root FRONTMATTER -> + WHITESPACE_ONLY_LINE* + !FRONTMATTER_INVALID + FRONTMATTER_MAIN + +WHITESPACE_ONLY_LINE -> (!LF WHITESPACE)* LF + +FRONTMATTER_INVALID -> (!LF WHITESPACE)+ `---` error + +FRONTMATTER_MAIN -> + `-`{n:3..=255} ^ FRONTMATTER_REST + +FRONTMATTER_REST -> + FRONTMATTER_FENCE_START + FRONTMATTER_LINE* + FRONTMATTER_FENCE_END + +FRONTMATTER_FENCE_START -> + MAYBE_INFOSTRING_OR_WS LF + +FRONTMATTER_FENCE_END -> + `-`{n} HORIZONTAL_WHITESPACE* ( LF | EOF ) + +FRONTMATTER_LINE -> !`-`{n} ~[LF CR]* LF + +MAYBE_INFOSTRING_OR_WS -> + HORIZONTAL_WHITESPACE* INFOSTRING? HORIZONTAL_WHITESPACE* + +INFOSTRING -> (XID_Start | `_`) ( XID_Continue | `-` | `.` )* + +HORIZONTAL_WHITESPACE -> + U+0009 // Horizontal tab, `'\t'` + | U+0020 // Space, `' '` +"; diff --git a/tools/grammar/src/lib.rs b/tools/grammar/src/lib.rs index e2736e218a..29bc231e9b 100644 --- a/tools/grammar/src/lib.rs +++ b/tools/grammar/src/lib.rs @@ -9,8 +9,11 @@ use std::sync::LazyLock; use walkdir::WalkDir; mod display; +mod frontmatter; mod parser; +pub use frontmatter::load_grammar_with_frontmatter; + #[derive(Debug, Default)] pub struct Grammar { pub productions: HashMap, @@ -149,6 +152,12 @@ impl Display for Character { } impl Grammar { + pub fn grammar_from_str(input: &str, category: &str) -> Result { + let mut grammar = Grammar::default(); + parser::parse_grammar(input, &mut grammar, category, Path::new(""))?; + Ok(grammar) + } + /// Generates a new unique expression ID. pub fn next_id(&mut self) -> u32 { let id = self.next_id; diff --git a/tools/parser/Cargo.toml b/tools/parser/Cargo.toml new file mode 100644 index 0000000000..4d01e982e8 --- /dev/null +++ b/tools/parser/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "parser" +edition = "2024" +license = "MIT OR Apache-2.0" + +[dependencies] +diagnostics = { path = "../diagnostics" } +grammar = { path = "../grammar" } +tracing = "0.1.43" +tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } +tracing-tree = "0.4.1" +unicode-ident = "1.0.22" diff --git a/tools/parser/README.md b/tools/parser/README.md new file mode 100644 index 0000000000..828ad18381 --- /dev/null +++ b/tools/parser/README.md @@ -0,0 +1,25 @@ +# Reference grammar parser + +This is a primitive interpreter that parses an input using the Reference grammar and can generate tokens or a generic tree representation of that source. + +## Overview + +The parser itself is fairly straightforward as it uses the Reference productions to drive an interpreter to parse some input into a tree of nodes. + +There are some hard-coded handlers for some of the English-based rules such as the suffixes. Ideally the grammar should be changed to remove those and use parseable expressions (like negative lookahead). + +## To lex or not to lex + +The tooling is currently designed to keep lexing separate from parsing. I'm still uncertain if this is the right thing to do. It adds some complexity. For example, the parser has a `Source` abstraction so that its input can either be a string of bytes (which is used for lexing) or a sequence of tokens. An alternative is to drop the separate lexing phase, and instead somehow automatically insert "whitespace or comments" in between each expression in the non-lexer productions. However, this is not simple and itself would add its own complexity. It might be worth exploring, though. + +## Token splitting + +Token splitting is not implemented. That is, when the parser sees `Option<`, it will need to split the `<<` into two `<` tokens. + +This is a primary blocker for getting tree-based parsing working well enough to parse a typical Rust file. + +This is not an easy problem if we want to have parity with `rustc` because `rustc` does not always split tokens. It might be sufficient for a naive approach to split everything, and hope that there aren't any test cases where they diverge. Unfortunately this could cause problems with the permutation or fuzzing-based testing. Or, we could hard-code where `rustc` does split. + +An alternative approach to splitting would be to change the Reference grammar so that it uses the proc-macro model where tokens keep track of their "spacing" so that you know if you can join two tokens (like `:` `:` into `::`). I believe there is desire to move `rustc` itself to this model, but the work there hasn't been done. This in itself would add some complexity, though. The Reference would also probably need to be clearer about how tokens are translated between the two models (because `macro_rules` uses the joined model whereas proc-macros use the split model). I'm not sure which approach will be easier or better. + +See https://github.com/rust-lang/rust/issues/152398 for my analysis on this. diff --git a/tools/parser/src/coverage.rs b/tools/parser/src/coverage.rs new file mode 100644 index 0000000000..cb6d6759e0 --- /dev/null +++ b/tools/parser/src/coverage.rs @@ -0,0 +1,601 @@ +//! Support for recording and rendering coverage of the grammar. + +use grammar::{Character, Expression, ExpressionKind, Grammar}; + +#[derive(Default)] +pub struct Coverage { + /// Count of the repetitions for each expression. + /// + /// The index is the expression ID. The value is the number of times a + /// particular number of repetitions was found for that expression (index + /// N means it matched N repetitions of the given number of times). + pub match_count: Vec>, + + /// Count of how often an expression failed to match its input. + /// + /// The index is the expression ID, the count is the number of times it failed. + pub no_match_count: Vec, + + /// Count of how often the expression caused a `ParseError`. + /// + /// The index is the expression ID, the count is the number of times it caused an error. + pub parse_error: Vec, +} + +impl Coverage { + /// Marks a node as being matched. + pub fn cov_match(&mut self, id: u32, count: u32) { + if self.match_count.len() < (id + 1) as usize { + self.match_count.resize((id + 1) as usize, Vec::new()); + } + let ns = self.match_count.get_mut(id as usize).unwrap(); + if ns.len() < (count + 1) as usize { + ns.resize((count + 1) as usize, 0); + } + *ns.get_mut(count as usize).unwrap() += 1; + } + + /// Marks a node that failed to match its input. + pub fn cov_no_match(&mut self, id: u32) { + if self.no_match_count.len() < (id + 1) as usize { + self.no_match_count.resize((id + 1) as usize, 0); + } + *self.no_match_count.get_mut(id as usize).unwrap() += 1; + } + + /// Marks a node that caused a `ParseError`. + pub fn cov_parse_error(&mut self, id: u32) { + if self.parse_error.len() < (id + 1) as usize { + self.parse_error.resize((id + 1) as usize, 0); + } + *self.parse_error.get_mut(id as usize).unwrap() += 1; + } + + /// Merge one `Coverage` into this one. + pub fn merge(&mut self, other: Coverage) { + if self.match_count.len() < other.match_count.len() { + self.match_count.resize(other.match_count.len(), Vec::new()); + } + for (id, counts) in other.match_count.into_iter().enumerate() { + let this = self.match_count.get_mut(id).unwrap(); + if this.len() < counts.len() { + this.resize(counts.len(), 0); + } + for (count, value) in counts.into_iter().enumerate() { + this[count] += value; + } + } + + if self.no_match_count.len() < other.no_match_count.len() { + self.no_match_count.resize(other.no_match_count.len(), 0); + } + for (id, value) in other.no_match_count.into_iter().enumerate() { + self.no_match_count[id] += value; + } + + if self.parse_error.len() < other.parse_error.len() { + self.parse_error.resize(other.parse_error.len(), 0); + } + for (id, value) in other.parse_error.into_iter().enumerate() { + self.parse_error[id] += value; + } + } + + /// Saves the coverage data to a file called `coverage.html`. + pub fn save(&self, grammar: &Grammar) { + let mut html = String::new(); + let mut span_stack = Vec::new(); + self.render_html(&mut html, &mut span_stack, grammar); + std::fs::write("coverage.html", html).expect("failed to write coverage.html"); + } + + fn get_coverage_status(&self, id: u32, kind: &ExpressionKind) -> CoverageStatus { + let match_count = self.match_count.get(id as usize); + let no_match = self.no_match_count.get(id as usize).copied().unwrap_or(0); + let parse_error = self.parse_error.get(id as usize).copied().unwrap_or(0); + + let has_matches = match_count + .map(|counts| counts.iter().any(|&c| c > 0)) + .unwrap_or(false); + let has_no_match = no_match > 0; + + // Special case logic for specific expression kinds + match kind { + ExpressionKind::Optional(_) => { + // Green means match_count contains both 0 and 1 + if let Some(counts) = match_count { + let has_zero = counts.get(0).copied().unwrap_or(0) > 0; + let has_one = counts.get(1).copied().unwrap_or(0) > 0; + if has_zero && has_one { + return CoverageStatus::Green; + } else if has_zero || has_one { + return CoverageStatus::Yellow; + } + } + CoverageStatus::Red + } + ExpressionKind::Repeat(_) => { + // Green means match_count contains 0, 1, and more than 1 + if let Some(counts) = match_count { + let has_zero = counts.get(0).copied().unwrap_or(0) > 0; + let has_one = counts.get(1).copied().unwrap_or(0) > 0; + let has_more = counts.iter().skip(2).any(|&c| c > 0); + if has_zero && has_one && has_more { + return CoverageStatus::Green; + } else if has_zero || has_one || has_more { + return CoverageStatus::Yellow; + } + } + CoverageStatus::Red + } + ExpressionKind::RepeatPlus(_) => { + // Green means match_count contains 1 and more than 1 and no_match_count is not zero + if let Some(counts) = match_count { + let has_one = counts.get(1).copied().unwrap_or(0) > 0; + let has_more = counts.iter().skip(2).any(|&c| c > 0); + if has_one && has_more && has_no_match { + return CoverageStatus::Green; + } else if (has_one || has_more) || has_no_match { + return CoverageStatus::Yellow; + } + } + CoverageStatus::Red + } + ExpressionKind::RepeatRange { min, max, .. } => { + // Green means match_count contains the minimum and maximum values + if let Some(counts) = match_count { + let min_val = min.unwrap_or(0) as usize; + let has_min = counts.get(min_val).copied().unwrap_or(0) > 0; + + let has_max = if let Some(max_val) = max { + counts.get(*max_val as usize).copied().unwrap_or(0) > 0 + } else { + // If max is None, consider it green if there are matches for any count over min + counts.iter().skip(min_val + 1).any(|&c| c > 0) + }; + + if has_min && has_max { + return CoverageStatus::Green; + } else if has_min || has_max { + return CoverageStatus::Yellow; + } + } + CoverageStatus::Red + } + ExpressionKind::Cut(_) => { + // Green means match_count contains a nonzero value and parse_error contains a nonzero value + if has_matches && parse_error > 0 { + return CoverageStatus::Green; + } else if has_matches || parse_error > 0 { + return CoverageStatus::Yellow; + } + CoverageStatus::Red + } + _ => { + // Default logic for other expression kinds + if !has_matches && no_match == 0 && parse_error == 0 { + CoverageStatus::Red + } else if has_matches && !has_no_match { + CoverageStatus::Yellow + } else if has_matches && has_no_match { + CoverageStatus::Green + } else { + CoverageStatus::Red + } + } + } + } + + fn render_html(&self, output: &mut String, span_stack: &mut Vec, grammar: &Grammar) { + output.push_str(HTML_HEADER); + + // Group productions by category, preserving first-appearance order. + let mut category_order: Vec = Vec::new(); + let mut categories: std::collections::HashMap> = + std::collections::HashMap::new(); + for name in &grammar.name_order { + if let Some(prod) = grammar.productions.get(name) { + let cat = &prod.category; + if !categories.contains_key(cat) { + category_order.push(cat.clone()); + categories.insert(cat.clone(), Vec::new()); + } + categories.get_mut(cat).unwrap().push(name.as_str()); + } + } + + for category in &category_order { + output.push_str(&format!( + "

{}

\n", + html_escape(category) + )); + if let Some(names) = categories.get(category) { + for name in names { + if let Some(prod) = grammar.productions.get(*name) { + self.render_production(prod, output, span_stack); + } + } + } + output.push_str("
\n"); + } + + output.push_str(HTML_FOOTER); + } + + fn render_production( + &self, + prod: &grammar::Production, + output: &mut String, + span_stack: &mut Vec, + ) { + output.push_str("
"); + output.push_str(&format!( + "{}", + html_escape(&prod.name) + )); + output.push_str(" → "); + self.render_expression(&prod.expression, output, span_stack); + output.push_str("
\n"); + } + + fn render_expression( + &self, + expr: &Expression, + output: &mut String, + span_stack: &mut Vec, + ) { + if let ExpressionKind::Break(indent) = &expr.kind { + for _ in 0..span_stack.len() { + output.push_str(""); + } + output.push_str("
\n"); + for span in span_stack { + output.push_str(span); + } + for _ in 0..*indent { + output.push_str(" "); + } + return; + } + + if let ExpressionKind::Comment(s) = &expr.kind { + output.push_str(&format!( + "// {}", + html_escape(s) + )); + return; + } + + let status = self.get_coverage_status(expr.id, &expr.kind); + let bg_color = status.color(); + let has_error = self.parse_error.get(expr.id as usize).copied().unwrap_or(0) > 0; + + let tooltip = self.generate_tooltip(expr.id); + + let span = format!( + "", + bg_color, + html_escape(&tooltip).replace("'", "'") + ); + output.push_str(&span); + span_stack.push(span); + + if has_error { + output.push_str("●"); + } + + self.render_expression_kind(&expr.kind, output, span_stack); + + if let Some(suffix) = &expr.suffix { + output.push_str(&format!("{}", html_escape(suffix))); + } + + output.push_str(""); + span_stack.pop(); + } + + fn render_expression_kind( + &self, + kind: &ExpressionKind, + output: &mut String, + span_stack: &mut Vec, + ) { + match kind { + ExpressionKind::Grouped(e) => { + output.push_str("( "); + self.render_expression(e, output, span_stack); + output.push_str(" )"); + } + ExpressionKind::Alt(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + self.render_expression(e, output, span_stack); + if iter.peek().is_some() { + if !e.last_expr().is_break() { + output.push(' '); + } + output.push_str("| "); + } + } + } + ExpressionKind::Sequence(es) => { + let mut iter = es.iter().peekable(); + while let Some(e) = iter.next() { + self.render_expression(e, output, span_stack); + if iter.peek().is_some() && !e.last_expr().is_break() { + output.push(' '); + } + } + } + ExpressionKind::Optional(e) => { + self.render_expression(e, output, span_stack); + output.push_str("?"); + } + ExpressionKind::NegativeLookahead(e) => { + output.push('!'); + self.render_expression(e, output, span_stack); + } + ExpressionKind::Repeat(e) => { + self.render_expression(e, output, span_stack); + output.push_str("*"); + } + ExpressionKind::RepeatPlus(e) => { + self.render_expression(e, output, span_stack); + output.push_str("+"); + } + ExpressionKind::RepeatRange { + expr, + name, + min, + max, + limit, + } => { + self.render_expression(expr, output, span_stack); + output.push_str(""); + if let Some(n) = name { + output.push_str(&html_escape(n)); + output.push(':'); + } + if let Some(m) = min { + output.push_str(&m.to_string()); + } + output.push_str(&format!("{}", limit)); + if let Some(m) = max { + output.push_str(&m.to_string()); + } + output.push_str(""); + } + ExpressionKind::RepeatRangeNamed(e, name) => { + self.render_expression(e, output, span_stack); + output.push_str(&format!("{}", html_escape(name))); + } + ExpressionKind::Nt(nt) => { + output.push_str(&format!( + "{}", + html_escape(nt) + )); + } + ExpressionKind::Terminal(t) => { + output.push_str(&format!( + "`{}`", + html_escape(t) + )); + } + ExpressionKind::Prose(s) => { + output.push_str(&format!( + "<{}>", + html_escape(s) + )); + } + ExpressionKind::Break(_) | ExpressionKind::Comment(_) => { + // These are handled in render_expression to avoid coverage spans + unreachable!("Break and Comment should be handled in render_expression") + } + ExpressionKind::Charset(set) => { + output.push('['); + for (i, chars) in set.iter().enumerate() { + if i > 0 { + output.push(' '); + } + self.render_expression(chars, output, span_stack); + } + output.push(']'); + } + ExpressionKind::CharacterRange(a, b) => { + // TODO: It would be nice if this showed more info about the + // actual range that was covered. + let render_ch = |ch: &Character| -> String { + match ch { + Character::Char(c) => format!("`{}`", html_escape(&c.to_string())), + Character::Unicode((_, s)) => format!("U+{}", html_escape(s)), + } + }; + output.push_str(&render_ch(a)); + output.push('-'); + output.push_str(&render_ch(b)); + } + ExpressionKind::NegExpression(e) => { + output.push('~'); + self.render_expression(e, output, span_stack); + } + ExpressionKind::Cut(e) => { + output.push_str("^ "); + self.render_expression(e, output, span_stack); + } + ExpressionKind::Unicode((_, s)) => { + output.push_str(&format!("U+{}", html_escape(s))); + } + } + } + + fn generate_tooltip(&self, id: u32) -> String { + let mut tooltip = String::new(); + + tooltip.push_str(&format!("ID: {}\\n", id)); + + if let Some(counts) = self.match_count.get(id as usize) { + if counts.iter().any(|&c| c > 0) { + tooltip.push_str("Match counts:\\n"); + for (n, &count) in counts.iter().enumerate() { + if count > 0 { + let bar = "█".repeat((count.min(50) / 5).max(1) as usize); + tooltip.push_str(&format!(" {}: {} {}\\n", n, count, bar)); + } + } + } + } + + let no_match = self.no_match_count.get(id as usize).copied().unwrap_or(0); + if no_match > 0 { + tooltip.push_str(&format!("No match: {}\\n", no_match)); + } + + let parse_error = self.parse_error.get(id as usize).copied().unwrap_or(0); + if parse_error > 0 { + tooltip.push_str(&format!("Parse errors: {}\\n", parse_error)); + } + + if tooltip.ends_with("\\n") { + tooltip.truncate(tooltip.len() - 2); + } + + tooltip + } +} + +/// An indication of how well a node was covered. +#[derive(Debug, Clone, Copy)] +enum CoverageStatus { + /// Indicates the node was not covered at all. + Red, + /// Indicates the node was only partially covered. + Yellow, + /// Indicates the node was completely covered. + Green, +} + +impl CoverageStatus { + fn color(&self) -> &'static str { + match self { + CoverageStatus::Red => "#ffcccc", + CoverageStatus::Yellow => "#ffffcc", + CoverageStatus::Green => "#ccffcc", + } + } +} + +fn html_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +const HTML_HEADER: &str = r#" + + + + Grammar Coverage Report + + + +

Grammar Coverage Report

+
+
+"#; + +const HTML_FOOTER: &str = r#"
+ + + +"#; diff --git a/tools/parser/src/lexer.rs b/tools/parser/src/lexer.rs new file mode 100644 index 0000000000..a12647bc57 --- /dev/null +++ b/tools/parser/src/lexer.rs @@ -0,0 +1,281 @@ +//! Parser that can take Rust source and generates a sequence of tokens. + +use super::{Node, ParseError}; +use crate::coverage::Coverage; +use crate::parser::{SourceIndex, parse_production}; +use grammar::{ExpressionKind, Grammar, Production}; +use tracing::debug; + +#[derive(Clone)] +pub struct Tokens { + pub tokens: Vec, + /// Byte range of the shebang. + /// + /// The reference lexer is the only tool that sets this. + pub shebang: Option, + /// Byte range of the frontmatter. + /// + /// The reference lexer is the only tool that sets this. + pub frontmatter: Option, +} + +pub fn tokenize( + grammar: &Grammar, + coverage: &mut Coverage, + original_src: &str, +) -> Result { + let (normalized_src, removed_indices) = normalize_crlf(original_src); + + tokenize_normalized(grammar, coverage, &normalized_src) + .map(|mut tokens| { + for token in &mut tokens.tokens { + adjust_node(&removed_indices, token); + } + if let Some(shebang) = &mut tokens.shebang { + adjust_node(&removed_indices, shebang); + } + if let Some(frontmatter) = &mut tokens.frontmatter { + adjust_node(&removed_indices, frontmatter); + } + tokens + }) + .map_err(|mut err| { + err.byte_offset = map_offset(&removed_indices, err.byte_offset); + err + }) +} + +fn normalize_crlf(src: &str) -> (String, Vec) { + let mut normalized_src = String::with_capacity(src.len()); + let mut removed_indices = Vec::new(); + let mut chars = src.chars().peekable(); + while let Some(ch) = chars.next() { + if ch == '\r' + && let Some(&'\n') = chars.peek() + { + removed_indices.push(normalized_src.len()); + continue; + } + normalized_src.push(ch); + } + (normalized_src, removed_indices) +} + +fn map_offset(removed_indices: &[usize], offset: usize) -> usize { + offset + removed_indices.partition_point(|&x| x < offset) +} + +/// Adjusts the node range for CRLF normalization so that the range matches +/// the original source with the carriage returns. +fn adjust_node(removed_indices: &[usize], node: &mut Node) { + node.range.start = map_offset(removed_indices, node.range.start); + node.range.end = map_offset(removed_indices, node.range.end); + for child in &mut node.children.0 { + adjust_node(removed_indices, child); + } +} + +/// Tokenize source after it has been normalized. +fn tokenize_normalized( + grammar: &Grammar, + coverage: &mut Coverage, + src: &str, +) -> Result { + let top_prods = get_top_prods(grammar); + + let mut index = SourceIndex(0); + // Remove BOM + if src.starts_with('\u{FEFF}') { + index.0 += 3; + } + + let shebang; + (shebang, index) = parse_shebang(grammar, coverage, src, index)?; + let frontmatter; + (frontmatter, index) = parse_frontmatter(grammar, coverage, src, index)?; + let tokens = parse_tokens(grammar, coverage, &top_prods, src, index)?; + validate_delimiters_balanced(&tokens, src)?; + + debug!("lexing complete"); + + let tokens = Tokens { + tokens, + shebang, + frontmatter, + }; + + Ok(tokens) +} + +/// Returns the [`Production`]s that correspond to top-level tokens. +fn get_top_prods(grammar: &Grammar) -> Vec<&Production> { + let mut top_prods = Vec::new(); + let mut collect = |name| { + let prod = grammar.productions.get(name).unwrap(); + let ExpressionKind::Alt(es) = &prod.expression.kind else { + panic!("expected alts"); + }; + for e in es { + let nt = match &e.kind { + ExpressionKind::Sequence(es) => { + let seq: Vec<_> = es + .iter() + .filter_map(|e| match &e.kind { + ExpressionKind::Nt(nt) => Some(nt), + ExpressionKind::Break(_) | ExpressionKind::Comment(_) => None, + kind => panic!("unexpected kind {kind:?}"), + }) + .collect(); + assert_eq!(seq.len(), 1); + seq[0] + } + ExpressionKind::Nt(nt) => nt, + kind => panic!("unexpected kind {kind:?}"), + }; + top_prods.push(grammar.productions.get(nt).unwrap()); + } + }; + collect("COMMENT"); + collect("Token"); + top_prods +} + +fn parse_shebang( + grammar: &Grammar, + coverage: &mut Coverage, + src: &str, + index: SourceIndex, +) -> Result<(Option, SourceIndex), ParseError> { + let shebang = grammar.productions.get("SHEBANG").unwrap(); + if let Some((node, next_index)) = parse_production(grammar, coverage, shebang, &src, index)? { + Ok((Some(node), next_index)) + } else { + Ok((None, index)) + } +} + +fn parse_frontmatter( + grammar: &Grammar, + coverage: &mut Coverage, + src: &str, + index: SourceIndex, +) -> Result<(Option, SourceIndex), ParseError> { + let frontmatter = grammar.productions.get("FRONTMATTER").unwrap(); + if let Some((node, next_index)) = parse_production(grammar, coverage, frontmatter, &src, index) + .map_err(|e| ParseError { + message: format!("invalid frontmatter: {}", e.message), + byte_offset: e.byte_offset, + })? + { + Ok((Some(node), next_index)) + } else { + Ok((None, index)) + } +} + +/// Performs the actual parsing of all the tokens in the source. +fn parse_tokens( + grammar: &Grammar, + coverage: &mut Coverage, + top_prods: &[&Production], + src: &str, + mut index: SourceIndex, +) -> Result, ParseError> { + let mut tokens = Vec::new(); + let whitespace = grammar.productions.get("WHITESPACE").unwrap(); + + while index.0 < src.len() { + if let Some((_node, next_index)) = + parse_production(grammar, coverage, whitespace, &src, index)? + { + index = next_index; + continue; + } + + let mut matched_token = None; + for token_prod in top_prods { + debug!("try top-level token `{}`", token_prod.name); + if let Some((node, next_index)) = + parse_production(grammar, coverage, token_prod, &src, index)? + && node.byte_len() > 0 + { + index = next_index; + matched_token = Some(node); + break; + } + } + + match matched_token { + Some(mut node) => { + normalize_line_doc(&mut node, src); + tokens.push(node); + } + None => { + return Err(ParseError { + byte_offset: index.0, + message: String::from("no tokens matched"), + }); + } + } + } + Ok(tokens) +} + +fn validate_delimiters_balanced(tokens: &[Node], src: &str) -> Result<(), ParseError> { + let mut stack = Vec::new(); + for token in tokens { + let text = &src[token.range.clone()]; + match text { + "(" | "[" | "{" => stack.push((text, token.range.start)), + ")" => { + if stack.pop().map(|(s, _)| s) != Some("(") { + return Err(ParseError { + byte_offset: token.range.start, + message: "unbalanced `)`".to_string(), + }); + } + } + "]" => { + if stack.pop().map(|(s, _)| s) != Some("[") { + return Err(ParseError { + byte_offset: token.range.start, + message: "unbalanced `]`".to_string(), + }); + } + } + "}" => { + if stack.pop().map(|(s, _)| s) != Some("{") { + return Err(ParseError { + byte_offset: token.range.start, + message: "unbalanced `}`".to_string(), + }); + } + } + _ => {} + } + } + if let Some((_, offset)) = stack.pop() { + return Err(ParseError { + byte_offset: offset, + message: "unclosed delimiter".to_string(), + }); + } + Ok(()) +} + +/// Fix line doc comment range. +/// +/// The Reference models line doc comments as *content* followed by a +/// linefeed. However, rustc and proc-macro2 model it as everything excluding +/// the linefeed. For convenience, this normalizes the range so that it +/// matches the other tools. +/// +/// A real implementation using the Reference lexer would extract the content +/// from `LINE_DOC_COMMENT_CONTENT`, which does not include the linefeed. +fn normalize_line_doc(node: &mut Node, src: &str) { + if matches!(node.name.as_str(), "INNER_LINE_DOC" | "OUTER_LINE_DOC") + && src[node.range.clone()].ends_with('\n') + { + node.range.end -= 1; + } +} diff --git a/tools/parser/src/lib.rs b/tools/parser/src/lib.rs new file mode 100644 index 0000000000..e8e1c9daf0 --- /dev/null +++ b/tools/parser/src/lib.rs @@ -0,0 +1,119 @@ +//! Rust parser based on the Reference grammar. + +use std::ops::Range; +use std::str::FromStr; + +pub mod coverage; +pub mod lexer; +mod parser; +pub mod tree; + +#[derive(Clone, Debug)] +pub struct ParseError { + pub byte_offset: usize, + pub message: String, +} + +impl ParseError { + pub fn display(&self, src: &str) -> String { + let s = &src[self.byte_offset..]; + match s.char_indices().nth(100) { + Some((i, _)) => format!("{} at `{}…`", self.message, &s[..i]), + None => format!("{} at `{s}`", self.message), + } + } +} + +#[derive(Clone, Copy, PartialEq, PartialOrd, Debug, Eq)] +pub enum Edition { + Edition2015, + Edition2018, + Edition2021, + Edition2024, +} + +impl FromStr for Edition { + type Err = (); + fn from_str(s: &str) -> Result { + match s { + "2015" => Ok(Edition::Edition2015), + "2018" => Ok(Edition::Edition2018), + "2021" => Ok(Edition::Edition2021), + "2024" => Ok(Edition::Edition2024), + _ => Err(()), + } + } +} + +/// A parsed section of source corresponding to some grammar expression. +#[derive(Clone, Debug, Default)] +pub struct Node { + pub name: String, + /// Range in bytes of the original source that this node covers. + pub range: Range, + pub children: Nodes, +} + +impl Node { + pub fn new(name: String, range: Range) -> Node { + Node { + name, + range, + children: Nodes::default(), + } + } + + /// Returns a new `Node` with the given children. + fn with_children(name: String, start: usize, children: Nodes) -> Node { + let range = if children.0.is_empty() { + Range { start, end: start } + } else { + Range { + start: children.0.first().unwrap().range.start, + end: children.0.last().unwrap().range.end, + } + }; + Node { + name, + range, + children, + } + } + + /// Length in bytes of this node. + fn byte_len(&self) -> usize { + self.range.end - self.range.start + } +} + +/// Abstraction over a sequence of nodes. +#[derive(Clone, Debug, Default)] +pub struct Nodes(pub Vec); + +impl Nodes { + fn new(name: String, range: Range) -> Nodes { + let node = Node { + name, + range, + children: Nodes::default(), + }; + Nodes(vec![node]) + } + + /// Converts this `Nodes` to one with a single `Node`. + fn wrap(self, name: String, start: usize) -> Nodes { + Nodes(vec![Node::with_children(name.to_string(), start, self)]) + } + + fn extend(&mut self, other: Nodes) { + self.0.extend(other.0) + } + + fn byte_len(&self) -> usize { + if self.0.is_empty() { + 0 + } else { + self.0.last().unwrap().range.end - self.0.first().unwrap().range.start + } + } +} diff --git a/tools/parser/src/main.rs b/tools/parser/src/main.rs new file mode 100644 index 0000000000..46ad77b95c --- /dev/null +++ b/tools/parser/src/main.rs @@ -0,0 +1,41 @@ +use diagnostics::Diagnostics; +use parser::coverage::Coverage; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +fn main() { + let filter = tracing_subscriber::EnvFilter::builder() + .with_env_var("GRAMMAR_LOG") + .with_default_directive(tracing_subscriber::filter::LevelFilter::INFO.into()) + .from_env_lossy(); + + tracing_subscriber::registry() + .with(filter) + .with( + tracing_tree::HierarchicalLayer::new(2) + .with_writer(std::io::stderr) + .with_ansi(std::io::IsTerminal::is_terminal(&std::io::stderr())), + ) + .init(); + + let src = r###"r"test""###; + + let mut diag = Diagnostics::new(); + let grammar = grammar::load_grammar(&mut diag); + let mut coverage = Coverage::default(); + let ts = match parser::lexer::tokenize(&grammar, &mut coverage, src) { + Ok(ts) => ts, + Err(e) => { + eprintln!("error: {}", e.display(src)); + std::process::exit(1); + } + }; + for token in ts.tokens { + eprintln!( + "{} {:?}: `{}`", + token.name, + token.range.clone(), + &src[token.range] + ); + } +} diff --git a/tools/parser/src/parser.rs b/tools/parser/src/parser.rs new file mode 100644 index 0000000000..16e9be3de1 --- /dev/null +++ b/tools/parser/src/parser.rs @@ -0,0 +1,630 @@ +//! The generic interpreter of the Reference grammar. + +use super::{Node, Nodes, ParseError}; +use crate::coverage::Coverage; +use grammar::{Expression, ExpressionKind, Grammar, Production, RangeLimit}; +use std::collections::HashMap; +use std::ops::Range; +use tracing::instrument; + +/// This stores named repetitions. +/// +/// The key is the name, and the value is the number of repetitions that +/// happened. +#[derive(Debug, Default)] +struct Environment { + map: HashMap, +} + +/// A wrapper around an index for referring to elements in a [`Source`]. +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Copy, Clone)] +pub(crate) struct SourceIndex(pub(crate) usize); + +/// Abstracts different kinds of sources for the parser. +/// +/// This allows the parser to be used for both string sources and tokenized +/// sources. String sources work in elements of bytes of a string, whereas +/// token sources work in elements of tokens. The offsets are based on +/// elements in the sequence represented with [`SourceIndex`]. +pub(crate) trait Source { + /// Returns a substring from the given offset of the given length in bytes. + /// + /// If this does not match an entire token, it returns None. + fn get_substring(&self, offset: SourceIndex, bytes: usize) -> Option<(&str, Range)>; + + /// Returns the element at the given offset. + fn get_element(&self, offset: SourceIndex) -> Option<(&str, Range)>; + + /// Returns the number of elements in the source. + fn len(&self) -> SourceIndex; + + /// Returns what the next index should be when advanced from the current + /// index with the given number of bytes. + fn advance(&self, index: SourceIndex, bytes: usize) -> SourceIndex; + + /// If this is a token source, returns the node at the given index. + /// + /// Returns `None` if past the end of the input. + /// + /// This is essentially a hack to create a boundary between the lexer and + /// the tree parser. + fn get_node(&self, index: SourceIndex) -> Option<&Node>; + + /// Returns the byte offset of the start of the given element. + /// + /// When the index is at the end, returns the offset of the last element. + fn index_to_bytes(&self, index: SourceIndex) -> usize; +} + +impl Source for &str { + fn get_substring(&self, offset: SourceIndex, bytes: usize) -> Option<(&str, Range)> { + let end = offset.0.checked_add(bytes)?; + if end > (*self).len() { + return None; + } + if !self.is_char_boundary(offset.0) || !self.is_char_boundary(end) { + return None; + } + let s = &self[offset.0..end]; + let range = Range { + start: offset.0, + end, + }; + Some((s, range)) + } + + fn get_element(&self, offset: SourceIndex) -> Option<(&str, Range)> { + let ch = self[offset.0..].chars().next()?; + let len = ch.len_utf8(); + let s = &self[offset.0..offset.0 + len]; + let range = Range { + start: offset.0, + end: offset.0 + len, + }; + Some((s, range)) + } + + fn len(&self) -> SourceIndex { + SourceIndex((*self).len()) + } + + fn advance(&self, index: SourceIndex, bytes: usize) -> SourceIndex { + SourceIndex(index.0 + bytes) + } + + fn get_node(&self, _index: SourceIndex) -> Option<&Node> { + None + } + + fn index_to_bytes(&self, index: SourceIndex) -> usize { + index.0 + } +} + +/// Parse a production and return the Node with name from the production. +pub(crate) fn parse_production( + grammar: &Grammar, + coverage: &mut Coverage, + prod: &Production, + src: &dyn Source, + index: SourceIndex, +) -> Result, ParseError> { + let r = parse( + grammar, + coverage, + &prod.expression, + src, + index, + &mut Environment::default(), + )? + .map(|(children, next_index)| { + let children = Node::with_children(prod.name.clone(), src.index_to_bytes(index), children); + (children, next_index) + }); + Ok(r) +} + +/// Parse an expression. +/// +/// Returns `Ok(None)` if the expression does not match. Otherwise, it +/// returns the [`Nodes`] that match, along with the new index pointing +/// just after the matched nodes. +/// +/// Note that some expressions match zero elements (like `e*` when `e` doesn't +/// match), and those are treated as a successful match where `Nodes` is +/// empty. +/// +/// Returns `Err` if there is some kind of syntax error. +#[instrument(level = "debug", skip(grammar, e, src, coverage), ret)] +fn parse( + grammar: &Grammar, + coverage: &mut Coverage, + e: &Expression, + src: &dyn Source, + index: SourceIndex, + env: &mut Environment, +) -> Result, ParseError> { + tracing::debug!("e={e}"); + if index < src.len() { + tracing::debug!("next={:?}", src.get_element(index)); + } else { + tracing::debug!("eof"); + } + let cov_match = |coverage: &mut Coverage, count| coverage.cov_match(e.id, count as u32); + let cov_no_match = |coverage: &mut Coverage| coverage.cov_no_match(e.id); + let cov_parse_error = |coverage: &mut Coverage| coverage.cov_parse_error(e.id); + match &e.kind { + ExpressionKind::Grouped(group) => { + assert_eq!(e.suffix, None); + match parse(grammar, coverage, group, src, index, env)? { + Some((nodes, i)) => { + cov_match(coverage, 1); + Ok(Some(( + nodes.wrap(format!("Group({group})"), src.index_to_bytes(index)), + i, + ))) + } + None => { + cov_no_match(coverage); + Ok(None) + } + } + } + ExpressionKind::Alt(es) => { + assert_eq!(e.suffix, None); + for e in es { + if let Some(r) = parse(grammar, coverage, e, src, index, env)? { + cov_match(coverage, 1); + return Ok(Some(r)); + } + } + cov_no_match(coverage); + Ok(None) + } + ExpressionKind::Sequence(es) => { + assert_eq!(e.suffix, None); + let mut current = index; + let mut children = Vec::new(); + for e in es { + if matches!( + e.kind, + ExpressionKind::Break(_) | ExpressionKind::Comment(_) + ) { + continue; + } + match parse(grammar, coverage, e, src, current, env)? { + Some((nodes, next_index)) => { + current = next_index; + children.extend(nodes.0); + } + None => { + cov_no_match(coverage); + return Ok(None); + } + } + } + cov_match(coverage, 1); + Ok(Some((Nodes(children), current))) + } + ExpressionKind::Optional(opt) => { + assert_eq!(e.suffix, None); + match parse(grammar, coverage, opt, src, index, env)? { + Some((children, next_index)) => { + cov_match(coverage, 1); + Ok(Some(( + children.wrap(format!("Optional({opt})"), src.index_to_bytes(index)), + next_index, + ))) + } + None => { + cov_match(coverage, 0); + Ok(Some((Nodes::default(), index))) + } + } + } + ExpressionKind::NegativeLookahead(n) => { + assert_eq!(e.suffix, None); + match parse(grammar, coverage, n, src, index, env)? { + Some(_) => { + cov_match(coverage, 1); + Ok(None) + } + None => { + cov_no_match(coverage); + Ok(Some((Nodes::default(), index))) + } + } + } + ExpressionKind::Repeat(r) => { + assert_eq!(e.suffix, None); + let mut current = index; + let mut children = Nodes::default(); + while current < src.len() { + match parse(grammar, coverage, r, src, current, env)? { + Some((nodes, next_index)) => { + current = next_index; + children.extend(nodes); + } + None => break, + } + } + cov_match(coverage, children.0.len()); + Ok(Some(( + children.wrap(format!("Repeat({r})"), src.index_to_bytes(index)), + current, + ))) + } + ExpressionKind::RepeatPlus(r) => { + assert_eq!(e.suffix, None); + let mut current = index; + let mut children = Nodes::default(); + while current < src.len() { + match parse(grammar, coverage, r, src, current, env)? { + Some((nodes, next_index)) => { + current = next_index; + children.extend(nodes); + } + None => break, + } + } + if current == index { + cov_no_match(coverage); + Ok(None) + } else { + cov_match(coverage, children.0.len()); + Ok(Some(( + children.wrap(format!("RepeatPlus({r})"), src.index_to_bytes(index)), + current, + ))) + } + } + ExpressionKind::RepeatRange { + expr: r, + name, + min, + max, + limit, + } => { + let max = max.map(|max| match limit { + RangeLimit::HalfOpen => max - 1, + RangeLimit::Closed => max, + }); + let mut current = index; + let mut children = Nodes::default(); + let mut count = 0; + while current < src.len() { + match parse(grammar, coverage, r, src, current, env)? { + Some((nodes, next_index)) => { + current = next_index; + children.extend(nodes); + count += 1; + if let Some(max) = max + && count == max + { + break; + } + } + None => break, + } + } + if let Some(min) = min + && count < *min + { + cov_no_match(coverage); + return Ok(None); + } + if let Some(name) = name { + assert!(env.map.insert(name.clone(), count).is_none()); + } + + let start_byte_offset = src.index_to_bytes(index); + match e.suffix.as_deref() { + Some("valid hex char value") => { + let end = src.index_to_bytes(current); + let len = end - start_byte_offset; + let (hex, _) = src.get_substring(index, len).unwrap(); + let hex_no_underscores = hex.replace('_', ""); + let value = u32::from_str_radix(&hex_no_underscores, 16).map_err(|_| { + cov_parse_error(coverage); + ParseError { + byte_offset: start_byte_offset, + message: format!("invalid hex value: {hex}"), + } + })?; + if char::from_u32(value).is_none() { + cov_parse_error(coverage); + return Err(ParseError { + byte_offset: start_byte_offset, + message: format!("invalid Unicode scalar value: {hex}"), + }); + } + } + Some(s) => panic!("unknown suffix {s:?}"), + None => {} + } + + cov_match(coverage, children.0.len()); + Ok(Some(( + children.wrap(format!("RepatRange({r})"), start_byte_offset), + current, + ))) + } + ExpressionKind::RepeatRangeNamed(r, name) => { + assert_eq!(e.suffix, None); + let Some(count) = env.map.get(name) else { + panic!("expected {name} in environment for {r}"); + }; + let mut current = index; + let mut children = Nodes::default(); + for _ in 0..*count { + match parse(grammar, coverage, r, src, current, env)? { + Some((nodes, next_index)) => { + current = next_index; + children.extend(nodes); + } + None => { + cov_no_match(coverage); + return Ok(None); + } + } + } + cov_match(coverage, children.0.len()); + Ok(Some(( + children.wrap( + format!("RepeatRangeNamed({r}, {name})"), + src.index_to_bytes(index), + ), + current, + ))) + } + ExpressionKind::Nt(s) => { + let Some((nodes, next_index)) = parse_nt(grammar, s, src, index, env, coverage)? else { + cov_no_match(coverage); + return Ok(None); + }; + let len = nodes.byte_len(); + let (matched, _) = src.get_substring(index, len).unwrap(); + match e.suffix.as_deref() { + Some("except `b` or `c` or `r` or `br` or `cr`") => { + if matches!(matched, "b" | "c" | "r" | "br" | "cr") { + cov_no_match(coverage); + return Ok(None); + } + } + Some("except `b`") => { + if matched == "b" { + cov_no_match(coverage); + return Ok(None); + } + } + Some("except `r` or `br` or `cr`") => { + if matches!(matched, "r" | "br" | "cr") { + cov_no_match(coverage); + return Ok(None); + } + } + Some("except `r`") => { + if matched == "r" { + cov_no_match(coverage); + return Ok(None); + } + } + Some( + "except a [strict][lex.keywords.strict] or [reserved][lex.keywords.reserved] keyword", + ) => { + let strict = grammar.productions.get("STRICT_KEYWORDS").unwrap(); + let reserved = grammar.productions.get("RESERVED_KEYWORDS").unwrap(); + for e in [&strict.expression, &reserved.expression] { + if let Ok(Some((nodes, _))) = parse(grammar, coverage, e, src, index, env) + && nodes.byte_len() > 0 + { + cov_no_match(coverage); + return Ok(None); + } + } + } + Some("except [delimiters][lex.token.delim]") => { + if matches!(matched, "{" | "}" | "[" | "]" | "(" | ")") { + cov_no_match(coverage); + return Ok(None); + } + } + Some(suffix) => panic!("unknown suffix {suffix:?}"), + None => {} + } + cov_match(coverage, 1); + Ok(Some((nodes, next_index))) + } + ExpressionKind::Terminal(s) => { + let Some((next_s, range)) = src.get_substring(index, s.len()) else { + cov_no_match(coverage); + return Ok(None); + }; + if next_s != s { + cov_no_match(coverage); + return Ok(None); + } + let next_index = src.advance(index, s.len()); + match e.suffix.as_deref() { + Some("immediately followed by LF") => { + if let Some((next_s, _)) = src.get_element(next_index) + && next_s != "\n" + { + cov_no_match(coverage); + return Ok(None); + } + } + Some(suffix) => panic!("unknown suffix {suffix:?}"), + None => {} + } + let nodes = Nodes::new(format!("Terminal {s:?}"), range); + cov_match(coverage, 1); + Ok(Some((nodes, next_index))) + } + ExpressionKind::Prose(s) => { + assert_eq!(e.suffix, None); + match match_prose(s, src, index) { + Some(r) => { + cov_match(coverage, 1); + Ok(Some(r)) + } + None => { + cov_no_match(coverage); + Ok(None) + } + } + } + ExpressionKind::Break(_) => unreachable!(), + ExpressionKind::Comment(_) => unreachable!(), + ExpressionKind::Charset(chars) => { + assert_eq!(e.suffix, None); + for ch in chars { + if let Some(r) = parse(grammar, coverage, ch, src, index, env)? { + cov_match(coverage, 1); + return Ok(Some(r)); + } + } + cov_no_match(coverage); + Ok(None) + } + ExpressionKind::CharacterRange(a, b) => { + let Some((next, range)) = src.get_element(index) else { + cov_no_match(coverage); + return Ok(None); + }; + if next.chars().count() == 1 { + let ch = next.chars().next().unwrap(); + if ch >= a.get_ch() && ch <= b.get_ch() { + let next_index = src.advance(index, ch.len_utf8()); + let nodes = Nodes::new(format!("Range {a:?} to {b:?}"), range); + // TODO: Would be nice to record coverage of how much of the range is covered. + cov_match(coverage, 1); + return Ok(Some((nodes, next_index))); + } + } + cov_no_match(coverage); + Ok(None) + } + ExpressionKind::NegExpression(neg) => { + assert_eq!(e.suffix, None); + match parse(grammar, coverage, neg, src, index, env)? { + Some(_) => { + cov_no_match(coverage); + Ok(None) + } + None => { + if let Some((s, range)) = src.get_element(index) { + let next_index = src.advance(index, s.len()); + let nodes = Nodes::new(format!("NegExpression {neg}"), range); + cov_match(coverage, 1); + Ok(Some((nodes, next_index))) + } else { + cov_no_match(coverage); + Ok(None) + } + } + } + } + ExpressionKind::Cut(inner) => { + assert_eq!(e.suffix, None); + match parse(grammar, coverage, inner, src, index, env)? { + Some(r) => { + cov_match(coverage, 1); + Ok(Some(r)) + } + None => { + cov_parse_error(coverage); + Err(ParseError { + byte_offset: src.index_to_bytes(index), + message: format!("expected {}", inner), + }) + } + } + } + ExpressionKind::Unicode((ch, s)) => { + assert_eq!(e.suffix, None); + let mut buf = [0u8; 4]; + let c_str = ch.encode_utf8(&mut buf); + if let Some((next_s, range)) = src.get_element(index) + && next_s == c_str + { + let next_index = src.advance(index, ch.len_utf8()); + cov_match(coverage, 1); + Ok(Some(( + Nodes::new(format!("Unicode {s}"), range), + next_index, + ))) + } else { + cov_no_match(coverage); + Ok(None) + } + } + } +} + +fn parse_nt( + grammar: &Grammar, + prod_name: &str, + src: &dyn Source, + index: SourceIndex, + env: &mut Environment, + coverage: &mut Coverage, +) -> Result, ParseError> { + let prod = grammar.productions.get(prod_name).unwrap(); + // If this matches a lexer token, don't parse it and use the token + // directly. The lexer rules are incompatible when reading tokens. + let (nodes, next_index) = if let Some(node) = src.get_node(index) + && node.name == prod.name + { + (Nodes(vec![node.clone()]), SourceIndex(index.0 + 1)) + } else { + let nodes = parse(grammar, coverage, &prod.expression, src, index, env)?; + let Some((nodes, next_index)) = nodes else { + return Ok(None); + }; + ( + nodes.wrap(prod.name.clone(), src.index_to_bytes(index)), + next_index, + ) + }; + Ok(Some((nodes, next_index))) +} + +fn match_prose(prose: &str, src: &dyn Source, index: SourceIndex) -> Option<(Nodes, SourceIndex)> { + let next_as_ch = || { + src.get_element(index).and_then(|(next, range)| { + let mut chars = next.chars(); + let ch = chars.next().unwrap(); + if chars.next().is_some() { + None + } else { + Some((ch, range)) + } + }) + }; + + match prose { + "`XID_Start` defined by Unicode" => { + if let Some((ch, range)) = next_as_ch() { + unicode_ident::is_xid_start(ch).then(|| { + let nodes = Nodes::new(format!("Prose: {prose}"), range); + (nodes, src.advance(index, ch.len_utf8())) + }) + } else { + None + } + } + "`XID_Continue` defined by Unicode" => { + if let Some((ch, range)) = next_as_ch() { + unicode_ident::is_xid_continue(ch).then(|| { + let nodes = Nodes::new(format!("Prose: {prose}"), range); + (nodes, src.advance(index, ch.len_utf8())) + }) + } else { + None + } + } + + p => panic!("unknown prose {p}"), + } +} diff --git a/tools/parser/src/tree.rs b/tools/parser/src/tree.rs new file mode 100644 index 0000000000..75bfc7bea3 --- /dev/null +++ b/tools/parser/src/tree.rs @@ -0,0 +1,89 @@ +//! Parser that can take Rust source and generate a parse tree. + +use super::{Node, ParseError}; +use crate::coverage::Coverage; +use crate::lexer::tokenize; +use crate::parser::parse_production; +use crate::parser::{Source, SourceIndex}; +use grammar::Grammar; +use std::ops::Range; + +struct TokenSource<'src> { + src: &'src str, + tokens: Vec, +} + +impl Source for TokenSource<'_> { + fn get_substring(&self, offset: SourceIndex, bytes: usize) -> Option<(&str, Range)> { + self.tokens.get(offset.0).and_then(|t| { + let s = &self.src[t.range.clone()]; + if !s.len() == bytes { + None + } else { + Some((s, t.range.clone())) + } + }) + } + + fn get_element(&self, offset: SourceIndex) -> Option<(&str, Range)> { + self.tokens.get(offset.0).map(|t| { + let s = &self.src[t.range.clone()]; + (s, t.range.clone()) + }) + } + + fn len(&self) -> SourceIndex { + SourceIndex(self.tokens.len()) + } + + fn advance(&self, index: SourceIndex, bytes: usize) -> SourceIndex { + let token = &self.tokens[index.0]; + if token.byte_len() != bytes { + panic!("advancing {bytes} at {index:?} is not equal to {token:?}"); + } + SourceIndex(index.0 + 1) + } + + fn get_node(&self, index: SourceIndex) -> Option<&Node> { + self.tokens.get(index.0) + } + + fn index_to_bytes(&self, index: SourceIndex) -> usize { + if index.0 == self.tokens.len() { + self.tokens[index.0 - 1].range.end + } else { + self.tokens[index.0].range.start + } + } +} + +/// Parse Rust source for the given named production, and return a [`Node`] tree. +pub fn parse(grammar: &Grammar, src: &str, production: &str) -> Result { + let mut coverage = Coverage::default(); + + let krate = grammar.productions.get(production).unwrap(); + + let tokens = tokenize(grammar, &mut coverage, src)?; + + // Strip comments. + let tokens = tokens + .tokens + .into_iter() + .filter(|token| !matches!(token.name.as_str(), "LINE_COMMENT" | "BLOCK_COMMENT")) + .collect(); + + let token_source = TokenSource { src, tokens }; + + match parse_production(grammar, &mut coverage, krate, &token_source, SourceIndex(0))? { + Some((node, next_index)) => { + if next_index < token_source.len() { + return Err(ParseError { + message: format!("{production} production failed to parse all tokens"), + byte_offset: token_source.index_to_bytes(next_index), + }); + } + Ok(node) + } + None => panic!("input did not match {production}"), + } +} From 508c53c649c00fb59f6d2e5a68252884ab24e152 Mon Sep 17 00:00:00 2001 From: Travis Cross Date: Tue, 18 Aug 2026 21:56:14 +0000 Subject: [PATCH 11/11] Fix typo --- tools/grammar-check/src/permute.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/grammar-check/src/permute.rs b/tools/grammar-check/src/permute.rs index c8c3cebfbb..17bcbf0a2b 100644 --- a/tools/grammar-check/src/permute.rs +++ b/tools/grammar-check/src/permute.rs @@ -5,7 +5,7 @@ //! both valid and invalid inputs (particularly those that are truncated). //! //! It generates representative inputs for some of the expressions. For -//! example, a a repetition generates an output that includes 0, 1, or 2 +//! example, a repetition generates an output that includes 0, 1, or 2 //! repetitions of the expression. Or something like "Identifier" just does a //! few representative values like "a", "ab", and "abb" (with the assumption //! that the Identifier grammar is already correct).