diff --git a/crates/oxc_angular_compiler/src/parser/expression/lexer.rs b/crates/oxc_angular_compiler/src/parser/expression/lexer.rs index f0680c00e..958f0ace3 100644 --- a/crates/oxc_angular_compiler/src/parser/expression/lexer.rs +++ b/crates/oxc_angular_compiler/src/parser/expression/lexer.rs @@ -454,7 +454,8 @@ impl<'a> Lexer<'a> { /// Advances the index by the current character's byte length and returns the character. fn advance(&mut self) -> char { let ch = self.peek(); - if ch != chars::EOF { + // By the index, not the character: a NUL in the input reads as `EOF` too. + if !self.is_eof() { self.index += ch.len_utf8() as u32; } ch @@ -465,9 +466,15 @@ impl<'a> Lexer<'a> { self.index >= self.length } + /// Whitespace as Angular's `scanToken` skips it: every character up to the space (NUL and the + /// other control characters too), and NBSP. + fn is_skipped_whitespace(ch: char) -> bool { + ch <= chars::SPACE || ch == chars::NBSP + } + /// Skips whitespace characters. fn skip_whitespace(&mut self) { - while !self.is_eof() && chars::is_whitespace(self.peek()) { + while !self.is_eof() && Self::is_skipped_whitespace(self.peek()) { self.advance(); } } @@ -477,7 +484,7 @@ impl<'a> Lexer<'a> { let ch = self.advance(); // Whitespace - if chars::is_whitespace(ch) { + if Self::is_skipped_whitespace(ch) { return; } diff --git a/crates/oxc_angular_compiler/src/parser/html/lexer.rs b/crates/oxc_angular_compiler/src/parser/html/lexer.rs index ea9e69c8d..e42b3afd3 100644 --- a/crates/oxc_angular_compiler/src/parser/html/lexer.rs +++ b/crates/oxc_angular_compiler/src/parser/html/lexer.rs @@ -706,7 +706,8 @@ impl<'a> HtmlLexer<'a> { /// Tokenizes the input. pub fn tokenize(mut self) -> HtmlTokenizeResult { - while self.index < self.length { + // Like Angular's loop, stop at a NUL: its cursor peeks one as `$EOF`. + while self.index < self.length && self.peek() != chars::EOF { self.scan_token(); } @@ -1508,6 +1509,11 @@ impl<'a> HtmlLexer<'a> { continue; } + // A NUL ends the parameter, but not inside a quote, as in Angular. + if ch == chars::EOF && in_quote.is_none() { + break; + } + // If in a quote, check for quote end if let Some(quote_char) = in_quote { if ch == quote_char { @@ -1603,7 +1609,8 @@ impl<'a> HtmlLexer<'a> { let mut in_quote: Option = None; let mut in_comment = false; - while self.index < self.length { + // A NUL ends the interpolation, even inside a quote, as in Angular. + while self.index < self.length && self.peek() != chars::EOF { let ch = self.peek(); // Check for tag start - break out of interpolation @@ -1652,6 +1659,11 @@ impl<'a> HtmlLexer<'a> { in_comment = true; self.advance(); self.advance(); + // Like Angular, also take the character after `//` before checking again. + if self.peek() == '\\' { + self.advance(); + } + self.advance(); continue; } @@ -1945,7 +1957,8 @@ impl<'a> HtmlLexer<'a> { } } - if self.peek() == chars::EOF { + // Raw text reads through a NUL, as Angular's `_consumeRawText` does. + if self.index >= self.length { // End of input - emit what we have (if non-empty) if consume_entities { if text_start < self.index { @@ -2149,7 +2162,7 @@ impl<'a> HtmlLexer<'a> { // Not our closing tag - continue scanning } - if self.peek() == chars::EOF { + if self.index >= self.length { // EOF reached - emit whatever content we have let content = &self.input[content_start as usize..self.index as usize]; let normalized = normalize_line_endings(content); @@ -2495,7 +2508,7 @@ impl<'a> HtmlLexer<'a> { let starts_with_bracket = self.peek() == '['; let mut open_brackets: i32 = 0; - while self.peek() != chars::EOF { + while self.index < self.length { let ch = self.peek(); // Track bracket depth (can go negative for mismatched brackets) @@ -2511,9 +2524,10 @@ impl<'a> HtmlLexer<'a> { // - When brackets are balanced or mismatched (open_brackets <= 0): // stop at name-end characters (whitespace, =, >, <, /, ', ", EOF) // - When brackets are unbalanced with more opens (open_brackets > 0): - // only stop on newline (interrupts matching) + // only stop on newline (interrupts matching), so a NUL is read if open_brackets <= 0 { - if chars::is_whitespace(ch) + if ch == chars::EOF + || chars::is_whitespace(ch) || ch == '=' || ch == '>' || ch == '<' @@ -2531,7 +2545,8 @@ impl<'a> HtmlLexer<'a> { } } else { // Normal attribute name - stop at standard name-end characters - if chars::is_whitespace(ch) + if ch == chars::EOF + || chars::is_whitespace(ch) || ch == '=' || ch == '>' || ch == '<' @@ -2739,7 +2754,8 @@ impl<'a> HtmlLexer<'a> { let mut text_start = self.index; - while self.peek() != quote && self.peek() != chars::EOF { + // Only the quote ends the value: like Angular, read a NUL as text (an interpolation still stops there). + while self.peek() != quote && self.index < self.length { if self.starts_with(self.interpolation_start) { // Emit any accumulated text first (or empty text token if at start of interpolation) let text = &self.input[text_start as usize..self.index as usize]; @@ -3124,9 +3140,10 @@ impl<'a> HtmlLexer<'a> { } /// Reads characters until the given terminator (does not consume the terminator). + /// Like Angular's `_readUntil`, a NUL is read as text. fn read_until(&mut self, terminator: char) -> String { let start = self.index; - while self.peek() != terminator && self.peek() != chars::EOF { + while self.peek() != terminator && self.index < self.length { self.advance(); } self.input[start as usize..self.index as usize].to_string() @@ -3333,8 +3350,8 @@ impl<'a> HtmlLexer<'a> { // Numeric entity doesn't end with semicolon - this is an error // Advance cursor to include the peeked character in the error message - // (unless we're at EOF) - if self.peek() == chars::EOF { + // (unless we're at the end; a NUL is read like Angular's cursor does) + if self.index >= self.length { // EOF - report "Unexpected character EOF" self.errors.push(HtmlTokenError { msg: "Unexpected character \"EOF\"".to_string(), diff --git a/crates/oxc_angular_compiler/tests/expression_lexer_test.rs b/crates/oxc_angular_compiler/tests/expression_lexer_test.rs index 8fcdbd6b2..01d2205df 100644 --- a/crates/oxc_angular_compiler/tests/expression_lexer_test.rs +++ b/crates/oxc_angular_compiler/tests/expression_lexer_test.rs @@ -355,6 +355,17 @@ mod token { expect_identifier_token(&tokens[1], 8, 9, "b"); } + #[test] + fn should_skip_control_characters_like_whitespace() { + // Angular's `scanToken` skips every character up to the space, NUL among them. + for input in ["a\0b", "a\x01b"] { + let tokens = lex(input); + assert_eq!(tokens.len(), 2); + expect_identifier_token(&tokens[0], 0, 1, "a"); + expect_identifier_token(&tokens[1], 2, 3, "b"); + } + } + #[test] fn should_tokenize_quoted_string() { let tokens = lex(r#"['\'', "\""]"#); @@ -632,6 +643,17 @@ mod template_literals { assert_eq!(tokens[0].str_value, "\u{00A0}"); } + #[test] + fn should_skip_a_nul_inside_an_interpolation() { + let tokens = lex("`${a\0}`"); + assert_eq!(tokens.len(), 5); + expect_string_token(&tokens[0], 0, 1, "", StringTokenKind::TemplateLiteralPart); + expect_operator_token(&tokens[1], 1, 3, "${"); + expect_identifier_token(&tokens[2], 3, 4, "a"); + expect_character_token(&tokens[3], 5, 6, "}"); + expect_string_token(&tokens[4], 6, 7, "", StringTokenKind::TemplateLiteralEnd); + } + #[test] fn should_tokenize_template_literal_with_interpolation_at_end() { let tokens = lex("`hello ${name}`"); @@ -1176,6 +1198,16 @@ mod regular_expressions { mod errors { use super::*; + #[test] + fn should_report_a_nul_in_a_string_as_unterminated() { + // As in Angular: the string ends at the NUL, `b` is read on its own, and the last quote opens another string. + let tokens = lex("'a\0b'"); + assert_eq!(tokens.len(), 3); + assert!(tokens[0].is_error()); + expect_identifier_token(&tokens[1], 3, 4, "b"); + assert!(tokens[2].is_error()); + } + #[test] fn should_report_unterminated_string() { let tokens = lex(r#""hello"#); diff --git a/crates/oxc_angular_compiler/tests/html_lexer_test.rs b/crates/oxc_angular_compiler/tests/html_lexer_test.rs index b950f3ce6..acf01a450 100644 --- a/crates/oxc_angular_compiler/tests/html_lexer_test.rs +++ b/crates/oxc_angular_compiler/tests/html_lexer_test.rs @@ -1900,6 +1900,162 @@ mod positions { mod edge_cases { use super::*; + /// Tokenizes and returns humanized parts, checking that there are no errors. + fn humanize_parts_without_errors(input: &str, icu: bool) -> Vec<(HtmlTokenType, Vec)> { + let result = HtmlLexer::new(input).with_expansion_forms(icu).tokenize(); + assert!(result.errors.is_empty(), "{input:?}: {:?}", result.errors); + result.tokens.into_iter().map(|t| (t.token_type, t.parts)).collect() + } + + #[test] + fn should_stop_at_a_nul_in_text_and_interpolation() { + // Angular's cursor peeks a NUL as `$EOF`, which ends text and interpolations. + assert_eq!( + humanize_parts_without_errors("a\0{{ b }}", false), + vec![ + (HtmlTokenType::TagOpenStart, vec![String::new(), "b".to_string()]), + (HtmlTokenType::TagOpenEnd, vec![]), + (HtmlTokenType::Text, vec!["a".to_string()]), + (HtmlTokenType::TagClose, vec![String::new(), "b".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + assert_eq!( + humanize_parts_without_errors("{{ a\0 }}", false), + vec![ + (HtmlTokenType::Text, vec![String::new()]), + (HtmlTokenType::Interpolation, vec!["{{".to_string(), " a".to_string()]), + (HtmlTokenType::Text, vec![String::new()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_read_a_nul_in_comments_and_cdata() { + // These read to their closing marker, so a NUL is just text in them. + assert_eq!( + humanize_parts_without_errors("c", false), + vec![ + (HtmlTokenType::CommentStart, vec![]), + (HtmlTokenType::RawText, vec!["a\0b".to_string()]), + (HtmlTokenType::CommentEnd, vec![]), + (HtmlTokenType::Text, vec!["c".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + assert_eq!( + humanize_parts_without_errors("c", false), + vec![ + (HtmlTokenType::CdataStart, vec![]), + (HtmlTokenType::RawText, vec!["a\0b".to_string()]), + (HtmlTokenType::CdataEnd, vec![]), + (HtmlTokenType::Text, vec!["c".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_read_a_nul_in_raw_text() { + assert_eq!( + humanize_parts_without_errors("c", false), + vec![ + (HtmlTokenType::TagOpenStart, vec![String::new(), "textarea".to_string()]), + (HtmlTokenType::TagOpenEnd, vec![]), + (HtmlTokenType::EscapableRawText, vec!["a\0b".to_string()]), + (HtmlTokenType::TagClose, vec![String::new(), "textarea".to_string()]), + (HtmlTokenType::Text, vec!["c".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_read_a_nul_in_a_quoted_attribute_value() { + assert_eq!( + humanize_parts_without_errors("
z
", false), + vec![ + (HtmlTokenType::TagOpenStart, vec![String::new(), "div".to_string()]), + (HtmlTokenType::AttrName, vec![String::new(), "a".to_string()]), + (HtmlTokenType::AttrQuote, vec!["\"".to_string()]), + (HtmlTokenType::AttrValueText, vec!["x\0y".to_string()]), + (HtmlTokenType::AttrQuote, vec!["\"".to_string()]), + (HtmlTokenType::TagOpenEnd, vec![]), + (HtmlTokenType::Text, vec!["z".to_string()]), + (HtmlTokenType::TagClose, vec![String::new(), "div".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_stop_a_block_parameter_at_a_nul_outside_quotes() { + assert_eq!( + humanize_parts_without_errors("@if (a\0) {x}", false), + vec![ + (HtmlTokenType::IncompleteBlockOpen, vec!["if".to_string()]), + (HtmlTokenType::BlockParameter, vec!["a".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + assert_eq!( + humanize_parts_without_errors("@if (\"a\0b\") {x}", false), + vec![ + (HtmlTokenType::BlockOpenStart, vec!["if".to_string()]), + (HtmlTokenType::BlockParameter, vec!["\"a\0b\"".to_string()]), + (HtmlTokenType::BlockOpenEnd, vec![]), + (HtmlTokenType::Text, vec!["x".to_string()]), + (HtmlTokenType::BlockClose, vec![]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_read_a_nul_in_an_expansion_form_value() { + assert_eq!( + humanize_parts_without_errors("{a\0, select, x {b}}", true), + vec![ + (HtmlTokenType::ExpansionFormStart, vec![]), + (HtmlTokenType::RawText, vec!["a\0".to_string()]), + (HtmlTokenType::RawText, vec!["select".to_string()]), + (HtmlTokenType::ExpansionCaseValue, vec!["x".to_string()]), + (HtmlTokenType::ExpansionCaseExpStart, vec![]), + (HtmlTokenType::Text, vec!["b".to_string()]), + (HtmlTokenType::ExpansionCaseExpEnd, vec![]), + (HtmlTokenType::ExpansionFormEnd, vec![]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + + #[test] + fn should_read_the_character_after_a_comment_start_in_an_interpolation() { + // Angular reads one more character after `//` before it checks for `}}` or a NUL again. + assert_eq!( + humanize_parts_without_errors("{{ a //}}b", false), + vec![ + (HtmlTokenType::Text, vec![String::new()]), + (HtmlTokenType::Interpolation, vec!["{{".to_string(), " a //}}b".to_string()]), + (HtmlTokenType::Text, vec![String::new()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + assert_eq!( + humanize_parts_without_errors("{{ a //\0 }}b", false), + vec![ + (HtmlTokenType::Text, vec![String::new()]), + ( + HtmlTokenType::Interpolation, + vec!["{{".to_string(), " a //\0 ".to_string(), "}}".to_string()] + ), + (HtmlTokenType::Text, vec!["b".to_string()]), + (HtmlTokenType::Eof, vec![]), + ] + ); + } + #[test] fn should_handle_empty_input() { let result = tokenize("");