From 77506ffd0aed11f379c4cc3574092c25b19c818f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Sun, 20 Sep 2026 17:59:15 +0200 Subject: [PATCH 1/2] tokenizer: Shrink current_line_start_position. We already truncate column numbers to u32, so this gives the same answer it would otherwise give, and saves some storage (not in this patch because ParserState is still 24 bytes, but we can improve on that in a bit). --- src/parser.rs | 6 ++++-- src/tokenizer.rs | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 2ca1b4dc..72eda4f5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -16,7 +16,7 @@ use std::ops::BitOr; #[derive(Debug, Clone, Default)] pub struct ParserState { pub(crate) position: usize, - pub(crate) current_line_start_position: usize, + pub(crate) current_line_start_position: u32, pub(crate) current_line_number: u32, pub(crate) at_start_of: Option, } @@ -33,7 +33,9 @@ impl ParserState { pub fn source_location(&self) -> SourceLocation { SourceLocation { line: self.current_line_number, - column: (self.position - self.current_line_start_position + 1) as u32, + column: (self.position as u32) + .wrapping_sub(self.current_line_start_position) + .wrapping_add(1), } } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7b3ef09f..2bb19d47 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -456,7 +456,7 @@ impl<'a> Parser<'a> { if byte == b'\r' && self.next_byte() == Some(b'\n') { self.state.position += 1; } - self.state.current_line_start_position = self.state.position; + self.state.current_line_start_position = self.state.position as u32; self.state.current_line_number += 1; } @@ -476,7 +476,7 @@ impl<'a> Parser<'a> { self.state.current_line_start_position = self .state .current_line_start_position - .wrapping_add(len_utf8 - c.len_utf16()); + .wrapping_add((len_utf8 - c.len_utf16()) as u32); c } @@ -1218,7 +1218,7 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result, ()> { parser.state.current_line_number += newlines; // No need for wrapping_add here, because there's no possible // way to wrap. - parser.state.current_line_start_position = start_position + last_newline + 1; + parser.state.current_line_start_position = (start_position + last_newline + 1) as u32; } if found_printable_char { From 29b598740bf69fe24ca76e73591b631268a1bc1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Sun, 20 Sep 2026 18:06:37 +0200 Subject: [PATCH 2/2] parser: Shrink ParserState by collapsing line number and opening block type. The line number range gets a bit smaller, but it pays off by making state restoration faster, which is a lot hotter. --- src/parser.rs | 89 +++++++++++++++++++++++++++++++++++++------- src/size_of_tests.rs | 4 +- src/tokenizer.rs | 16 ++++---- 3 files changed, 86 insertions(+), 23 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 72eda4f5..264cc980 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -13,15 +13,33 @@ use std::ops::BitOr; /// /// Can be used with the `Parser::reset` method to restore that state. /// Should only be used with the `Parser` instance it came from. -#[derive(Debug, Clone, Default)] +#[derive(Clone, Default)] pub struct ParserState { pub(crate) position: usize, pub(crate) current_line_start_position: u32, - pub(crate) current_line_number: u32, - pub(crate) at_start_of: Option, + /// Current line number shifted by `Self::BLOCK_TYPE_BITS`, with the low 2 bits holding the + /// `BlockType` discriminant of the block the last returned token opened (if any). + line_number_and_block_type: u32, +} + +impl fmt::Debug for ParserState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParserState") + .field("position", &self.position) + .field( + "current_line_start_position", + &self.current_line_start_position, + ) + .field("current_line_number", &self.line_number()) + .field("at_start_of", &self.at_start_of()) + .finish() + } } impl ParserState { + const BLOCK_TYPE_BITS: u32 = 2; + const BLOCK_TYPE_MASK: u32 = (1 << Self::BLOCK_TYPE_BITS) - 1; + /// The position from the start of the input, counted in UTF-8 bytes. #[inline] pub fn position(&self) -> SourcePosition { @@ -32,12 +50,42 @@ impl ParserState { #[inline] pub fn source_location(&self) -> SourceLocation { SourceLocation { - line: self.current_line_number, + line: self.line_number(), column: (self.position as u32) .wrapping_sub(self.current_line_start_position) .wrapping_add(1), } } + + #[inline] + pub(crate) fn line_number(&self) -> u32 { + self.line_number_and_block_type >> Self::BLOCK_TYPE_BITS + } + + #[inline] + pub(crate) fn advance_line_number(&mut self, count: u32) { + self.line_number_and_block_type += count << Self::BLOCK_TYPE_BITS; + } + + #[inline] + pub(crate) fn at_start_of(&self) -> Option { + let block_type = (self.line_number_and_block_type & Self::BLOCK_TYPE_MASK) as u8; + BlockType::from_byte(block_type) + } + + #[inline] + pub(crate) fn set_at_start_of(&mut self, block_type: BlockType) { + debug_assert!(self.at_start_of().is_none()); + self.line_number_and_block_type |= block_type as u32; + debug_assert_eq!(self.at_start_of(), Some(block_type)); + } + + #[inline] + pub(crate) fn take_at_start_of(&mut self) -> Option { + let block_type = self.at_start_of(); + self.line_number_and_block_type &= !Self::BLOCK_TYPE_MASK; + block_type + } } /// When parsing until a given token, sometimes the caller knows that parsing is going to restart @@ -243,20 +291,35 @@ struct CachedToken<'i> { } #[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u8)] pub(crate) enum BlockType { - Parenthesis, + // NOTE: We rely on the discriminants being 1, 2, 3 for the `ParserState` + // bit packing (and from_byte). + Parenthesis = 1, SquareBracket, CurlyBracket, } impl BlockType { + #[inline] + fn from_byte(b: u8) -> Option { + let ty = match b { + 1 => BlockType::Parenthesis, + 2 => BlockType::SquareBracket, + 3 => BlockType::CurlyBracket, + _ => return None, + }; + debug_assert_eq!(ty as u8, b); + Some(ty) + } + fn closing(token: &Token) -> Option { - match *token { - Token::CloseParenthesis => Some(BlockType::Parenthesis), - Token::CloseSquareBracket => Some(BlockType::SquareBracket), - Token::CloseCurlyBracket => Some(BlockType::CurlyBracket), - _ => None, - } + Some(match *token { + Token::CloseParenthesis => BlockType::Parenthesis, + Token::CloseSquareBracket => BlockType::SquareBracket, + Token::CloseCurlyBracket => BlockType::CurlyBracket, + _ => return None, + }) } } @@ -941,7 +1004,7 @@ pub fn parse_nested_block<'i, F, T, E>( where F: FnOnce(&mut Parser<'i>) -> Result>, { - let block_type = parser.state.at_start_of.take().expect( + let block_type = parser.state.take_at_start_of().expect( "\ A nested parser can only be created when a Function, \ ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \ @@ -982,7 +1045,7 @@ impl Parser<'_> { // FIXME: have a special-purpose tokenizer method for this that does less work. while !self.is_eof() { let token = self.next_unchecked(); - if let Some(nested_block_type) = self.state.at_start_of.take() { + if let Some(nested_block_type) = self.state.take_at_start_of() { stack.push(nested_block_type); continue; } diff --git a/src/size_of_tests.rs b/src/size_of_tests.rs index 9d58dee1..f7567767 100644 --- a/src/size_of_tests.rs +++ b/src/size_of_tests.rs @@ -42,9 +42,9 @@ size_of_test!(token, Token, 32); size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32); size_of_test!(cow_rc_str, CowRcStr, 16); -size_of_test!(parser, crate::parser::Parser, 168); +size_of_test!(parser, crate::parser::Parser, 152); size_of_test!(source_position, crate::SourcePosition, 8); -size_of_test!(parser_state, crate::ParserState, 24); +size_of_test!(parser_state, crate::ParserState, 16); size_of_test!(basic_parse_error, crate::BasicParseError, 1); size_of_test!(parse_error_lower_bound, crate::ParseError<()>, 1); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2bb19d47..64a7798f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -238,14 +238,14 @@ impl<'a> Parser<'a> { /// Assumes non-EOF. #[inline] pub(crate) fn next_unchecked(&mut self) -> Token<'a> { - debug_assert!(self.state.at_start_of.is_none()); + debug_assert!(self.state.at_start_of().is_none()); next_token_unchecked(self) } /// If the last token returned opened a block, skip until after the end of that block. #[inline] pub(crate) fn skip_block_at_start(&mut self) { - if let Some(block_type) = self.state.at_start_of.take() { + if let Some(block_type) = self.state.take_at_start_of() { self.consume_until_end_of_block(block_type); } } @@ -457,7 +457,7 @@ impl<'a> Parser<'a> { self.state.position += 1; } self.state.current_line_start_position = self.state.position as u32; - self.state.current_line_number += 1; + self.state.advance_line_number(1); } #[inline] @@ -602,7 +602,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'\'' => consume_string(parser, true), b'(' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); ParenthesisBlock }, b')' => { parser.advance(1); CloseParenthesis }, @@ -683,7 +683,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser), b'[' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::SquareBracket); + parser.state.set_at_start_of(BlockType::SquareBracket); SquareBracketBlock }, b'\\' => { @@ -697,7 +697,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { }, b'{' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::CurlyBracket); + parser.state.set_at_start_of(BlockType::CurlyBracket); CurlyBracketBlock }, b'|' => { @@ -945,7 +945,7 @@ fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> { return url; } } - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); parser.arbitrary_substitution_functions.see_function(&value); Function(value) } else { @@ -1215,7 +1215,7 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result, ()> { } if newlines > 0 { - parser.state.current_line_number += newlines; + parser.state.advance_line_number(newlines); // No need for wrapping_add here, because there's no possible // way to wrap. parser.state.current_line_start_position = (start_position + last_newline + 1) as u32;