Shrink ParserState (#457)

* tokenizer: Shrink current_line_start_position.

We already truncate column numbers to u32, so this gives the same answer
it would otherwise give, and saves some storage (not in this patch
because ParserState is still 24 bytes, but we can improve on that in a
bit).

* parser: Shrink ParserState by collapsing line number and opening block type.

The line number range gets a bit smaller, but it pays off by making
state restoration faster, which is a lot hotter.

Author: emilio

src/parser.rs

@@ -13,15 +13,33 @@ use std::ops::BitOr;
 ///
 /// Can be used with the `Parser::reset` method to restore that state.
 /// Should only be used with the `Parser` instance it came from.
-#[derive(Debug, Clone, Default)]
+#[derive(Clone, Default)]
 pub struct ParserState {
     pub(crate) position: usize,
-    pub(crate) current_line_start_position: usize,
-    pub(crate) current_line_number: u32,
-    pub(crate) at_start_of: Option<BlockType>,
+    pub(crate) current_line_start_position: u32,
+    /// Current line number shifted by `Self::BLOCK_TYPE_BITS`, with the low 2 bits holding the
+    /// `BlockType` discriminant of the block the last returned token opened (if any).
+    line_number_and_block_type: u32,
+}
+
+impl fmt::Debug for ParserState {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("ParserState")
+            .field("position", &self.position)
+            .field(
+                "current_line_start_position",
+                &self.current_line_start_position,
+            )
+            .field("current_line_number", &self.line_number())
+            .field("at_start_of", &self.at_start_of())
+            .finish()
+    }
 }
 
 impl ParserState {
+    const BLOCK_TYPE_BITS: u32 = 2;
+    const BLOCK_TYPE_MASK: u32 = (1 << Self::BLOCK_TYPE_BITS) - 1;
+
     /// The position from the start of the input, counted in UTF-8 bytes.
     #[inline]
     pub fn position(&self) -> SourcePosition {
@@ -32,10 +50,42 @@ impl ParserState {
     #[inline]
     pub fn source_location(&self) -> SourceLocation {
         SourceLocation {
-            line: self.current_line_number,
-            column: (self.position - self.current_line_start_position + 1) as u32,
+            line: self.line_number(),
+            column: (self.position as u32)
+                .wrapping_sub(self.current_line_start_position)
+                .wrapping_add(1),
         }
     }
+
+    #[inline]
+    pub(crate) fn line_number(&self) -> u32 {
+        self.line_number_and_block_type >> Self::BLOCK_TYPE_BITS
+    }
+
+    #[inline]
+    pub(crate) fn advance_line_number(&mut self, count: u32) {
+        self.line_number_and_block_type += count << Self::BLOCK_TYPE_BITS;
+    }
+
+    #[inline]
+    pub(crate) fn at_start_of(&self) -> Option<BlockType> {
+        let block_type = (self.line_number_and_block_type & Self::BLOCK_TYPE_MASK) as u8;
+        BlockType::from_byte(block_type)
+    }
+
+    #[inline]
+    pub(crate) fn set_at_start_of(&mut self, block_type: BlockType) {
+        debug_assert!(self.at_start_of().is_none());
+        self.line_number_and_block_type |= block_type as u32;
+        debug_assert_eq!(self.at_start_of(), Some(block_type));
+    }
+
+    #[inline]
+    pub(crate) fn take_at_start_of(&mut self) -> Option<BlockType> {
+        let block_type = self.at_start_of();
+        self.line_number_and_block_type &= !Self::BLOCK_TYPE_MASK;
+        block_type
+    }
 }
 
 /// When parsing until a given token, sometimes the caller knows that parsing is going to restart
@@ -241,20 +291,35 @@ struct CachedToken<'i> {
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
+#[repr(u8)]
 pub(crate) enum BlockType {
-    Parenthesis,
+    // NOTE: We rely on the discriminants being 1, 2, 3 for the `ParserState`
+    // bit packing (and from_byte).
+    Parenthesis = 1,
     SquareBracket,
     CurlyBracket,
 }
 
 impl BlockType {
+    #[inline]
+    fn from_byte(b: u8) -> Option<BlockType> {
+        let ty = match b {
+            1 => BlockType::Parenthesis,
+            2 => BlockType::SquareBracket,
+            3 => BlockType::CurlyBracket,
+            _ => return None,
+        };
+        debug_assert_eq!(ty as u8, b);
+        Some(ty)
+    }
+
     fn closing(token: &Token) -> Option<BlockType> {
-        match *token {
-            Token::CloseParenthesis => Some(BlockType::Parenthesis),
-            Token::CloseSquareBracket => Some(BlockType::SquareBracket),
-            Token::CloseCurlyBracket => Some(BlockType::CurlyBracket),
-            _ => None,
-        }
+        Some(match *token {
+            Token::CloseParenthesis => BlockType::Parenthesis,
+            Token::CloseSquareBracket => BlockType::SquareBracket,
+            Token::CloseCurlyBracket => BlockType::CurlyBracket,
+            _ => return None,
+        })
     }
 }
 
@@ -939,7 +1004,7 @@ pub fn parse_nested_block<'i, F, T, E>(
 where
     F: FnOnce(&mut Parser<'i>) -> Result<T, ParseError<E>>,
 {
-    let block_type = parser.state.at_start_of.take().expect(
+    let block_type = parser.state.take_at_start_of().expect(
         "\
          A nested parser can only be created when a Function, \
          ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \
@@ -980,7 +1045,7 @@ impl Parser<'_> {
         // FIXME: have a special-purpose tokenizer method for this that does less work.
         while !self.is_eof() {
             let token = self.next_unchecked();
-            if let Some(nested_block_type) = self.state.at_start_of.take() {
+            if let Some(nested_block_type) = self.state.take_at_start_of() {
                 stack.push(nested_block_type);
                 continue;
             }

src/size_of_tests.rs

@@ -42,9 +42,9 @@ size_of_test!(token, Token, 32);
 size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32);
 size_of_test!(cow_rc_str, CowRcStr, 16);
 
-size_of_test!(parser, crate::parser::Parser, 168);
+size_of_test!(parser, crate::parser::Parser, 152);
 size_of_test!(source_position, crate::SourcePosition, 8);
-size_of_test!(parser_state, crate::ParserState, 24);
+size_of_test!(parser_state, crate::ParserState, 16);
 
 size_of_test!(basic_parse_error, crate::BasicParseError, 1);
 size_of_test!(parse_error_lower_bound, crate::ParseError<()>, 1);

src/tokenizer.rs

@@ -238,14 +238,14 @@ impl<'a> Parser<'a> {
     /// Assumes non-EOF.
     #[inline]
     pub(crate) fn next_unchecked(&mut self) -> Token<'a> {
-        debug_assert!(self.state.at_start_of.is_none());
+        debug_assert!(self.state.at_start_of().is_none());
         next_token_unchecked(self)
     }
 
     /// If the last token returned opened a block, skip until after the end of that block.
     #[inline]
     pub(crate) fn skip_block_at_start(&mut self) {
-        if let Some(block_type) = self.state.at_start_of.take() {
+        if let Some(block_type) = self.state.take_at_start_of() {
             self.consume_until_end_of_block(block_type);
         }
     }
@@ -456,8 +456,8 @@ impl<'a> Parser<'a> {
         if byte == b'\r' && self.next_byte() == Some(b'\n') {
             self.state.position += 1;
         }
-        self.state.current_line_start_position = self.state.position;
-        self.state.current_line_number += 1;
+        self.state.current_line_start_position = self.state.position as u32;
+        self.state.advance_line_number(1);
     }
 
     #[inline]
@@ -476,7 +476,7 @@ impl<'a> Parser<'a> {
         self.state.current_line_start_position = self
             .state
             .current_line_start_position
-            .wrapping_add(len_utf8 - c.len_utf16());
+            .wrapping_add((len_utf8 - c.len_utf16()) as u32);
         c
     }
 
@@ -602,7 +602,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
         b'\'' => consume_string(parser, true),
         b'(' => {
             parser.advance(1);
-            parser.state.at_start_of = Some(BlockType::Parenthesis);
+            parser.state.set_at_start_of(BlockType::Parenthesis);
             ParenthesisBlock
         },
         b')' => { parser.advance(1); CloseParenthesis },
@@ -683,7 +683,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
         b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser),
         b'[' => {
             parser.advance(1);
-            parser.state.at_start_of = Some(BlockType::SquareBracket);
+            parser.state.set_at_start_of(BlockType::SquareBracket);
             SquareBracketBlock
         },
         b'\\' => {
@@ -697,7 +697,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> {
         },
         b'{' => {
             parser.advance(1);
-            parser.state.at_start_of = Some(BlockType::CurlyBracket);
+            parser.state.set_at_start_of(BlockType::CurlyBracket);
             CurlyBracketBlock
         },
         b'|' => {
@@ -945,7 +945,7 @@ fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> {
                 return url;
             }
         }
-        parser.state.at_start_of = Some(BlockType::Parenthesis);
+        parser.state.set_at_start_of(BlockType::Parenthesis);
         parser.arbitrary_substitution_functions.see_function(&value);
         Function(value)
     } else {
@@ -1215,10 +1215,10 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result<Token<'a>, ()> {
     }
 
     if newlines > 0 {
-        parser.state.current_line_number += newlines;
+        parser.state.advance_line_number(newlines);
         // No need for wrapping_add here, because there's no possible
         // way to wrap.
-        parser.state.current_line_start_position = start_position + last_newline + 1;
+        parser.state.current_line_start_position = (start_position + last_newline + 1) as u32;
     }
 
     if found_printable_char {