Auto merge of #94693 - nnethercote:parser-inlining, r=petrochenkov

Inline some parser functions

Some crates that do a lot of complex declarative macro expansion spend a lot of time parsing (and reparsing) tokens. These commits inline some functions for some minor speed wins.

r? `@ghost`

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -100,11 +100,12 @@ struct LazyTokenStreamImpl {
 
 impl CreateTokenStream for LazyTokenStreamImpl {
     fn create_token_stream(&self) -> AttrAnnotatedTokenStream {
-        // The token produced by the final call to `next` or `next_desugared`
-        // was not actually consumed by the callback. The combination
-        // of chaining the initial token and using `take` produces the desired
-        // result - we produce an empty `TokenStream` if no calls were made,
-        // and omit the final token otherwise.
+        // The token produced by the final call to `{,inlined_}next` or
+        // `{,inlined_}next_desugared` was not actually consumed by the
+        // callback. The combination of chaining the initial token and using
+        // `take` produces the desired result - we produce an empty
+        // `TokenStream` if no calls were made, and omit the final token
+        // otherwise.
         let mut cursor_snapshot = self.cursor_snapshot.clone();
         let tokens =
             std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1))

compiler/rustc_parse/src/parser/mod.rs

@@ -206,8 +206,9 @@ struct TokenCursor {
     frame: TokenCursorFrame,
     stack: Vec<TokenCursorFrame>,
     desugar_doc_comments: bool,
-    // Counts the number of calls to `next` or `next_desugared`,
-    // depending on whether `desugar_doc_comments` is set.
+    // Counts the number of calls to `{,inlined_}next` or
+    // `{,inlined_}next_desugared`, depending on whether
+    // `desugar_doc_comments` is set.
     num_next_calls: usize,
     // During parsing, we may sometimes need to 'unglue' a
     // glued token into two component tokens
@@ -256,6 +257,12 @@ fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self {
 
 impl TokenCursor {
     fn next(&mut self) -> (Token, Spacing) {
+        self.inlined_next()
+    }
+
+    /// This always-inlined version should only be used on hot code paths.
+    #[inline(always)]
+    fn inlined_next(&mut self) -> (Token, Spacing) {
         loop {
             let (tree, spacing) = if !self.frame.open_delim {
                 self.frame.open_delim = true;
@@ -285,7 +292,13 @@ fn next(&mut self) -> (Token, Spacing) {
     }
 
     fn next_desugared(&mut self) -> (Token, Spacing) {
-        let (data, attr_style, sp) = match self.next() {
+        self.inlined_next_desugared()
+    }
+
+    /// This always-inlined version should only be used on hot code paths.
+    #[inline(always)]
+    fn inlined_next_desugared(&mut self) -> (Token, Spacing) {
+        let (data, attr_style, sp) = match self.inlined_next() {
             (Token { kind: token::DocComment(_, attr_style, data), span }, _) => {
                 (data, attr_style, span)
             }
@@ -463,12 +476,13 @@ pub fn new(
         parser
     }
 
+    #[inline]
     fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) {
         loop {
             let (mut next, spacing) = if self.desugar_doc_comments {
-                self.token_cursor.next_desugared()
+                self.token_cursor.inlined_next_desugared()
             } else {
-                self.token_cursor.next()
+                self.token_cursor.inlined_next()
             };
             self.token_cursor.num_next_calls += 1;
             // We've retrieved an token from the underlying
@@ -998,7 +1012,13 @@ fn parse_paren_comma_seq<T>(
     }
 
     /// Advance the parser by one token using provided token as the next one.
-    fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) {
+    fn bump_with(&mut self, next: (Token, Spacing)) {
+        self.inlined_bump_with(next)
+    }
+
+    /// This always-inlined version should only be used on hot code paths.
+    #[inline(always)]
+    fn inlined_bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) {
         // Bumping after EOF is a bad sign, usually an infinite loop.
         if self.prev_token.kind == TokenKind::Eof {
             let msg = "attempted to bump the parser past EOF (may be stuck in a loop)";
@@ -1016,7 +1036,7 @@ fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) {
     /// Advance the parser by one token.
     pub fn bump(&mut self) {
         let next_token = self.next_tok(self.token.span);
-        self.bump_with(next_token);
+        self.inlined_bump_with(next_token);
     }
 
     /// Look-ahead `dist` tokens of `self.token` and get access to that token there.