diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 568682cc3e4..f0ec86ca64a 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -100,11 +100,12 @@ struct LazyTokenStreamImpl { impl CreateTokenStream for LazyTokenStreamImpl { fn create_token_stream(&self) -> AttrAnnotatedTokenStream { - // The token produced by the final call to `next` or `next_desugared` - // was not actually consumed by the callback. The combination - // of chaining the initial token and using `take` produces the desired - // result - we produce an empty `TokenStream` if no calls were made, - // and omit the final token otherwise. + // The token produced by the final call to `{,inlined_}next` or + // `{,inlined_}next_desugared` was not actually consumed by the + // callback. The combination of chaining the initial token and using + // `take` produces the desired result - we produce an empty + // `TokenStream` if no calls were made, and omit the final token + // otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); let tokens = std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 4e229918b63..3a2f193d319 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -206,8 +206,9 @@ struct TokenCursor { frame: TokenCursorFrame, stack: Vec, desugar_doc_comments: bool, - // Counts the number of calls to `next` or `next_desugared`, - // depending on whether `desugar_doc_comments` is set. + // Counts the number of calls to `{,inlined_}next` or + // `{,inlined_}next_desugared`, depending on whether + // `desugar_doc_comments` is set. num_next_calls: usize, // During parsing, we may sometimes need to 'unglue' a // glued token into two component tokens @@ -256,6 +257,12 @@ fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { impl TokenCursor { fn next(&mut self) -> (Token, Spacing) { + self.inlined_next() + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_next(&mut self) -> (Token, Spacing) { loop { let (tree, spacing) = if !self.frame.open_delim { self.frame.open_delim = true; @@ -285,7 +292,13 @@ fn next(&mut self) -> (Token, Spacing) { } fn next_desugared(&mut self) -> (Token, Spacing) { - let (data, attr_style, sp) = match self.next() { + self.inlined_next_desugared() + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_next_desugared(&mut self) -> (Token, Spacing) { + let (data, attr_style, sp) = match self.inlined_next() { (Token { kind: token::DocComment(_, attr_style, data), span }, _) => { (data, attr_style, span) } @@ -463,12 +476,13 @@ pub fn new( parser } + #[inline] fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) { loop { let (mut next, spacing) = if self.desugar_doc_comments { - self.token_cursor.next_desugared() + self.token_cursor.inlined_next_desugared() } else { - self.token_cursor.next() + self.token_cursor.inlined_next() }; self.token_cursor.num_next_calls += 1; // We've retrieved an token from the underlying @@ -998,7 +1012,13 @@ fn parse_paren_comma_seq( } /// Advance the parser by one token using provided token as the next one. - fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { + fn bump_with(&mut self, next: (Token, Spacing)) { + self.inlined_bump_with(next) + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { // Bumping after EOF is a bad sign, usually an infinite loop. if self.prev_token.kind == TokenKind::Eof { let msg = "attempted to bump the parser past EOF (may be stuck in a loop)"; @@ -1016,7 +1036,7 @@ fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { /// Advance the parser by one token. pub fn bump(&mut self) { let next_token = self.next_tok(self.token.span); - self.bump_with(next_token); + self.inlined_bump_with(next_token); } /// Look-ahead `dist` tokens of `self.token` and get access to that token there.