From 4e700a023c2706136012c39cc30d8bf431d83f0a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 7 Mar 2022 15:17:38 +1100 Subject: [PATCH 1/2] Split `Parser::bump_with` into inlined and non-inlined halves. The call site within `Parser::bump` is hot. Also add an inline annotation to `Parser::next_tok`. It was already being inlined by the compiler; this just makes sure that continues. --- compiler/rustc_parse/src/parser/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 4e229918b63..5c773410891 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -463,6 +463,7 @@ pub fn new( parser } + #[inline] fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) { loop { let (mut next, spacing) = if self.desugar_doc_comments { @@ -998,7 +999,13 @@ fn parse_paren_comma_seq( } /// Advance the parser by one token using provided token as the next one. - fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { + fn bump_with(&mut self, next: (Token, Spacing)) { + self.inlined_bump_with(next) + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { // Bumping after EOF is a bad sign, usually an infinite loop. if self.prev_token.kind == TokenKind::Eof { let msg = "attempted to bump the parser past EOF (may be stuck in a loop)"; @@ -1016,7 +1023,7 @@ fn bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) { /// Advance the parser by one token. pub fn bump(&mut self) { let next_token = self.next_tok(self.token.span); - self.bump_with(next_token); + self.inlined_bump_with(next_token); } /// Look-ahead `dist` tokens of `self.token` and get access to that token there. From f8f1d3f00b9780e3053b15105f8ed0bfe57a5e9b Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 7 Mar 2022 15:55:39 +1100 Subject: [PATCH 2/2] Split `TokenCursor::{next,next_desugared}` into inlined and non-inlined halves. --- .../rustc_parse/src/parser/attr_wrapper.rs | 11 +++++---- compiler/rustc_parse/src/parser/mod.rs | 23 +++++++++++++++---- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 568682cc3e4..f0ec86ca64a 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -100,11 +100,12 @@ struct LazyTokenStreamImpl { impl CreateTokenStream for LazyTokenStreamImpl { fn create_token_stream(&self) -> AttrAnnotatedTokenStream { - // The token produced by the final call to `next` or `next_desugared` - // was not actually consumed by the callback. The combination - // of chaining the initial token and using `take` produces the desired - // result - we produce an empty `TokenStream` if no calls were made, - // and omit the final token otherwise. + // The token produced by the final call to `{,inlined_}next` or + // `{,inlined_}next_desugared` was not actually consumed by the + // callback. The combination of chaining the initial token and using + // `take` produces the desired result - we produce an empty + // `TokenStream` if no calls were made, and omit the final token + // otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); let tokens = std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 5c773410891..3a2f193d319 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -206,8 +206,9 @@ struct TokenCursor { frame: TokenCursorFrame, stack: Vec, desugar_doc_comments: bool, - // Counts the number of calls to `next` or `next_desugared`, - // depending on whether `desugar_doc_comments` is set. + // Counts the number of calls to `{,inlined_}next` or + // `{,inlined_}next_desugared`, depending on whether + // `desugar_doc_comments` is set. num_next_calls: usize, // During parsing, we may sometimes need to 'unglue' a // glued token into two component tokens @@ -256,6 +257,12 @@ fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { impl TokenCursor { fn next(&mut self) -> (Token, Spacing) { + self.inlined_next() + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_next(&mut self) -> (Token, Spacing) { loop { let (tree, spacing) = if !self.frame.open_delim { self.frame.open_delim = true; @@ -285,7 +292,13 @@ fn next(&mut self) -> (Token, Spacing) { } fn next_desugared(&mut self) -> (Token, Spacing) { - let (data, attr_style, sp) = match self.next() { + self.inlined_next_desugared() + } + + /// This always-inlined version should only be used on hot code paths. + #[inline(always)] + fn inlined_next_desugared(&mut self) -> (Token, Spacing) { + let (data, attr_style, sp) = match self.inlined_next() { (Token { kind: token::DocComment(_, attr_style, data), span }, _) => { (data, attr_style, span) } @@ -467,9 +480,9 @@ pub fn new( fn next_tok(&mut self, fallback_span: Span) -> (Token, Spacing) { loop { let (mut next, spacing) = if self.desugar_doc_comments { - self.token_cursor.next_desugared() + self.token_cursor.inlined_next_desugared() } else { - self.token_cursor.next() + self.token_cursor.inlined_next() }; self.token_cursor.num_next_calls += 1; // We've retrieved an token from the underlying