From 1680b791d57013ca2cf5839a279e3d4d826c336c Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:33:46 +1000 Subject: [PATCH 01/12] Simplify `CfgEval`. It can contain an owned value instead of a reference. --- compiler/rustc_builtin_macros/src/cfg_eval.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index 03aff6f9633..884cebf1939 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -38,16 +38,14 @@ pub(crate) fn cfg_eval( lint_node_id: NodeId, ) -> Annotatable { let features = Some(features); - CfgEval { cfg: &mut StripUnconfigured { sess, features, config_tokens: true, lint_node_id } } + CfgEval(StripUnconfigured { sess, features, config_tokens: true, lint_node_id }) .configure_annotatable(annotatable) // Since the item itself has already been configured by the `InvocationCollector`, // we know that fold result vector will contain exactly one element. .unwrap() } -struct CfgEval<'a, 'b> { - cfg: &'a mut StripUnconfigured<'b>, -} +struct CfgEval<'a>(StripUnconfigured<'a>); fn flat_map_annotatable( vis: &mut impl MutVisitor, @@ -125,9 +123,9 @@ fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool { res.is_break() } -impl CfgEval<'_, '_> { +impl CfgEval<'_> { fn configure(&mut self, node: T) -> Option { - self.cfg.configure(node) + self.0.configure(node) } fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option { @@ -196,7 +194,7 @@ impl CfgEval<'_, '_> { // Re-parse the tokens, setting the `capture_cfg` flag to save extra information // to the captured `AttrTokenStream` (specifically, we capture // `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`) - let mut parser = Parser::new(&self.cfg.sess.psess, orig_tokens, None); + let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None); parser.capture_cfg = true; match parse_annotatable_with(&mut parser) { Ok(a) => annotatable = a, @@ -212,16 +210,16 @@ impl CfgEval<'_, '_> { } } -impl MutVisitor for CfgEval<'_, '_> { +impl MutVisitor for CfgEval<'_> { #[instrument(level = "trace", skip(self))] fn visit_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr, false); + self.0.configure_expr(expr, false); mut_visit::noop_visit_expr(expr, self); } #[instrument(level = "trace", skip(self))] fn visit_method_receiver_expr(&mut self, expr: &mut P) { - self.cfg.configure_expr(expr, true); + self.0.configure_expr(expr, true); mut_visit::noop_visit_expr(expr, self); } From d6c0b8117e4ccccd83c4a6e70eee8b12c51d1a18 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:34:05 +1000 Subject: [PATCH 02/12] Fix a typo in a comment. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index b5480b6b7d2..4b1531c5ff3 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -340,7 +340,7 @@ impl<'a> Parser<'a> { // If we support tokens at all if let Some(target_tokens) = ret.tokens_mut() { if target_tokens.is_none() { - // Store se our newly captured tokens into the AST node + // Store our newly captured tokens into the AST node. *target_tokens = Some(tokens.clone()); } } From f852568fa601171f20f924a50478c33fd2661fba Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:42:46 +1000 Subject: [PATCH 03/12] Change `AttrTokenStream::to_tokenstream` to `to_token_trees`. I.e. change the return type from `TokenStream` to `Vec`. Most of the callsites require a `TokenStream`, but the recursive call used to create `target_tokens` requires a `Vec`. It's easy to convert a `Vec` to a `TokenStream` (just call `TokenStream::new`) but it's harder to convert a `TokenStream` to a `Vec` (either iterate/clone/collect, or use `Lrc::into_inner` if appropriate). So this commit changes the return value to simplify that `target_tokens` call site. --- compiler/rustc_ast/src/attr/mod.rs | 14 ++++++++------ compiler/rustc_ast/src/tokenstream.rs | 23 +++++++---------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 593c78df3cd..65f1b5dbaf5 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -204,12 +204,14 @@ impl Attribute { pub fn tokens(&self) -> TokenStream { match &self.kind { - AttrKind::Normal(normal) => normal - .tokens - .as_ref() - .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) - .to_attr_token_stream() - .to_tokenstream(), + AttrKind::Normal(normal) => TokenStream::new( + normal + .tokens + .as_ref() + .unwrap_or_else(|| panic!("attribute is missing tokens: {self:?}")) + .to_attr_token_stream() + .to_token_trees(), + ), &AttrKind::DocComment(comment_kind, data) => TokenStream::token_alone( token::DocComment(comment_kind, self.style, data), self.span, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index b4ddbe20689..5b2d673316a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -180,14 +180,13 @@ impl AttrTokenStream { AttrTokenStream(Lrc::new(tokens)) } - /// Converts this `AttrTokenStream` to a plain `TokenStream`. + /// Converts this `AttrTokenStream` to a plain `Vec`. /// During conversion, `AttrTokenTree::Attributes` get 'flattened' /// back to a `TokenStream` of the form `outer_attr attr_target`. /// If there are inner attributes, they are inserted into the proper /// place in the attribute target tokens. - pub fn to_tokenstream(&self) -> TokenStream { - let trees: Vec<_> = self - .0 + pub fn to_token_trees(&self) -> Vec { + self.0 .iter() .flat_map(|tree| match &tree { AttrTokenTree::Token(inner, spacing) => { @@ -198,7 +197,7 @@ impl AttrTokenStream { *span, *spacing, *delim, - stream.to_tokenstream() + TokenStream::new(stream.to_token_trees()) ),] .into_iter() } @@ -208,14 +207,7 @@ impl AttrTokenStream { .partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer)); let (outer_attrs, inner_attrs) = data.attrs.split_at(idx); - let mut target_tokens: Vec<_> = data - .tokens - .to_attr_token_stream() - .to_tokenstream() - .0 - .iter() - .cloned() - .collect(); + let mut target_tokens = data.tokens.to_attr_token_stream().to_token_trees(); if !inner_attrs.is_empty() { let mut found = false; // Check the last two trees (to account for a trailing semi) @@ -260,8 +252,7 @@ impl AttrTokenStream { flat.into_iter() } }) - .collect(); - TokenStream::new(trees) + .collect() } } @@ -461,7 +452,7 @@ impl TokenStream { AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)]) }; - attr_stream.to_tokenstream() + TokenStream::new(attr_stream.to_token_trees()) } pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { From 0cfd2473be4a5555018cfbca089a7ed1bfd16c53 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 10:49:12 +1000 Subject: [PATCH 04/12] Rename `TokenStream::new` argument. `tts` is a better name than `streams` for a `Vec`. --- compiler/rustc_ast/src/tokenstream.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 5b2d673316a..cabf18fd153 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -400,8 +400,8 @@ impl PartialEq for TokenStream { } impl TokenStream { - pub fn new(streams: Vec) -> TokenStream { - TokenStream(Lrc::new(streams)) + pub fn new(tts: Vec) -> TokenStream { + TokenStream(Lrc::new(tts)) } pub fn is_empty(&self) -> bool { From 7416c20cfd781de5608494d67975be94e8689c24 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 27 Jun 2024 11:15:38 +1000 Subject: [PATCH 05/12] Just `push` in `AttrTokenStream::to_token_trees`. Currently it uses a mixture of functional style (`flat_map`) and imperative style (`push`), which is a bit hard to read. This commit converts it to fully imperative, which is more concise and avoids the need for `smallvec`. --- compiler/rustc_ast/src/tokenstream.rs | 28 ++++++++++++--------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index cabf18fd153..655c18e4559 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -23,7 +23,6 @@ use rustc_data_structures::sync::{self, Lrc}; use rustc_macros::{Decodable, Encodable, HashStable_Generic}; use rustc_serialize::{Decodable, Encodable}; use rustc_span::{sym, Span, SpanDecoder, SpanEncoder, Symbol, DUMMY_SP}; -use smallvec::{smallvec, SmallVec}; use std::borrow::Cow; use std::{cmp, fmt, iter}; @@ -186,20 +185,19 @@ impl AttrTokenStream { /// If there are inner attributes, they are inserted into the proper /// place in the attribute target tokens. pub fn to_token_trees(&self) -> Vec { - self.0 - .iter() - .flat_map(|tree| match &tree { + let mut res = Vec::with_capacity(self.0.len()); + for tree in self.0.iter() { + match tree { AttrTokenTree::Token(inner, spacing) => { - smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() + res.push(TokenTree::Token(inner.clone(), *spacing)); } AttrTokenTree::Delimited(span, spacing, delim, stream) => { - smallvec![TokenTree::Delimited( + res.push(TokenTree::Delimited( *span, *spacing, *delim, - TokenStream::new(stream.to_token_trees()) - ),] - .into_iter() + TokenStream::new(stream.to_token_trees()), + )) } AttrTokenTree::Attributes(data) => { let idx = data @@ -243,16 +241,14 @@ impl AttrTokenStream { "Failed to find trailing delimited group in: {target_tokens:?}" ); } - let mut flat: SmallVec<[_; 1]> = - SmallVec::with_capacity(target_tokens.len() + outer_attrs.len()); for attr in outer_attrs { - flat.extend(attr.tokens().0.iter().cloned()); + res.extend(attr.tokens().0.iter().cloned()); } - flat.extend(target_tokens); - flat.into_iter() + res.extend(target_tokens); } - }) - .collect() + } + } + res } } From 36c30a968b60a2926a0f600feb78eaba618d8d9a Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 14:32:47 +1000 Subject: [PATCH 06/12] Fix comment. Both the indenting, and the missing `)`. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 4b1531c5ff3..a5aaa9f0522 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -233,9 +233,9 @@ impl<'a> Parser<'a> { // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`) // so there's nothing for us to do. // 2. Our target already has tokens set (e.g. we've parsed something - // like `#[my_attr] $item`. The actual parsing code takes care of prepending - // any attributes to the nonterminal, so we don't need to modify the - // already captured tokens. + // like `#[my_attr] $item`). The actual parsing code takes care of + // prepending any attributes to the nonterminal, so we don't need to + // modify the already captured tokens. // Note that this check is independent of `force_collect`- if we already // have tokens, or can't even store them, then there's never a need to // force collection of new tokens. From 2342770f49dcc7e98069efb2db3620d4803c7698 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:50:06 +1000 Subject: [PATCH 07/12] Flip an if/else in `AttrTokenStream::to_attr_token_stream`. To put the simple case first. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index a5aaa9f0522..244a4f26fba 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -112,7 +112,9 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { })) .take(self.num_calls); - if !self.replace_ranges.is_empty() { + if self.replace_ranges.is_empty() { + make_token_stream(tokens, self.break_last_token) + } else { let mut tokens: Vec<_> = tokens.collect(); let mut replace_ranges = self.replace_ranges.to_vec(); replace_ranges.sort_by_key(|(range, _)| range.start); @@ -165,8 +167,6 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { ); } make_token_stream(tokens.into_iter(), self.break_last_token) - } else { - make_token_stream(tokens, self.break_last_token) } } } From 8b5a7eb7f4840dcc27e1dcf4a0e102a417ad988f Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:12:22 +1000 Subject: [PATCH 08/12] Move things around in `collect_tokens_trailing_token`. So that the `capturing` state is adjusted immediately before and after the call to `f`. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 244a4f26fba..7c22cac350b 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -218,16 +218,16 @@ impl<'a> Parser<'a> { let start_token = (self.token.clone(), self.token_spacing); let cursor_snapshot = self.token_cursor.clone(); let start_pos = self.num_bump_calls; - let has_outer_attrs = !attrs.attrs.is_empty(); - let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); let replace_ranges_start = self.capture_state.replace_ranges.len(); - let ret = f(self, attrs.attrs); - - self.capture_state.capturing = prev_capturing; - - let (mut ret, trailing) = ret?; + let (mut ret, trailing) = { + let prev_capturing = + std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); + let ret_and_trailing = f(self, attrs.attrs); + self.capture_state.capturing = prev_capturing; + ret_and_trailing? + }; // When we're not in `capture-cfg` mode, then bail out early if: // 1. Our target doesn't support tokens at all (e.g we're parsing an `NtIdent`) From f5b28968db07c96d1e2fe239d380fe5a418e85c5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 1 Jul 2024 16:21:47 +1000 Subject: [PATCH 09/12] Move more things around in `collect_tokens_trailing_token`. To make things a little clearer, and to avoid some `mut` variables. --- .../rustc_parse/src/parser/attr_wrapper.rs | 41 ++++++++----------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 7c22cac350b..10feccd21e2 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -276,37 +276,32 @@ impl<'a> Parser<'a> { let replace_ranges_end = self.capture_state.replace_ranges.len(); - let mut end_pos = self.num_bump_calls; - - let mut captured_trailing = false; - // Capture a trailing token if requested by the callback 'f' - match trailing { - TrailingToken::None => {} + let captured_trailing = match trailing { + TrailingToken::None => false, TrailingToken::Gt => { assert_eq!(self.token.kind, token::Gt); + false } TrailingToken::Semi => { assert_eq!(self.token.kind, token::Semi); - end_pos += 1; - captured_trailing = true; + true } - TrailingToken::MaybeComma => { - if self.token.kind == token::Comma { - end_pos += 1; - captured_trailing = true; - } - } - } + TrailingToken::MaybeComma => self.token.kind == token::Comma, + }; - // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), - // then extend the range of captured tokens to include it, since the parser - // was not actually bumped past it. When the `LazyAttrTokenStream` gets converted - // into an `AttrTokenStream`, we will create the proper token. - if self.break_last_token { - assert!(!captured_trailing, "Cannot set break_last_token and have trailing token"); - end_pos += 1; - } + assert!( + !(self.break_last_token && captured_trailing), + "Cannot set break_last_token and have trailing token" + ); + + let end_pos = self.num_bump_calls + + captured_trailing as usize + // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then + // extend the range of captured tokens to include it, since the parser was not actually + // bumped past it. When the `LazyAttrTokenStream` gets converted into an + // `AttrTokenStream`, we will create the proper token. + + self.break_last_token as usize; let num_calls = end_pos - start_pos; From 3d750e270279d2d8689aee3ee5cdbf60faec42bd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 16:31:24 +1000 Subject: [PATCH 10/12] Shrink parser positions from `usize` to `u32`. The number of source code bytes can't exceed a `u32`'s range, so a token position also can't. This reduces the size of `Parser` and `LazyAttrTokenStreamImpl` by eight bytes each. --- compiler/rustc_expand/src/mbe/diagnostics.rs | 12 +++++------ compiler/rustc_expand/src/mbe/macro_parser.rs | 2 +- compiler/rustc_expand/src/mbe/macro_rules.rs | 4 ++-- .../rustc_parse/src/parser/attr_wrapper.rs | 21 ++++++++----------- compiler/rustc_parse/src/parser/mod.rs | 6 +++--- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs index bf475c1dc96..2df8b8f00f8 100644 --- a/compiler/rustc_expand/src/mbe/diagnostics.rs +++ b/compiler/rustc_expand/src/mbe/diagnostics.rs @@ -120,21 +120,21 @@ struct CollectTrackerAndEmitter<'a, 'cx, 'matcher> { struct BestFailure { token: Token, - position_in_tokenstream: usize, + position_in_tokenstream: u32, msg: &'static str, remaining_matcher: MatcherLoc, } impl BestFailure { - fn is_better_position(&self, position: usize) -> bool { + fn is_better_position(&self, position: u32) -> bool { position > self.position_in_tokenstream } } impl<'a, 'cx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'a, 'cx, 'matcher> { - type Failure = (Token, usize, &'static str); + type Failure = (Token, u32, &'static str); - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure { + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure { (tok, position, msg) } @@ -211,9 +211,9 @@ impl<'matcher> FailureForwarder<'matcher> { } impl<'matcher> Tracker<'matcher> for FailureForwarder<'matcher> { - type Failure = (Token, usize, &'static str); + type Failure = (Token, u32, &'static str); - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure { + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure { (tok, position, msg) } diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 2fbd09fd9ae..99a9d4f8912 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -452,7 +452,7 @@ impl TtParser { &mut self, matcher: &'matcher [MatcherLoc], token: &Token, - approx_position: usize, + approx_position: u32, track: &mut T, ) -> Option> { // Matcher positions that would be valid if the macro invocation was over now. Only diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index e43ba7c3a5a..88ec3d83664 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -153,7 +153,7 @@ pub(super) trait Tracker<'matcher> { /// Arm failed to match. If the token is `token::Eof`, it indicates an unexpected /// end of macro invocation. Otherwise, it indicates that no rules expected the given token. /// The usize is the approximate position of the token in the input token stream. - fn build_failure(tok: Token, position: usize, msg: &'static str) -> Self::Failure; + fn build_failure(tok: Token, position: u32, msg: &'static str) -> Self::Failure; /// This is called before trying to match next MatcherLoc on the current token. fn before_match_loc(&mut self, _parser: &TtParser, _matcher: &'matcher MatcherLoc) {} @@ -182,7 +182,7 @@ pub(super) struct NoopTracker; impl<'matcher> Tracker<'matcher> for NoopTracker { type Failure = (); - fn build_failure(_tok: Token, _position: usize, _msg: &'static str) -> Self::Failure {} + fn build_failure(_tok: Token, _position: u32, _msg: &'static str) -> Self::Failure {} fn description() -> &'static str { "none" diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 10feccd21e2..d0d01cec816 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -29,15 +29,15 @@ pub struct AttrWrapper { // The start of the outer attributes in the token cursor. // This allows us to create a `ReplaceRange` for the entire attribute // target, including outer attributes. - start_pos: usize, + start_pos: u32, } impl AttrWrapper { - pub(super) fn new(attrs: AttrVec, start_pos: usize) -> AttrWrapper { + pub(super) fn new(attrs: AttrVec, start_pos: u32) -> AttrWrapper { AttrWrapper { attrs, start_pos } } pub fn empty() -> AttrWrapper { - AttrWrapper { attrs: AttrVec::new(), start_pos: usize::MAX } + AttrWrapper { attrs: AttrVec::new(), start_pos: u32::MAX } } pub(crate) fn take_for_recovery(self, psess: &ParseSess) -> AttrVec { @@ -91,7 +91,7 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool { struct LazyAttrTokenStreamImpl { start_token: (Token, Spacing), cursor_snapshot: TokenCursor, - num_calls: usize, + num_calls: u32, break_last_token: bool, replace_ranges: Box<[ReplaceRange]>, } @@ -110,7 +110,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { let token = cursor_snapshot.next(); (FlatToken::Token(token.0), token.1) })) - .take(self.num_calls); + .take(self.num_calls as usize); if self.replace_ranges.is_empty() { make_token_stream(tokens, self.break_last_token) @@ -296,12 +296,12 @@ impl<'a> Parser<'a> { ); let end_pos = self.num_bump_calls - + captured_trailing as usize + + captured_trailing as u32 // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), then // extend the range of captured tokens to include it, since the parser was not actually // bumped past it. When the `LazyAttrTokenStream` gets converted into an // `AttrTokenStream`, we will create the proper token. - + self.break_last_token as usize; + + self.break_last_token as u32; let num_calls = end_pos - start_pos; @@ -313,14 +313,11 @@ impl<'a> Parser<'a> { // Grab any replace ranges that occur *inside* the current AST node. // We will perform the actual replacement when we convert the `LazyAttrTokenStream` // to an `AttrTokenStream`. - let start_calls: u32 = start_pos.try_into().unwrap(); self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] .iter() .cloned() .chain(inner_attr_replace_ranges.iter().cloned()) - .map(|(range, tokens)| { - ((range.start - start_calls)..(range.end - start_calls), tokens) - }) + .map(|(range, tokens)| ((range.start - start_pos)..(range.end - start_pos), tokens)) .collect() }; @@ -459,6 +456,6 @@ mod size_asserts { use rustc_data_structures::static_assert_size; // tidy-alphabetical-start static_assert_size!(AttrWrapper, 16); - static_assert_size!(LazyAttrTokenStreamImpl, 104); + static_assert_size!(LazyAttrTokenStreamImpl, 96); // tidy-alphabetical-end } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index cfd0a72c056..5f16a3e1f37 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -153,7 +153,7 @@ pub struct Parser<'a> { expected_tokens: Vec, token_cursor: TokenCursor, // The number of calls to `bump`, i.e. the position in the token stream. - num_bump_calls: usize, + num_bump_calls: u32, // During parsing we may sometimes need to 'unglue' a glued token into two // component tokens (e.g. '>>' into '>' and '>), so the parser can consume // them one at a time. This process bypasses the normal capturing mechanism @@ -192,7 +192,7 @@ pub struct Parser<'a> { // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure // it doesn't unintentionally get bigger. #[cfg(target_pointer_width = "64")] -rustc_data_structures::static_assert_size!(Parser<'_>, 264); +rustc_data_structures::static_assert_size!(Parser<'_>, 256); /// Stores span information about a closure. #[derive(Clone, Debug)] @@ -1572,7 +1572,7 @@ impl<'a> Parser<'a> { self.expected_tokens.clear(); } - pub fn approx_token_stream_pos(&self) -> usize { + pub fn approx_token_stream_pos(&self) -> u32 { self.num_bump_calls } } From 6f6015679fb0153bd35d1b56df2456bb6146a9af Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 17:38:43 +1000 Subject: [PATCH 11/12] Rename `make_token_stream`. And update the comment. Clearly the return type of this function was changed at some point in the past, but its name and comment weren't updated to match. --- compiler/rustc_parse/src/parser/attr_wrapper.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index d0d01cec816..3afd178cc2b 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -113,7 +113,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { .take(self.num_calls as usize); if self.replace_ranges.is_empty() { - make_token_stream(tokens, self.break_last_token) + make_attr_token_stream(tokens, self.break_last_token) } else { let mut tokens: Vec<_> = tokens.collect(); let mut replace_ranges = self.replace_ranges.to_vec(); @@ -166,7 +166,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { new_tokens.into_iter().chain(filler), ); } - make_token_stream(tokens.into_iter(), self.break_last_token) + make_attr_token_stream(tokens.into_iter(), self.break_last_token) } } } @@ -374,10 +374,10 @@ impl<'a> Parser<'a> { } } -/// Converts a flattened iterator of tokens (including open and close delimiter tokens) -/// into a `TokenStream`, creating a `TokenTree::Delimited` for each matching pair -/// of open and close delims. -fn make_token_stream( +/// Converts a flattened iterator of tokens (including open and close delimiter tokens) into an +/// `AttrTokenStream`, creating an `AttrTokenTree::Delimited` for each matching pair of open and +/// close delims. +fn make_attr_token_stream( mut iter: impl Iterator, break_last_token: bool, ) -> AttrTokenStream { From edeebe675b39cbdca93ec5b4c79f9a0765678112 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 2 Jul 2024 20:29:01 +1000 Subject: [PATCH 12/12] Import `std::{iter,mem}`. --- .../rustc_parse/src/parser/attr_wrapper.rs | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 3afd178cc2b..13a647adfe3 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -9,6 +9,7 @@ use rustc_session::parse::ParseSess; use rustc_span::{sym, Span, DUMMY_SP}; use std::ops::Range; +use std::{iter, mem}; /// A wrapper type to ensure that the parser handles outer attributes correctly. /// When we parse outer attributes, we need to ensure that we capture tokens @@ -53,7 +54,7 @@ impl AttrWrapper { // FIXME: require passing an NT to prevent misuse of this method pub(crate) fn prepend_to_nt_inner(self, attrs: &mut AttrVec) { let mut self_attrs = self.attrs; - std::mem::swap(attrs, &mut self_attrs); + mem::swap(attrs, &mut self_attrs); attrs.extend(self_attrs); } @@ -104,13 +105,12 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // produce an empty `TokenStream` if no calls were made, and omit the // final token otherwise. let mut cursor_snapshot = self.cursor_snapshot.clone(); - let tokens = - std::iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) - .chain(std::iter::repeat_with(|| { - let token = cursor_snapshot.next(); - (FlatToken::Token(token.0), token.1) - })) - .take(self.num_calls as usize); + let tokens = iter::once((FlatToken::Token(self.start_token.0.clone()), self.start_token.1)) + .chain(iter::repeat_with(|| { + let token = cursor_snapshot.next(); + (FlatToken::Token(token.0), token.1) + })) + .take(self.num_calls as usize); if self.replace_ranges.is_empty() { make_attr_token_stream(tokens, self.break_last_token) @@ -158,7 +158,7 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl { // This keeps the total length of `tokens` constant throughout the // replacement process, allowing us to use all of the `ReplaceRanges` entries // without adjusting indices. - let filler = std::iter::repeat((FlatToken::Empty, Spacing::Alone)) + let filler = iter::repeat((FlatToken::Empty, Spacing::Alone)) .take(range.len() - new_tokens.len()); tokens.splice( @@ -222,8 +222,7 @@ impl<'a> Parser<'a> { let replace_ranges_start = self.capture_state.replace_ranges.len(); let (mut ret, trailing) = { - let prev_capturing = - std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes); + let prev_capturing = mem::replace(&mut self.capture_state.capturing, Capturing::Yes); let ret_and_trailing = f(self, attrs.attrs); self.capture_state.capturing = prev_capturing; ret_and_trailing?