mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #114115 - nnethercote:less-token-tree-cloning, r=petrochenkov

Browse Source 
Less `TokenTree` cloning

`TokenTreeCursor` has this comment on it:
```
// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones.
```
This PR completes that FIXME. It doesn't have much perf effect, but at least we now know that.

r? `@petrochenkov`
This commit is contained in:
bors 2023-07-28 01:21:27 +00:00
commit 0699d99516
9 changed files with 83 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
compiler/rustc_ast/src/attr/mod.rs
View File

@ -285,17 +285,17 @@ pub fn value_str(&self) -> Option<Symbol> {
self.kind.value_str() self.kind.value_str()
} }
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem> fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
where where
I: Iterator<Item = TokenTree>, I: Iterator<Item = &'a TokenTree>,
{ {
// FIXME: Share code with `parse_path`. // FIXME: Share code with `parse_path`.
let path = match tokens.next().map(TokenTree::uninterpolate) { let path = match tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref() {
Some(TokenTree::Token( Some(&TokenTree::Token(
Token { kind: kind @ (token::Ident(..) | token::ModSep), span }, Token { kind: ref kind @ (token::Ident(..) | token::ModSep), span },
_, _,
)) => 'arm: { )) => 'arm: {
let mut segments = if let token::Ident(name, _) = kind { let mut segments = if let &token::Ident(name, _) = kind {
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) = if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
tokens.peek() tokens.peek()
{ {
@ -308,8 +308,8 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
thin_vec![PathSegment::path_root(span)] thin_vec![PathSegment::path_root(span)]
}; };
loop { loop {
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) = if let Some(&TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
tokens.next().map(TokenTree::uninterpolate) tokens.next().map(|tt| TokenTree::uninterpolate(tt)).as_deref()
{ {
segments.push(PathSegment::from_ident(Ident::new(name, span))); segments.push(PathSegment::from_ident(Ident::new(name, span)));
} else { } else {
@ -326,7 +326,7 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
let span = span.with_hi(segments.last().unwrap().ident.span.hi()); let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments, tokens: None } Path { span, segments, tokens: None }
} }
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &*nt { Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt {
token::Nonterminal::NtMeta(item) => return item.meta(item.path.span), token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(path) => (**path).clone(), token::Nonterminal::NtPath(path) => (**path).clone(),
_ => return None, _ => return None,
@ -354,7 +354,7 @@ pub fn value_str(&self) -> Option<Symbol> {
} }
fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> { fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
let mut tokens = tokens.into_trees().peekable(); let mut tokens = tokens.trees().peekable();
let mut result = ThinVec::new(); let mut result = ThinVec::new();
while tokens.peek().is_some() { while tokens.peek().is_some() {
let item = NestedMetaItem::from_tokens(&mut tokens)?; let item = NestedMetaItem::from_tokens(&mut tokens)?;
@ -367,12 +367,12 @@ fn list_from_tokens(tokens: TokenStream) -> Option<ThinVec<NestedMetaItem>> {
Some(result) Some(result)
} }
fn name_value_from_tokens( fn name_value_from_tokens<'a>(
tokens: &mut impl Iterator<Item = TokenTree>, tokens: &mut impl Iterator<Item = &'a TokenTree>,
) -> Option<MetaItemKind> { ) -> Option<MetaItemKind> {
match tokens.next() { match tokens.next() {
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees()) MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
} }
Some(TokenTree::Token(token, _)) => { Some(TokenTree::Token(token, _)) => {
MetaItemLit::from_token(&token).map(MetaItemKind::NameValue) MetaItemLit::from_token(&token).map(MetaItemKind::NameValue)
@ -381,8 +381,8 @@ fn name_value_from_tokens(
} }
} }
fn from_tokens( fn from_tokens<'a>(
tokens: &mut iter::Peekable<impl Iterator<Item = TokenTree>>, tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
) -> Option<MetaItemKind> { ) -> Option<MetaItemKind> {
match tokens.peek() { match tokens.peek() {
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => { Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
@ -501,9 +501,9 @@ pub fn is_meta_item(&self) -> bool {
self.meta_item().is_some() self.meta_item().is_some()
} }
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem> fn from_tokens<'a, I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
where where
I: Iterator<Item = TokenTree>, I: Iterator<Item = &'a TokenTree>,
{ {
match tokens.peek() { match tokens.peek() {
Some(TokenTree::Token(token, _)) Some(TokenTree::Token(token, _))
@ -513,9 +513,8 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
return Some(NestedMetaItem::Lit(lit)); return Some(NestedMetaItem::Lit(lit));
} }
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
let inner_tokens = inner_tokens.clone();
tokens.next(); tokens.next();
return NestedMetaItem::from_tokens(&mut inner_tokens.into_trees().peekable()); return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
} }
_ => {} _ => {}
} }

33
compiler/rustc_ast/src/tokenstream.rs
View File

@ -25,6 +25,7 @@
use rustc_span::{Span, DUMMY_SP}; use rustc_span::{Span, DUMMY_SP};
use smallvec::{smallvec, SmallVec}; use smallvec::{smallvec, SmallVec};
use std::borrow::Cow;
use std::{fmt, iter, mem}; use std::{fmt, iter, mem};
/// When the main Rust parser encounters a syntax-extension invocation, it /// When the main Rust parser encounters a syntax-extension invocation, it
@ -98,12 +99,13 @@ pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint) TokenTree::Token(Token::new(kind, span), Spacing::Joint)
} }
pub fn uninterpolate(self) -> TokenTree { pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
match self { match self {
TokenTree::Token(token, spacing) => { TokenTree::Token(token, spacing) => match token.uninterpolate() {
TokenTree::Token(token.uninterpolate().into_owned(), spacing) Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
} Cow::Borrowed(_) => Cow::Borrowed(self),
tt => tt, },
_ => Cow::Borrowed(self),
} }
} }
} }
@ -595,26 +597,21 @@ fn next(&mut self) -> Option<&'t TokenTree> {
} }
} }
/// Owning by-value iterator over a [`TokenStream`], that produces `TokenTree` /// Owning by-value iterator over a [`TokenStream`], that produces `&TokenTree`
/// items. /// items.
// FIXME: Many uses of this can be replaced with by-reference iterator to avoid clones. ///
/// Doesn't impl `Iterator` because Rust doesn't permit an owning iterator to
/// return `&T` from `next`; the need for an explicit lifetime in the `Item`
/// associated type gets in the way. Instead, use `next_ref` (which doesn't
/// involve associated types) for getting individual elements, or
/// `RefTokenTreeCursor` if you really want an `Iterator`, e.g. in a `for`
/// loop.
#[derive(Clone)] #[derive(Clone)]
pub struct TokenTreeCursor { pub struct TokenTreeCursor {
pub stream: TokenStream, pub stream: TokenStream,
index: usize, index: usize,
} }
impl Iterator for TokenTreeCursor {
type Item = TokenTree;
fn next(&mut self) -> Option<TokenTree> {
self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree.clone()
})
}
}
impl TokenTreeCursor { impl TokenTreeCursor {
fn new(stream: TokenStream) -> Self { fn new(stream: TokenStream) -> Self {
TokenTreeCursor { stream, index: 0 } TokenTreeCursor { stream, index: 0 }

6
compiler/rustc_expand/src/config.rs
View File

@ -365,9 +365,9 @@ fn expand_cfg_attr_item(
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
// for `attr` when we expand it to `#[attr]` // for `attr` when we expand it to `#[attr]`
let mut orig_trees = orig_tokens.into_trees(); let mut orig_trees = orig_tokens.trees();
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) =
orig_trees.next().unwrap() orig_trees.next().unwrap().clone()
else { else {
panic!("Bad tokens for attribute {:?}", attr); panic!("Bad tokens for attribute {:?}", attr);
}; };
@ -377,7 +377,7 @@ fn expand_cfg_attr_item(
if attr.style == AttrStyle::Inner { if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]` // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
orig_trees.next().unwrap() orig_trees.next().unwrap().clone()
else { else {
panic!("Bad tokens for attribute {:?}", attr); panic!("Bad tokens for attribute {:?}", attr);
}; };

4
compiler/rustc_expand/src/mbe/macro_rules.rs
View File

@ -500,7 +500,7 @@ pub fn compile_declarative_macro(
.map(|m| { .map(|m| {
if let MatchedTokenTree(tt) = m { if let MatchedTokenTree(tt) = m {
let tt = mbe::quoted::parse( let tt = mbe::quoted::parse(
TokenStream::new(vec![tt.clone()]), &TokenStream::new(vec![tt.clone()]),
true, true,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,
@ -524,7 +524,7 @@ pub fn compile_declarative_macro(
.map(|m| { .map(|m| {
if let MatchedTokenTree(tt) = m { if let MatchedTokenTree(tt) = m {
return mbe::quoted::parse( return mbe::quoted::parse(
TokenStream::new(vec![tt.clone()]), &TokenStream::new(vec![tt.clone()]),
false, false,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,

42
compiler/rustc_expand/src/mbe/quoted.rs
View File

@ -36,7 +36,7 @@
/// ///
/// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. /// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`.
pub(super) fn parse( pub(super) fn parse(
input: tokenstream::TokenStream, input: &tokenstream::TokenStream,
parsing_patterns: bool, parsing_patterns: bool,
sess: &ParseSess, sess: &ParseSess,
node_id: NodeId, node_id: NodeId,
@ -48,7 +48,7 @@ pub(super) fn parse(
// For each token tree in `input`, parse the token into a `self::TokenTree`, consuming // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming
// additional trees if need be. // additional trees if need be.
let mut trees = input.into_trees(); let mut trees = input.trees();
while let Some(tree) = trees.next() { while let Some(tree) = trees.next() {
// Given the parsed tree, if there is a metavar and we are expecting matchers, actually // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
// parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
@ -56,7 +56,7 @@ pub(super) fn parse(
match tree { match tree {
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => { TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
let span = match trees.next() { let span = match trees.next() {
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => { Some(&tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
match trees.next() { match trees.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() { Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
Some((frag, _)) => { Some((frag, _)) => {
@ -96,10 +96,10 @@ pub(super) fn parse(
} }
_ => token.span, _ => token.span,
}, },
tree => tree.as_ref().map_or(span, tokenstream::TokenTree::span), tree => tree.map_or(span, tokenstream::TokenTree::span),
} }
} }
tree => tree.as_ref().map_or(start_sp, tokenstream::TokenTree::span), tree => tree.map_or(start_sp, tokenstream::TokenTree::span),
}; };
result.push(TokenTree::MetaVarDecl(span, ident, None)); result.push(TokenTree::MetaVarDecl(span, ident, None));
@ -134,9 +134,9 @@ fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &ParseSess,
/// - `parsing_patterns`: same as [parse]. /// - `parsing_patterns`: same as [parse].
/// - `sess`: the parsing session. Any errors will be emitted to this session. /// - `sess`: the parsing session. Any errors will be emitted to this session.
/// - `features`: language features so we can do feature gating. /// - `features`: language features so we can do feature gating.
fn parse_tree( fn parse_tree<'a>(
tree: tokenstream::TokenTree, tree: &'a tokenstream::TokenTree,
outer_trees: &mut impl Iterator<Item = tokenstream::TokenTree>, outer_trees: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
parsing_patterns: bool, parsing_patterns: bool,
sess: &ParseSess, sess: &ParseSess,
node_id: NodeId, node_id: NodeId,
@ -146,13 +146,13 @@ fn parse_tree(
// Depending on what `tree` is, we could be parsing different parts of a macro // Depending on what `tree` is, we could be parsing different parts of a macro
match tree { match tree {
// `tree` is a `$` token. Look at the next token in `trees` // `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => { &tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way // FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing. // during parsing.
let mut next = outer_trees.next(); let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>; let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next { if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
trees = Box::new(tts.into_trees()); trees = Box::new(tts.trees());
next = trees.next(); next = trees.next();
} else { } else {
trees = Box::new(outer_trees); trees = Box::new(outer_trees);
@ -160,7 +160,7 @@ fn parse_tree(
match next { match next {
// `tree` is followed by a delimited set of token trees. // `tree` is followed by a delimited set of token trees.
Some(tokenstream::TokenTree::Delimited(delim_span, delim, tts)) => { Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => {
if parsing_patterns { if parsing_patterns {
if delim != Delimiter::Parenthesis { if delim != Delimiter::Parenthesis {
span_dollar_dollar_or_metavar_in_the_lhs_err( span_dollar_dollar_or_metavar_in_the_lhs_err(
@ -228,7 +228,7 @@ fn parse_tree(
} }
// `tree` is followed by another `$`. This is an escaped `$`. // `tree` is followed by another `$`. This is an escaped `$`.
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => { Some(&tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
if parsing_patterns { if parsing_patterns {
span_dollar_dollar_or_metavar_in_the_lhs_err( span_dollar_dollar_or_metavar_in_the_lhs_err(
sess, sess,
@ -256,11 +256,11 @@ fn parse_tree(
} }
// `tree` is an arbitrary token. Keep it. // `tree` is an arbitrary token. Keep it.
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token), tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token.clone()),
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it. // descend into the delimited set and further parse it.
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( &tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited(
span, span,
Delimited { Delimited {
delim, delim,
@ -286,16 +286,16 @@ fn kleene_op(token: &Token) -> Option<KleeneOp> {
/// - Ok(Ok((op, span))) if the next token tree is a KleeneOp /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp
/// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp
/// - Err(span) if the next token tree is not a token /// - Err(span) if the next token tree is not a token
fn parse_kleene_op( fn parse_kleene_op<'a>(
input: &mut impl Iterator<Item = tokenstream::TokenTree>, input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span, span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> { ) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() { match input.next() {
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) { Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
Some(op) => Ok(Ok((op, token.span))), Some(op) => Ok(Ok((op, token.span))),
None => Ok(Err(token)), None => Ok(Err(token.clone())),
}, },
tree => Err(tree.as_ref().map_or(span, tokenstream::TokenTree::span)), tree => Err(tree.map_or(span, tokenstream::TokenTree::span)),
} }
} }
@ -311,8 +311,8 @@ fn parse_kleene_op(
/// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene
/// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an
/// error with the appropriate span is emitted to `sess` and a dummy value is returned. /// error with the appropriate span is emitted to `sess` and a dummy value is returned.
fn parse_sep_and_kleene_op( fn parse_sep_and_kleene_op<'a>(
input: &mut impl Iterator<Item = tokenstream::TokenTree>, input: &mut impl Iterator<Item = &'a tokenstream::TokenTree>,
span: Span, span: Span,
sess: &ParseSess, sess: &ParseSess,
) -> (Option<Token>, KleeneToken) { ) -> (Option<Token>, KleeneToken) {

9
compiler/rustc_expand/src/parse/tests.rs
View File

@ -69,9 +69,8 @@ fn bad_path_expr_1() {
#[test] #[test]
fn string_to_tts_macro() { fn string_to_tts_macro() {
create_default_session_globals_then(|| { create_default_session_globals_then(|| {
let tts: Vec<_> = let stream = string_to_stream("macro_rules! zip (($a)=>($a))".to_string());
string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).into_trees().collect(); let tts = &stream.trees().collect::<Vec<_>>()[..];
let tts: &[TokenTree] = &tts[..];
match tts { match tts {
[ [
@ -300,9 +299,7 @@ fn parse_expr_from_source_str(
.unwrap(); .unwrap();
let ast::ExprKind::MacCall(mac) = &expr.kind else { panic!("not a macro") }; let ast::ExprKind::MacCall(mac) = &expr.kind else { panic!("not a macro") };
let tts: Vec<_> = mac.args.tokens.clone().into_trees().collect(); let span = mac.args.tokens.trees().last().unwrap().span();
let span = tts.iter().rev().next().unwrap().span();
match sess.source_map().span_to_snippet(span) { match sess.source_map().span_to_snippet(span) {
Ok(s) => assert_eq!(&s[..], "{ body }"), Ok(s) => assert_eq!(&s[..], "{ body }"),

4
compiler/rustc_expand/src/proc_macro_server.rs
View File

@ -94,10 +94,10 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
// Estimate the capacity as `stream.len()` rounded up to the next power // Estimate the capacity as `stream.len()` rounded up to the next power
// of two to limit the number of required reallocations. // of two to limit the number of required reallocations.
let mut trees = Vec::with_capacity(stream.len().next_power_of_two()); let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.into_trees(); let mut cursor = stream.trees();
while let Some(tree) = cursor.next() { while let Some(tree) = cursor.next() {
let (Token { kind, span }, joint) = match tree { let (Token { kind, span }, joint) = match tree.clone() {
tokenstream::TokenTree::Delimited(span, delim, tts) => { tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim); let delimiter = pm::Delimiter::from_internal(delim);
trees.push(TokenTree::Group(Group { trees.push(TokenTree::Group(Group {

10
src/librustdoc/clean/mod.rs
View File

@ -2489,19 +2489,19 @@ fn get_all_import_attributes<'hir>(
} }
fn filter_tokens_from_list( fn filter_tokens_from_list(
args_tokens: TokenStream, args_tokens: &TokenStream,
should_retain: impl Fn(&TokenTree) -> bool, should_retain: impl Fn(&TokenTree) -> bool,
) -> Vec<TokenTree> { ) -> Vec<TokenTree> {
let mut tokens = Vec::with_capacity(args_tokens.len()); let mut tokens = Vec::with_capacity(args_tokens.len());
let mut skip_next_comma = false; let mut skip_next_comma = false;
for token in args_tokens.into_trees() { for token in args_tokens.trees() {
match token { match token {
TokenTree::Token(Token { kind: TokenKind::Comma, .. }, _) if skip_next_comma => { TokenTree::Token(Token { kind: TokenKind::Comma, .. }, _) if skip_next_comma => {
skip_next_comma = false; skip_next_comma = false;
} }
token if should_retain(&token) => { token if should_retain(token) => {
skip_next_comma = false; skip_next_comma = false;
tokens.push(token); tokens.push(token.clone());
} }
_ => { _ => {
skip_next_comma = true; skip_next_comma = true;
@ -2559,7 +2559,7 @@ fn add_without_unwanted_attributes<'hir>(
match normal.item.args { match normal.item.args {
ast::AttrArgs::Delimited(ref mut args) => { ast::AttrArgs::Delimited(ref mut args) => {
let tokens = let tokens =
filter_tokens_from_list(args.tokens.clone(), |token| { filter_tokens_from_list(&args.tokens, |token| {
!matches!( !matches!(
token, token,
TokenTree::Token( TokenTree::Token(

28
src/tools/rustfmt/src/macros.rs
View File

@ -13,7 +13,7 @@
use std::panic::{catch_unwind, AssertUnwindSafe}; use std::panic::{catch_unwind, AssertUnwindSafe};
use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind}; use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor}; use rustc_ast::tokenstream::{RefTokenTreeCursor, TokenStream, TokenTree};
use rustc_ast::{ast, ptr}; use rustc_ast::{ast, ptr};
use rustc_ast_pretty::pprust; use rustc_ast_pretty::pprust;
use rustc_span::{ use rustc_span::{
@ -394,7 +394,7 @@ pub(crate) fn rewrite_macro_def(
} }
let ts = def.body.tokens.clone(); let ts = def.body.tokens.clone();
let mut parser = MacroParser::new(ts.into_trees()); let mut parser = MacroParser::new(ts.trees());
let parsed_def = match parser.parse() { let parsed_def = match parser.parse() {
Some(def) => def, Some(def) => def,
None => return snippet, None => return snippet,
@ -736,9 +736,9 @@ fn add_other(&mut self) {
self.buf.clear(); self.buf.clear();
} }
fn add_meta_variable(&mut self, iter: &mut TokenTreeCursor) -> Option<()> { fn add_meta_variable(&mut self, iter: &mut RefTokenTreeCursor<'_>) -> Option<()> {
match iter.next() { match iter.next() {
Some(TokenTree::Token( Some(&TokenTree::Token(
Token { Token {
kind: TokenKind::Ident(name, _), kind: TokenKind::Ident(name, _),
.. ..
@ -768,7 +768,7 @@ fn add_repeat(
&mut self, &mut self,
inner: Vec<ParsedMacroArg>, inner: Vec<ParsedMacroArg>,
delim: Delimiter, delim: Delimiter,
iter: &mut TokenTreeCursor, iter: &mut RefTokenTreeCursor<'_>,
) -> Option<()> { ) -> Option<()> {
let mut buffer = String::new(); let mut buffer = String::new();
let mut first = true; let mut first = true;
@ -868,11 +868,11 @@ fn need_space_prefix(&self) -> bool {
/// Returns a collection of parsed macro def's arguments. /// Returns a collection of parsed macro def's arguments.
fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> { fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
let mut iter = tokens.into_trees(); let mut iter = tokens.trees();
while let Some(tok) = iter.next() { while let Some(tok) = iter.next() {
match tok { match tok {
TokenTree::Token( &TokenTree::Token(
Token { Token {
kind: TokenKind::Dollar, kind: TokenKind::Dollar,
span, span,
@ -901,7 +901,7 @@ fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
self.add_meta_variable(&mut iter)?; self.add_meta_variable(&mut iter)?;
} }
TokenTree::Token(ref t, _) => self.update_buffer(t), TokenTree::Token(ref t, _) => self.update_buffer(t),
TokenTree::Delimited(_delimited_span, delimited, ref tts) => { &TokenTree::Delimited(_delimited_span, delimited, ref tts) => {
if !self.buf.is_empty() { if !self.buf.is_empty() {
if next_space(&self.last_tok.kind) == SpaceState::Always { if next_space(&self.last_tok.kind) == SpaceState::Always {
self.add_separator(); self.add_separator();
@ -1119,12 +1119,12 @@ pub(crate) fn macro_style(mac: &ast::MacCall, context: &RewriteContext<'_>) -> D
// A very simple parser that just parses a macros 2.0 definition into its branches. // A very simple parser that just parses a macros 2.0 definition into its branches.
// Currently we do not attempt to parse any further than that. // Currently we do not attempt to parse any further than that.
struct MacroParser { struct MacroParser<'a> {
toks: TokenTreeCursor, toks: RefTokenTreeCursor<'a>,
} }
impl MacroParser { impl<'a> MacroParser<'a> {
const fn new(toks: TokenTreeCursor) -> Self { const fn new(toks: RefTokenTreeCursor<'a>) -> Self {
Self { toks } Self { toks }
} }
@ -1143,9 +1143,9 @@ fn parse_branch(&mut self) -> Option<MacroBranch> {
let tok = self.toks.next()?; let tok = self.toks.next()?;
let (lo, args_paren_kind) = match tok { let (lo, args_paren_kind) = match tok {
TokenTree::Token(..) => return None, TokenTree::Token(..) => return None,
TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d), &TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d),
}; };
let args = TokenStream::new(vec![tok]); let args = TokenStream::new(vec![tok.clone()]);
match self.toks.next()? { match self.toks.next()? {
TokenTree::Token( TokenTree::Token(
Token { Token {