From 364b908d574a8e416ceb18fac968d5142765e5e5 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 25 Mar 2022 12:39:12 +1100 Subject: [PATCH] Remove `Nonterminal::NtTT`. It's only needed for macro expansion, not as a general element in the AST. This commit removes it, adds `NtOrTt` for the parser and macro expansion cases, and renames the variants in `NamedMatch` to better match the new type. --- compiler/rustc_ast/src/ast_like.rs | 4 +- compiler/rustc_ast/src/mut_visit.rs | 1 - compiler/rustc_ast/src/token.rs | 5 --- compiler/rustc_ast_pretty/src/pprust/state.rs | 1 - compiler/rustc_expand/src/mbe/macro_parser.rs | 37 +++++++++---------- compiler/rustc_expand/src/mbe/macro_rules.rs | 6 +-- compiler/rustc_expand/src/mbe/transcribe.rs | 15 ++++---- compiler/rustc_parse/src/lib.rs | 1 - compiler/rustc_parse/src/parser/mod.rs | 8 +++- .../rustc_parse/src/parser/nonterminal.rs | 29 +++++++++------ 10 files changed, 52 insertions(+), 55 deletions(-) diff --git a/compiler/rustc_ast/src/ast_like.rs b/compiler/rustc_ast/src/ast_like.rs index 9a24158ba35..1a271b0adef 100644 --- a/compiler/rustc_ast/src/ast_like.rs +++ b/compiler/rustc_ast/src/ast_like.rs @@ -51,7 +51,6 @@ fn attrs(&self) -> &[Attribute] { | Nonterminal::NtMeta(_) | Nonterminal::NtPath(_) | Nonterminal::NtVis(_) - | Nonterminal::NtTT(_) | Nonterminal::NtBlock(_) | Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(_) => &[], @@ -67,7 +66,6 @@ fn visit_attrs(&mut self, f: impl FnOnce(&mut Vec)) { | Nonterminal::NtMeta(_) | Nonterminal::NtPath(_) | Nonterminal::NtVis(_) - | Nonterminal::NtTT(_) | Nonterminal::NtBlock(_) | Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(_) => {} @@ -84,7 +82,7 @@ fn tokens_mut(&mut self) -> Option<&mut Option> { Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(), - Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) | Nonterminal::NtTT(..) => None, + Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } } diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 9d79fe53d5c..32621eb5f2f 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -787,7 +787,6 @@ pub fn visit_interpolated(nt: &mut token::Nonterminal, vis: &mut visit_lazy_tts(tokens, vis); } token::NtPath(path) => vis.visit_path(path), - token::NtTT(tt) => visit_tt(tt, vis), token::NtVis(visib) => vis.visit_vis(visib), } } diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index c367573de8a..7df385873c1 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -6,7 +6,6 @@ use crate::ast; use crate::ptr::P; -use crate::tokenstream::TokenTree; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::Lrc; @@ -680,7 +679,6 @@ pub enum Nonterminal { NtMeta(P), NtPath(ast::Path), NtVis(ast::Visibility), - NtTT(TokenTree), } // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. @@ -778,7 +776,6 @@ pub fn span(&self) -> Span { NtMeta(attr_item) => attr_item.span(), NtPath(path) => path.span, NtVis(vis) => vis.span, - NtTT(tt) => tt.span(), } } } @@ -790,7 +787,6 @@ fn eq(&self, rhs: &Self) -> bool { ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs } (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, - (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them // correctly based on data from AST. This will prevent them from matching each other // in macros. The comparison will become possible only when each nonterminal has an @@ -813,7 +809,6 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { NtLiteral(..) => f.pad("NtLiteral(..)"), NtMeta(..) => f.pad("NtMeta(..)"), NtPath(..) => f.pad("NtPath(..)"), - NtTT(..) => f.pad("NtTT(..)"), NtVis(..) => f.pad("NtVis(..)"), NtLifetime(..) => f.pad("NtLifetime(..)"), } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index b2c62383fb6..719fd271093 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -714,7 +714,6 @@ fn nonterminal_to_string(&self, nt: &Nonterminal) -> String { token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw).to_string(), token::NtLifetime(e) => e.to_string(), token::NtLiteral(ref e) => self.expr_to_string(e), - token::NtTT(ref tree) => self.tt_to_string(tree), token::NtVis(ref e) => self.vis_to_string(e), } } diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs index 5e97fc90320..a8db96a199d 100644 --- a/compiler/rustc_expand/src/mbe/macro_parser.rs +++ b/compiler/rustc_expand/src/mbe/macro_parser.rs @@ -76,7 +76,7 @@ use crate::mbe::{self, SequenceRepetition, TokenTree}; use rustc_ast::token::{self, DocComment, Nonterminal, Token}; -use rustc_parse::parser::Parser; +use rustc_parse::parser::{NtOrTt, Parser}; use rustc_session::parse::ParseSess; use rustc_span::symbol::MacroRulesNormalizedIdent; @@ -275,7 +275,7 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize { } /// `NamedMatch` is a pattern-match result for a single metavar. All -/// `MatchedNtNonTt`s in the `NamedMatch` have the same non-terminal type +/// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type /// (expr, item, etc). /// /// The in-memory structure of a particular `NamedMatch` represents the match @@ -306,17 +306,17 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize { /// ```rust /// MatchedSeq([ /// MatchedSeq([ -/// MatchedNtNonTt(a), -/// MatchedNtNonTt(b), -/// MatchedNtNonTt(c), -/// MatchedNtNonTt(d), +/// MatchedNonterminal(a), +/// MatchedNonterminal(b), +/// MatchedNonterminal(c), +/// MatchedNonterminal(d), /// ]), /// MatchedSeq([ -/// MatchedNtNonTt(a), -/// MatchedNtNonTt(b), -/// MatchedNtNonTt(c), -/// MatchedNtNonTt(d), -/// MatchedNtNonTt(e), +/// MatchedNonterminal(a), +/// MatchedNonterminal(b), +/// MatchedNonterminal(c), +/// MatchedNonterminal(d), +/// MatchedNonterminal(e), /// ]) /// ]) /// ``` @@ -324,14 +324,11 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize { crate enum NamedMatch { MatchedSeq(Lrc), - // This variant should never hold an `NtTT`. `MatchedNtTt` should be used - // for that case. - MatchedNtNonTt(Lrc), + // A metavar match of type `tt`. + MatchedTokenTree(rustc_ast::tokenstream::TokenTree), - // `NtTT` is handled without any cloning when transcribing, unlike other - // nonterminals. Therefore, an `Lrc` isn't helpful and causes unnecessary - // allocations. Hence this separate variant. - MatchedNtTt(rustc_ast::tokenstream::TokenTree), + // A metavar match of any type other than `tt`. + MatchedNonterminal(Lrc), } /// Takes a slice of token trees `ms` representing a matcher which successfully matched input @@ -677,8 +674,8 @@ pub(super) fn parse_tt( Ok(nt) => nt, }; let m = match nt { - Nonterminal::NtTT(tt) => MatchedNtTt(tt), - _ => MatchedNtNonTt(Lrc::new(nt)), + NtOrTt::Nt(nt) => MatchedNonterminal(Lrc::new(nt)), + NtOrTt::Tt(tt) => MatchedTokenTree(tt), }; item.push_match(match_cur, m); item.idx += 1; diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 7837de5c18d..10b2b9f07e2 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -4,7 +4,7 @@ use crate::mbe; use crate::mbe::macro_check; use crate::mbe::macro_parser::{Error, ErrorReported, Failure, Success, TtParser}; -use crate::mbe::macro_parser::{MatchedNtTt, MatchedSeq}; +use crate::mbe::macro_parser::{MatchedSeq, MatchedTokenTree}; use crate::mbe::transcribe::transcribe; use rustc_ast as ast; @@ -470,7 +470,7 @@ pub fn compile_declarative_macro( MatchedSeq(ref s) => s .iter() .map(|m| { - if let MatchedNtTt(ref tt) = *m { + if let MatchedTokenTree(ref tt) = *m { let mut tts = vec![]; mbe::quoted::parse( tt.clone().into(), @@ -495,7 +495,7 @@ pub fn compile_declarative_macro( MatchedSeq(ref s) => s .iter() .map(|m| { - if let MatchedNtTt(ref tt) = *m { + if let MatchedTokenTree(ref tt) = *m { let mut tts = vec![]; mbe::quoted::parse( tt.clone().into(), diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index 228ed04548d..cd016bfd8f4 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -1,8 +1,8 @@ use crate::base::ExtCtxt; -use crate::mbe::macro_parser::{MatchedNtNonTt, MatchedNtTt, MatchedSeq, NamedMatch}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree, NamedMatch}; use crate::mbe::{self, MetaVarExpr}; use rustc_ast::mut_visit::{self, MutVisitor}; -use rustc_ast::token::{self, Nonterminal, Token, TokenKind}; +use rustc_ast::token::{self, Token, TokenKind}; use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -234,17 +234,16 @@ pub(super) fn transcribe<'a>( let ident = MacroRulesNormalizedIdent::new(orignal_ident); if let Some(cur_matched) = lookup_cur_matched(ident, interp, &repeats) { match cur_matched { - MatchedNtTt(ref tt) => { + MatchedTokenTree(ref tt) => { // `tt`s are emitted into the output stream directly as "raw tokens", // without wrapping them into groups. let token = tt.clone(); result.push(token.into()); } - MatchedNtNonTt(ref nt) => { + MatchedNonterminal(ref nt) => { // Other variables are emitted into the output stream as groups with // `Delimiter::None` to maintain parsing priorities. // `Interpolated` is currently used for such groups in rustc parser. - debug_assert!(!matches!(**nt, Nonterminal::NtTT(_))); marker.visit_span(&mut sp); let token = TokenTree::token(token::Interpolated(nt.clone()), sp); result.push(token.into()); @@ -312,7 +311,7 @@ fn lookup_cur_matched<'a>( let mut matched = matched; for &(idx, _) in repeats { match matched { - MatchedNtTt(_) | MatchedNtNonTt(_) => break, + MatchedTokenTree(_) | MatchedNonterminal(_) => break, MatchedSeq(ref ads) => matched = ads.get(idx).unwrap(), } } @@ -402,7 +401,7 @@ fn lockstep_iter_size( let name = MacroRulesNormalizedIdent::new(name); match lookup_cur_matched(name, interpolations, repeats) { Some(matched) => match matched { - MatchedNtTt(_) | MatchedNtNonTt(_) => LockstepIterSize::Unconstrained, + MatchedTokenTree(_) | MatchedNonterminal(_) => LockstepIterSize::Unconstrained, MatchedSeq(ref ads) => LockstepIterSize::Constraint(ads.len(), name), }, _ => LockstepIterSize::Unconstrained, @@ -449,7 +448,7 @@ fn count<'a>( sp: &DelimSpan, ) -> PResult<'a, usize> { match matched { - MatchedNtTt(_) | MatchedNtNonTt(_) => { + MatchedTokenTree(_) | MatchedNonterminal(_) => { if declared_lhs_depth == 0 { return Err(cx.struct_span_err( sp.entire(), diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index a3e66464fbc..a614aed692d 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -289,7 +289,6 @@ pub fn nt_to_tokenstream( Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()), Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()), Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()), - Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { prepend_attrs(&expr.attrs, expr.tokens.as_ref()) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 3a2f193d319..5d244ef9118 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -19,7 +19,7 @@ pub use path::PathStyle; use rustc_ast::ptr::P; -use rustc_ast::token::{self, DelimToken, Token, TokenKind}; +use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind}; use rustc_ast::tokenstream::AttributesData; use rustc_ast::tokenstream::{self, DelimSpan, Spacing}; use rustc_ast::tokenstream::{TokenStream, TokenTree}; @@ -1507,3 +1507,9 @@ pub enum FlatToken { /// handling of replace ranges. Empty, } + +#[derive(Debug)] +pub enum NtOrTt { + Nt(Nonterminal), + Tt(TokenTree), +} diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 40902fa1833..c105fbfaee0 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -1,12 +1,12 @@ use rustc_ast::ptr::P; -use rustc_ast::token::{self, Nonterminal, NonterminalKind, Token}; +use rustc_ast::token::{self, NonterminalKind, Token}; use rustc_ast::AstLike; use rustc_ast_pretty::pprust; use rustc_errors::PResult; use rustc_span::symbol::{kw, Ident}; use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma}; -use crate::parser::{FollowedByType, ForceCollect, Parser, PathStyle}; +use crate::parser::{FollowedByType, ForceCollect, NtOrTt, Parser, PathStyle}; impl<'a> Parser<'a> { /// Checks whether a non-terminal may begin with a particular token. @@ -85,7 +85,7 @@ fn may_be_ident(nt: &token::Nonterminal) -> bool { NonterminalKind::Lifetime => match token.kind { token::Lifetime(_) => true, token::Interpolated(ref nt) => { - matches!(**nt, token::NtLifetime(_) | token::NtTT(_)) + matches!(**nt, token::NtLifetime(_)) } _ => false, }, @@ -96,7 +96,7 @@ fn may_be_ident(nt: &token::Nonterminal) -> bool { } /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`). - pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonterminal> { + pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, NtOrTt> { // Any `Nonterminal` which stores its tokens (currently `NtItem` and `NtExpr`) // needs to have them force-captured here. // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro, @@ -104,6 +104,8 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter // in advance whether or not a proc-macro will be (transitively) invoked, // we always capture tokens for any `Nonterminal` which needs them. let mut nt = match kind { + // Note that TT is treated differently to all the others. + NonterminalKind::TT => return Ok(NtOrTt::Tt(self.parse_token_tree())), NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? { Some(item) => token::NtItem(item), None => { @@ -124,9 +126,12 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr { .. } => { token::NtPat(self.collect_tokens_no_attrs(|this| match kind { NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None), - NonterminalKind::PatWithOr { .. } => { - this.parse_pat_allow_top_alt(None, RecoverComma::No, RecoverColon::No, CommaRecoveryMode::EitherTupleOrPipe) - } + NonterminalKind::PatWithOr { .. } => this.parse_pat_allow_top_alt( + None, + RecoverComma::No, + RecoverColon::No, + CommaRecoveryMode::EitherTupleOrPipe, + ), _ => unreachable!(), })?) } @@ -139,9 +144,10 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter ) } - NonterminalKind::Ty => { - token::NtTy(self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?) - } + NonterminalKind::Ty => token::NtTy( + self.collect_tokens_no_attrs(|this| this.parse_no_question_mark_recover())?, + ), + // this could be handled like a token, since it is one NonterminalKind::Ident if let Some((ident, is_raw)) = get_macro_ident(&self.token) => @@ -158,7 +164,6 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?, ), NonterminalKind::Meta => token::NtMeta(P(self.parse_attr_item(true)?)), - NonterminalKind::TT => token::NtTT(self.parse_token_tree()), NonterminalKind::Vis => token::NtVis( self.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?, ), @@ -183,7 +188,7 @@ pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, Nonter ); } - Ok(nt) + Ok(NtOrTt::Nt(nt)) } }