diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 3f4911c4ecf..5e4288c344b 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -24,7 +24,7 @@ use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter}; -use crate::tokenstream::{DelimSpan, LazyTokenStream, TokenStream}; +use crate::tokenstream::{DelimSpan, LazyAttrTokenStream, TokenStream}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::Lrc; @@ -91,7 +91,7 @@ pub struct Path { /// The segments in the path: the things separated by `::`. /// Global paths begin with `kw::PathRoot`. pub segments: Vec, - pub tokens: Option, + pub tokens: Option, } impl PartialEq for Path { @@ -534,7 +534,7 @@ pub struct Block { /// Distinguishes between `unsafe { ... }` and `{ ... }`. pub rules: BlockCheckMode, pub span: Span, - pub tokens: Option, + pub tokens: Option, /// The following *isn't* a parse error, but will cause multiple errors in following stages. /// ```compile_fail /// let x = { @@ -553,7 +553,7 @@ pub struct Pat { pub id: NodeId, pub kind: PatKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } impl Pat { @@ -937,8 +937,8 @@ pub fn has_trailing_semicolon(&self) -> bool { /// a trailing semicolon. /// /// This only modifies the parsed AST struct, not the attached - /// `LazyTokenStream`. The parser is responsible for calling - /// `CreateTokenStream::add_trailing_semi` when there is actually + /// `LazyAttrTokenStream`. The parser is responsible for calling + /// `ToAttrTokenStream::add_trailing_semi` when there is actually /// a semicolon in the tokenstream. pub fn add_trailing_semicolon(mut self) -> Self { self.kind = match self.kind { @@ -984,7 +984,7 @@ pub struct MacCallStmt { pub mac: P, pub style: MacStmtStyle, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug)] @@ -1009,7 +1009,7 @@ pub struct Local { pub kind: LocalKind, pub span: Span, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -1108,7 +1108,7 @@ pub struct Expr { pub kind: ExprKind, pub span: Span, pub attrs: AttrVec, - pub tokens: Option, + pub tokens: Option, } impl Expr { @@ -1967,7 +1967,7 @@ pub struct Ty { pub id: NodeId, pub kind: TyKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } impl Clone for Ty { @@ -2532,7 +2532,7 @@ fn decode(_: &mut D) -> AttrId { pub struct AttrItem { pub path: Path, pub args: MacArgs, - pub tokens: Option, + pub tokens: Option, } /// A list of attributes. @@ -2552,7 +2552,7 @@ pub struct Attribute { #[derive(Clone, Encodable, Decodable, Debug)] pub struct NormalAttr { pub item: AttrItem, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2603,7 +2603,7 @@ pub fn new(generic_params: Vec, path: Path, span: Span) -> Self { pub struct Visibility { pub kind: VisibilityKind, pub span: Span, - pub tokens: Option, + pub tokens: Option, } #[derive(Clone, Encodable, Decodable, Debug)] @@ -2689,7 +2689,7 @@ pub struct Item { /// /// Note that the tokens here do not include the outer attributes, but will /// include inner attributes. - pub tokens: Option, + pub tokens: Option, } impl Item { diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs index 79f5820230e..1b31be07f7a 100644 --- a/compiler/rustc_ast/src/ast_traits.rs +++ b/compiler/rustc_ast/src/ast_traits.rs @@ -4,7 +4,7 @@ use crate::ptr::P; use crate::token::Nonterminal; -use crate::tokenstream::LazyTokenStream; +use crate::tokenstream::LazyAttrTokenStream; use crate::{Arm, Crate, ExprField, FieldDef, GenericParam, Param, PatField, Variant}; use crate::{AssocItem, Expr, ForeignItem, Item, NodeId}; use crate::{AttrItem, AttrKind, Block, Pat, Path, Ty, Visibility}; @@ -124,18 +124,18 @@ fn span(&self) -> Span { /// A trait for AST nodes having (or not having) collected tokens. pub trait HasTokens { - fn tokens(&self) -> Option<&LazyTokenStream>; - fn tokens_mut(&mut self) -> Option<&mut Option>; + fn tokens(&self) -> Option<&LazyAttrTokenStream>; + fn tokens_mut(&mut self) -> Option<&mut Option>; } macro_rules! impl_has_tokens { ($($T:ty),+ $(,)?) => { $( impl HasTokens for $T { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { self.tokens.as_ref() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { Some(&mut self.tokens) } } @@ -147,10 +147,10 @@ macro_rules! impl_has_tokens_none { ($($T:ty),+ $(,)?) => { $( impl HasTokens for $T { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { None } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { None } } @@ -162,25 +162,25 @@ fn tokens_mut(&mut self) -> Option<&mut Option> { impl_has_tokens_none!(Arm, ExprField, FieldDef, GenericParam, Param, PatField, Variant); impl> HasTokens for T { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { self.ast_deref().tokens() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.ast_deref_mut().tokens_mut() } } impl HasTokens for Option { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { self.as_ref().and_then(|inner| inner.tokens()) } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.as_mut().and_then(|inner| inner.tokens_mut()) } } impl HasTokens for StmtKind { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { match self { StmtKind::Local(local) => local.tokens.as_ref(), StmtKind::Item(item) => item.tokens(), @@ -189,7 +189,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> { StmtKind::MacCall(mac) => mac.tokens.as_ref(), } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { match self { StmtKind::Local(local) => Some(&mut local.tokens), StmtKind::Item(item) => item.tokens_mut(), @@ -201,16 +201,16 @@ fn tokens_mut(&mut self) -> Option<&mut Option> { } impl HasTokens for Stmt { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { self.kind.tokens() } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { self.kind.tokens_mut() } } impl HasTokens for Attribute { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { match &self.kind { AttrKind::Normal(normal) => normal.tokens.as_ref(), kind @ AttrKind::DocComment(..) => { @@ -218,7 +218,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> { } } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { Some(match &mut self.kind { AttrKind::Normal(normal) => &mut normal.tokens, kind @ AttrKind::DocComment(..) => { @@ -229,7 +229,7 @@ fn tokens_mut(&mut self) -> Option<&mut Option> { } impl HasTokens for Nonterminal { - fn tokens(&self) -> Option<&LazyTokenStream> { + fn tokens(&self) -> Option<&LazyAttrTokenStream> { match self { Nonterminal::NtItem(item) => item.tokens(), Nonterminal::NtStmt(stmt) => stmt.tokens(), @@ -243,7 +243,7 @@ fn tokens(&self) -> Option<&LazyTokenStream> { Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None, } } - fn tokens_mut(&mut self) -> Option<&mut Option> { + fn tokens_mut(&mut self) -> Option<&mut Option> { match self { Nonterminal::NtItem(item) => item.tokens_mut(), Nonterminal::NtStmt(stmt) => stmt.tokens_mut(), diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs index 6b0dac7c2f0..a40508494cd 100644 --- a/compiler/rustc_ast/src/attr/mod.rs +++ b/compiler/rustc_ast/src/attr/mod.rs @@ -7,9 +7,8 @@ use crate::ast::{Path, PathSegment}; use crate::ptr::P; use crate::token::{self, CommentKind, Delimiter, Token}; -use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree}; use crate::tokenstream::{DelimSpan, Spacing, TokenTree}; -use crate::tokenstream::{LazyTokenStream, TokenStream}; +use crate::tokenstream::{LazyAttrTokenStream, TokenStream}; use crate::util::comments; use rustc_index::bit_set::GrowableBitSet; @@ -296,20 +295,18 @@ pub fn meta_kind(&self) -> Option { } } - pub fn tokens(&self) -> AttrAnnotatedTokenStream { + pub fn tokens(&self) -> TokenStream { match self.kind { AttrKind::Normal(ref normal) => normal .tokens .as_ref() .unwrap_or_else(|| panic!("attribute is missing tokens: {:?}", self)) - .create_token_stream(), - AttrKind::DocComment(comment_kind, data) => AttrAnnotatedTokenStream::from(( - AttrAnnotatedTokenTree::Token(Token::new( - token::DocComment(comment_kind, self.style, data), - self.span, - )), + .to_attr_token_stream() + .to_tokenstream(), + AttrKind::DocComment(comment_kind, data) => TokenStream::new(vec![TokenTree::Token( + Token::new(token::DocComment(comment_kind, self.style, data), self.span), Spacing::Alone, - )), + )]), } } } @@ -356,7 +353,7 @@ pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attri pub fn mk_attr_from_item( item: AttrItem, - tokens: Option, + tokens: Option, style: AttrStyle, span: Span, ) -> Attribute { diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs index e5435e3a3d4..4d3620ee8b0 100644 --- a/compiler/rustc_ast/src/lib.rs +++ b/compiler/rustc_ast/src/lib.rs @@ -15,6 +15,7 @@ #![feature(if_let_guard)] #![cfg_attr(bootstrap, feature(label_break_value))] #![feature(let_chains)] +#![feature(let_else)] #![feature(min_specialization)] #![feature(negative_impls)] #![feature(slice_internals)] diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs index 458d1156ec2..ad68d6e755e 100644 --- a/compiler/rustc_ast/src/mut_visit.rs +++ b/compiler/rustc_ast/src/mut_visit.rs @@ -642,17 +642,17 @@ pub fn noop_visit_meta_item(mi: &mut MetaItem, vis: &mut T) { } // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. -pub fn visit_attr_annotated_tt(tt: &mut AttrAnnotatedTokenTree, vis: &mut T) { +pub fn visit_attr_tt(tt: &mut AttrTokenTree, vis: &mut T) { match tt { - AttrAnnotatedTokenTree::Token(token) => { + AttrTokenTree::Token(token, _) => { visit_token(token, vis); } - AttrAnnotatedTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { + AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { vis.visit_span(open); vis.visit_span(close); - visit_attr_annotated_tts(tts, vis); + visit_attr_tts(tts, vis); } - AttrAnnotatedTokenTree::Attributes(data) => { + AttrTokenTree::Attributes(data) => { for attr in &mut *data.attrs { match &mut attr.kind { AttrKind::Normal(normal) => { @@ -690,27 +690,27 @@ pub fn visit_tts(TokenStream(tts): &mut TokenStream, vis: &mut T) } } -pub fn visit_attr_annotated_tts( - AttrAnnotatedTokenStream(tts): &mut AttrAnnotatedTokenStream, - vis: &mut T, -) { +pub fn visit_attr_tts(AttrTokenStream(tts): &mut AttrTokenStream, vis: &mut T) { if T::VISIT_TOKENS && !tts.is_empty() { let tts = Lrc::make_mut(tts); - visit_vec(tts, |(tree, _is_joint)| visit_attr_annotated_tt(tree, vis)); + visit_vec(tts, |tree| visit_attr_tt(tree, vis)); } } -pub fn visit_lazy_tts_opt_mut(lazy_tts: Option<&mut LazyTokenStream>, vis: &mut T) { +pub fn visit_lazy_tts_opt_mut( + lazy_tts: Option<&mut LazyAttrTokenStream>, + vis: &mut T, +) { if T::VISIT_TOKENS { if let Some(lazy_tts) = lazy_tts { - let mut tts = lazy_tts.create_token_stream(); - visit_attr_annotated_tts(&mut tts, vis); - *lazy_tts = LazyTokenStream::new(tts); + let mut tts = lazy_tts.to_attr_token_stream(); + visit_attr_tts(&mut tts, vis); + *lazy_tts = LazyAttrTokenStream::new(tts); } } } -pub fn visit_lazy_tts(lazy_tts: &mut Option, vis: &mut T) { +pub fn visit_lazy_tts(lazy_tts: &mut Option, vis: &mut T) { visit_lazy_tts_opt_mut(lazy_tts.as_mut(), vis); } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index b6684c0669b..875cd620dfc 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -121,12 +121,12 @@ fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { } } -pub trait CreateTokenStream: sync::Send + sync::Sync { - fn create_token_stream(&self) -> AttrAnnotatedTokenStream; +pub trait ToAttrTokenStream: sync::Send + sync::Sync { + fn to_attr_token_stream(&self) -> AttrTokenStream; } -impl CreateTokenStream for AttrAnnotatedTokenStream { - fn create_token_stream(&self) -> AttrAnnotatedTokenStream { +impl ToAttrTokenStream for AttrTokenStream { + fn to_attr_token_stream(&self) -> AttrTokenStream { self.clone() } } @@ -135,68 +135,68 @@ fn create_token_stream(&self) -> AttrAnnotatedTokenStream { /// of an actual `TokenStream` until it is needed. /// `Box` is here only to reduce the structure size. #[derive(Clone)] -pub struct LazyTokenStream(Lrc>); +pub struct LazyAttrTokenStream(Lrc>); -impl LazyTokenStream { - pub fn new(inner: impl CreateTokenStream + 'static) -> LazyTokenStream { - LazyTokenStream(Lrc::new(Box::new(inner))) +impl LazyAttrTokenStream { + pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream { + LazyAttrTokenStream(Lrc::new(Box::new(inner))) } - pub fn create_token_stream(&self) -> AttrAnnotatedTokenStream { - self.0.create_token_stream() + pub fn to_attr_token_stream(&self) -> AttrTokenStream { + self.0.to_attr_token_stream() } } -impl fmt::Debug for LazyTokenStream { +impl fmt::Debug for LazyAttrTokenStream { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "LazyTokenStream({:?})", self.create_token_stream()) + write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream()) } } -impl Encodable for LazyTokenStream { +impl Encodable for LazyAttrTokenStream { fn encode(&self, s: &mut S) { // Used by AST json printing. - Encodable::encode(&self.create_token_stream(), s); + Encodable::encode(&self.to_attr_token_stream(), s); } } -impl Decodable for LazyTokenStream { +impl Decodable for LazyAttrTokenStream { fn decode(_d: &mut D) -> Self { - panic!("Attempted to decode LazyTokenStream"); + panic!("Attempted to decode LazyAttrTokenStream"); } } -impl HashStable for LazyTokenStream { +impl HashStable for LazyAttrTokenStream { fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { - panic!("Attempted to compute stable hash for LazyTokenStream"); + panic!("Attempted to compute stable hash for LazyAttrTokenStream"); } } -/// A `AttrAnnotatedTokenStream` is similar to a `TokenStream`, but with extra +/// An `AttrTokenStream` is similar to a `TokenStream`, but with extra /// information about the tokens for attribute targets. This is used /// during expansion to perform early cfg-expansion, and to process attributes /// during proc-macro invocations. #[derive(Clone, Debug, Default, Encodable, Decodable)] -pub struct AttrAnnotatedTokenStream(pub Lrc>); +pub struct AttrTokenStream(pub Lrc>); -/// Like `TokenTree`, but for `AttrAnnotatedTokenStream` +/// Like `TokenTree`, but for `AttrTokenStream`. #[derive(Clone, Debug, Encodable, Decodable)] -pub enum AttrAnnotatedTokenTree { - Token(Token), - Delimited(DelimSpan, Delimiter, AttrAnnotatedTokenStream), +pub enum AttrTokenTree { + Token(Token, Spacing), + Delimited(DelimSpan, Delimiter, AttrTokenStream), /// Stores the attributes for an attribute target, /// along with the tokens for that attribute target. /// See `AttributesData` for more information Attributes(AttributesData), } -impl AttrAnnotatedTokenStream { - pub fn new(tokens: Vec<(AttrAnnotatedTokenTree, Spacing)>) -> AttrAnnotatedTokenStream { - AttrAnnotatedTokenStream(Lrc::new(tokens)) +impl AttrTokenStream { + pub fn new(tokens: Vec) -> AttrTokenStream { + AttrTokenStream(Lrc::new(tokens)) } - /// Converts this `AttrAnnotatedTokenStream` to a plain `TokenStream - /// During conversion, `AttrAnnotatedTokenTree::Attributes` get 'flattened' + /// Converts this `AttrTokenStream` to a plain `TokenStream`. + /// During conversion, `AttrTokenTree::Attributes` get 'flattened' /// back to a `TokenStream` of the form `outer_attr attr_target`. /// If there are inner attributes, they are inserted into the proper /// place in the attribute target tokens. @@ -204,31 +204,27 @@ pub fn to_tokenstream(&self) -> TokenStream { let trees: Vec<_> = self .0 .iter() - .flat_map(|tree| match &tree.0 { - AttrAnnotatedTokenTree::Token(inner) => { - smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter() + .flat_map(|tree| match &tree { + AttrTokenTree::Token(inner, spacing) => { + smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter() } - AttrAnnotatedTokenTree::Delimited(span, delim, stream) => { + AttrTokenTree::Delimited(span, delim, stream) => { smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),] .into_iter() } - AttrAnnotatedTokenTree::Attributes(data) => { + AttrTokenTree::Attributes(data) => { let mut outer_attrs = Vec::new(); let mut inner_attrs = Vec::new(); for attr in &data.attrs { match attr.style { - crate::AttrStyle::Outer => { - outer_attrs.push(attr); - } - crate::AttrStyle::Inner => { - inner_attrs.push(attr); - } + crate::AttrStyle::Outer => outer_attrs.push(attr), + crate::AttrStyle::Inner => inner_attrs.push(attr), } } let mut target_tokens: Vec<_> = data .tokens - .create_token_stream() + .to_attr_token_stream() .to_tokenstream() .0 .iter() @@ -239,9 +235,9 @@ pub fn to_tokenstream(&self) -> TokenStream { // Check the last two trees (to account for a trailing semi) for tree in target_tokens.iter_mut().rev().take(2) { if let TokenTree::Delimited(span, delim, delim_tokens) = tree { - // Inner attributes are only supported on extern blocks, functions, impls, - // and modules. All of these have their inner attributes placed at - // the beginning of the rightmost outermost braced group: + // Inner attributes are only supported on extern blocks, functions, + // impls, and modules. All of these have their inner attributes + // placed at the beginning of the rightmost outermost braced group: // e.g. fn foo() { #![my_attr} } // // Therefore, we can insert them back into the right location @@ -255,7 +251,7 @@ pub fn to_tokenstream(&self) -> TokenStream { let mut builder = TokenStreamBuilder::new(); for inner_attr in inner_attrs { - builder.push(inner_attr.tokens().to_tokenstream()); + builder.push(inner_attr.tokens()); } builder.push(delim_tokens.clone()); *tree = TokenTree::Delimited(*span, *delim, builder.build()); @@ -273,7 +269,7 @@ pub fn to_tokenstream(&self) -> TokenStream { let mut flat: SmallVec<[_; 1]> = SmallVec::new(); for attr in outer_attrs { // FIXME: Make this more efficient - flat.extend(attr.tokens().to_tokenstream().0.clone().iter().cloned()); + flat.extend(attr.tokens().0.clone().iter().cloned()); } flat.extend(target_tokens); flat.into_iter() @@ -300,7 +296,7 @@ pub struct AttributesData { pub attrs: AttrVec, /// The underlying tokens for the attribute target that `attrs` /// are applied to - pub tokens: LazyTokenStream, + pub tokens: LazyAttrTokenStream, } /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. @@ -363,12 +359,6 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> { } } -impl From<(AttrAnnotatedTokenTree, Spacing)> for AttrAnnotatedTokenStream { - fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedTokenStream { - AttrAnnotatedTokenStream::new(vec![(tree, spacing)]) - } -} - impl iter::FromIterator for TokenStream { fn from_iter>(iter: I) -> Self { TokenStream::new(iter.into_iter().collect::>()) @@ -420,22 +410,6 @@ pub fn map_enumerated TokenTree>(self, mut f: F) TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect())) } - fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option { - let tokens = node.tokens()?; - let attrs = node.attrs(); - let attr_annotated = if attrs.is_empty() { - tokens.create_token_stream() - } else { - let attr_data = - AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; - AttrAnnotatedTokenStream::new(vec![( - AttrAnnotatedTokenTree::Attributes(attr_data), - Spacing::Alone, - )]) - }; - Some(attr_annotated.to_tokenstream()) - } - // Create a token stream containing a single token with alone spacing. pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream { TokenStream::new(vec![TokenTree::token_alone(kind, span)]) @@ -452,8 +426,18 @@ pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenSt } pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { - TokenStream::opt_from_ast(node) - .unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node)) + let Some(tokens) = node.tokens() else { + panic!("missing tokens for node at {:?}: {:?}", node.span(), node); + }; + let attrs = node.attrs(); + let attr_stream = if attrs.is_empty() { + tokens.to_attr_token_stream() + } else { + let attr_data = + AttributesData { attrs: attrs.iter().cloned().collect(), tokens: tokens.clone() }; + AttrTokenStream::new(vec![AttrTokenTree::Attributes(attr_data)]) + }; + attr_stream.to_tokenstream() } pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index 89b2c329236..e673dff0dea 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -188,14 +188,14 @@ fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option(&self, mut node: T) -> Option { fn try_configure_tokens(&self, node: &mut T) { if self.config_tokens { if let Some(Some(tokens)) = node.tokens_mut() { - let attr_annotated_tokens = tokens.create_token_stream(); - *tokens = LazyTokenStream::new(self.configure_tokens(&attr_annotated_tokens)); + let attr_stream = tokens.to_attr_token_stream(); + *tokens = LazyAttrTokenStream::new(self.configure_tokens(&attr_stream)); } } } @@ -270,16 +270,16 @@ fn configure_krate_attrs(&self, mut attrs: ast::AttrVec) -> Option if self.in_cfg(&attrs) { Some(attrs) } else { None } } - /// Performs cfg-expansion on `stream`, producing a new `AttrAnnotatedTokenStream`. + /// Performs cfg-expansion on `stream`, producing a new `AttrTokenStream`. /// This is only used during the invocation of `derive` proc-macros, /// which require that we cfg-expand their entire input. /// Normal cfg-expansion operates on parsed AST nodes via the `configure` method - fn configure_tokens(&self, stream: &AttrAnnotatedTokenStream) -> AttrAnnotatedTokenStream { - fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool { - stream.0.iter().all(|(tree, _spacing)| match tree { - AttrAnnotatedTokenTree::Attributes(_) => false, - AttrAnnotatedTokenTree::Token(_) => true, - AttrAnnotatedTokenTree::Delimited(_, _, inner) => can_skip(inner), + fn configure_tokens(&self, stream: &AttrTokenStream) -> AttrTokenStream { + fn can_skip(stream: &AttrTokenStream) -> bool { + stream.0.iter().all(|tree| match tree { + AttrTokenTree::Attributes(_) => false, + AttrTokenTree::Token(..) => true, + AttrTokenTree::Delimited(_, _, inner) => can_skip(inner), }) } @@ -290,36 +290,36 @@ fn can_skip(stream: &AttrAnnotatedTokenStream) -> bool { let trees: Vec<_> = stream .0 .iter() - .flat_map(|(tree, spacing)| match tree.clone() { - AttrAnnotatedTokenTree::Attributes(mut data) => { + .flat_map(|tree| match tree.clone() { + AttrTokenTree::Attributes(mut data) => { data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(attr)); if self.in_cfg(&data.attrs) { - data.tokens = LazyTokenStream::new( - self.configure_tokens(&data.tokens.create_token_stream()), + data.tokens = LazyAttrTokenStream::new( + self.configure_tokens(&data.tokens.to_attr_token_stream()), ); - Some((AttrAnnotatedTokenTree::Attributes(data), *spacing)).into_iter() + Some(AttrTokenTree::Attributes(data)).into_iter() } else { None.into_iter() } } - AttrAnnotatedTokenTree::Delimited(sp, delim, mut inner) => { + AttrTokenTree::Delimited(sp, delim, mut inner) => { inner = self.configure_tokens(&inner); - Some((AttrAnnotatedTokenTree::Delimited(sp, delim, inner), *spacing)) + Some(AttrTokenTree::Delimited(sp, delim, inner)) .into_iter() } - AttrAnnotatedTokenTree::Token(ref token) if let TokenKind::Interpolated(ref nt) = token.kind => { + AttrTokenTree::Token(ref token, _) if let TokenKind::Interpolated(ref nt) = token.kind => { panic!( "Nonterminal should have been flattened at {:?}: {:?}", token.span, nt ); } - AttrAnnotatedTokenTree::Token(token) => { - Some((AttrAnnotatedTokenTree::Token(token), *spacing)).into_iter() + AttrTokenTree::Token(token, spacing) => { + Some(AttrTokenTree::Token(token, spacing)).into_iter() } }) .collect(); - AttrAnnotatedTokenStream::new(trees) + AttrTokenStream::new(trees) } /// Parse and expand all `cfg_attr` attributes into a list of attributes @@ -388,7 +388,7 @@ fn expand_cfg_attr_item( attr: &Attribute, (item, item_span): (ast::AttrItem, Span), ) -> Attribute { - let orig_tokens = attr.tokens().to_tokenstream(); + let orig_tokens = attr.tokens(); // We are taking an attribute of the form `#[cfg_attr(pred, attr)]` // and producing an attribute of the form `#[attr]`. We @@ -404,26 +404,26 @@ fn expand_cfg_attr_item( }; let pound_span = pound_token.span; - let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)]; + let mut trees = vec![AttrTokenTree::Token(pound_token, Spacing::Alone)]; if attr.style == AttrStyle::Inner { // For inner attributes, we do the same thing for the `!` in `#![some_attr]` let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else { panic!("Bad tokens for attribute {:?}", attr); }; - trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone)); + trees.push(AttrTokenTree::Token(bang_token, Spacing::Alone)); } // We don't really have a good span to use for the synthesized `[]` // in `#[attr]`, so just use the span of the `#` token. - let bracket_group = AttrAnnotatedTokenTree::Delimited( + let bracket_group = AttrTokenTree::Delimited( DelimSpan::from_single(pound_span), Delimiter::Bracket, item.tokens .as_ref() .unwrap_or_else(|| panic!("Missing tokens for {:?}", item)) - .create_token_stream(), + .to_attr_token_stream(), ); - trees.push((bracket_group, Spacing::Alone)); - let tokens = Some(LazyTokenStream::new(AttrAnnotatedTokenStream::new(trees))); + trees.push(bracket_group); + let tokens = Some(LazyAttrTokenStream::new(AttrTokenStream::new(trees))); let attr = attr::mk_attr_from_item(item, tokens, attr.style, item_span); if attr.has_name(sym::crate_type) { self.sess.parse_sess.buffer_lint( diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 77a6bde1c16..a37f828eafb 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -301,9 +301,9 @@ pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> { if let Some(attr) = attr { let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap(); // If we are currently capturing tokens, mark the location of this inner attribute. - // If capturing ends up creating a `LazyTokenStream`, we will include + // If capturing ends up creating a `LazyAttrTokenStream`, we will include // this replace range with it, removing the inner attribute from the final - // `AttrAnnotatedTokenStream`. Inner attributes are stored in the parsed AST note. + // `AttrTokenStream`. Inner attributes are stored in the parsed AST note. // During macro expansion, they are selectively inserted back into the // token stream (the first inner attribute is removed each time we invoke the // corresponding macro). diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 2e58605cf19..5fdafd187c6 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -1,7 +1,7 @@ use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken}; use rustc_ast::token::{self, Delimiter, Token, TokenKind}; -use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttributesData, CreateTokenStream}; -use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, Spacing}; +use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream}; +use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing}; use rustc_ast::{self as ast}; use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens}; use rustc_errors::PResult; @@ -88,7 +88,7 @@ fn has_cfg_or_cfg_attr(attrs: &[Attribute]) -> bool { // This also makes `Parser` very cheap to clone, since // there is no intermediate collection buffer to clone. #[derive(Clone)] -struct LazyTokenStreamImpl { +struct LazyAttrTokenStreamImpl { start_token: (Token, Spacing), cursor_snapshot: TokenCursor, num_calls: usize, @@ -97,10 +97,10 @@ struct LazyTokenStreamImpl { } #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] -rustc_data_structures::static_assert_size!(LazyTokenStreamImpl, 144); +rustc_data_structures::static_assert_size!(LazyAttrTokenStreamImpl, 144); -impl CreateTokenStream for LazyTokenStreamImpl { - fn create_token_stream(&self) -> AttrAnnotatedTokenStream { +impl ToAttrTokenStream for LazyAttrTokenStreamImpl { + fn to_attr_token_stream(&self) -> AttrTokenStream { // The token produced by the final call to `{,inlined_}next` was not // actually consumed by the callback. The combination of chaining the // initial token and using `take` produces the desired result - we @@ -179,7 +179,7 @@ fn create_token_stream(&self) -> AttrAnnotatedTokenStream { impl<'a> Parser<'a> { /// Records all tokens consumed by the provided callback, /// including the current token. These tokens are collected - /// into a `LazyTokenStream`, and returned along with the result + /// into a `LazyAttrTokenStream`, and returned along with the result /// of the callback. /// /// Note: If your callback consumes an opening delimiter @@ -297,8 +297,8 @@ pub fn collect_tokens_trailing_token( // If we 'broke' the last token (e.g. breaking a '>>' token to two '>' tokens), // then extend the range of captured tokens to include it, since the parser - // was not actually bumped past it. When the `LazyTokenStream` gets converted - // into an `AttrAnnotatedTokenStream`, we will create the proper token. + // was not actually bumped past it. When the `LazyAttrTokenStream` gets converted + // into an `AttrTokenStream`, we will create the proper token. if self.token_cursor.break_last_token { assert_eq!( trailing, @@ -316,8 +316,8 @@ pub fn collect_tokens_trailing_token( Box::new([]) } else { // Grab any replace ranges that occur *inside* the current AST node. - // We will perform the actual replacement when we convert the `LazyTokenStream` - // to an `AttrAnnotatedTokenStream` + // We will perform the actual replacement when we convert the `LazyAttrTokenStream` + // to an `AttrTokenStream`. let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap(); self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end] .iter() @@ -329,7 +329,7 @@ pub fn collect_tokens_trailing_token( .collect() }; - let tokens = LazyTokenStream::new(LazyTokenStreamImpl { + let tokens = LazyAttrTokenStream::new(LazyAttrTokenStreamImpl { start_token, num_calls, cursor_snapshot, @@ -392,12 +392,12 @@ pub fn collect_tokens_trailing_token( fn make_token_stream( mut iter: impl Iterator, break_last_token: bool, -) -> AttrAnnotatedTokenStream { +) -> AttrTokenStream { #[derive(Debug)] struct FrameData { // This is `None` for the first frame, `Some` for all others. open_delim_sp: Option<(Delimiter, Span)>, - inner: Vec<(AttrAnnotatedTokenTree, Spacing)>, + inner: Vec, } let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; let mut token_and_spacing = iter.next(); @@ -418,48 +418,47 @@ struct FrameData { open_delim, span ); let dspan = DelimSpan::from_pair(open_sp, span); - let stream = AttrAnnotatedTokenStream::new(frame_data.inner); - let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream); + let stream = AttrTokenStream::new(frame_data.inner); + let delimited = AttrTokenTree::Delimited(dspan, delim, stream); stack .last_mut() .unwrap_or_else(|| { panic!("Bottom token frame is missing for token: {:?}", token) }) .inner - .push((delimited, Spacing::Alone)); + .push(delimited); } FlatToken::Token(token) => stack .last_mut() .expect("Bottom token frame is missing!") .inner - .push((AttrAnnotatedTokenTree::Token(token), spacing)), + .push(AttrTokenTree::Token(token, spacing)), FlatToken::AttrTarget(data) => stack .last_mut() .expect("Bottom token frame is missing!") .inner - .push((AttrAnnotatedTokenTree::Attributes(data), spacing)), + .push(AttrTokenTree::Attributes(data)), FlatToken::Empty => {} } token_and_spacing = iter.next(); } let mut final_buf = stack.pop().expect("Missing final buf!"); if break_last_token { - let (last_token, spacing) = final_buf.inner.pop().unwrap(); - if let AttrAnnotatedTokenTree::Token(last_token) = last_token { + let last_token = final_buf.inner.pop().unwrap(); + if let AttrTokenTree::Token(last_token, spacing) = last_token { let unglued_first = last_token.kind.break_two_token_op().unwrap().0; // An 'unglued' token is always two ASCII characters let mut first_span = last_token.span.shrink_to_lo(); first_span = first_span.with_hi(first_span.lo() + rustc_span::BytePos(1)); - final_buf.inner.push(( - AttrAnnotatedTokenTree::Token(Token::new(unglued_first, first_span)), - spacing, - )); + final_buf + .inner + .push(AttrTokenTree::Token(Token::new(unglued_first, first_span), spacing)); } else { panic!("Unexpected last token {:?}", last_token) } } assert!(stack.is_empty(), "Stack should be empty: final_buf={:?} stack={:?}", final_buf, stack); - AttrAnnotatedTokenStream::new(final_buf.inner) + AttrTokenStream::new(final_buf.inner) } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 5c8f374255c..4cb198561e0 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -170,7 +170,7 @@ pub struct ClosureSpans { /// attribute, we parse a nested AST node that has `#[cfg]` or `#[cfg_attr]` /// In this case, we use a `ReplaceRange` to replace the entire inner AST node /// with `FlatToken::AttrTarget`, allowing us to perform eager cfg-expansion -/// on an `AttrAnnotatedTokenStream` +/// on an `AttrTokenStream`. /// /// 2. When we parse an inner attribute while collecting tokens. We /// remove inner attributes from the token stream entirely, and @@ -183,7 +183,7 @@ pub struct ClosureSpans { /// Controls how we capture tokens. Capturing can be expensive, /// so we try to avoid performing capturing in cases where -/// we will never need an `AttrAnnotatedTokenStream` +/// we will never need an `AttrTokenStream`. #[derive(Copy, Clone)] pub enum Capturing { /// We aren't performing any capturing - this is the default mode. @@ -237,7 +237,7 @@ struct TokenCursor { // the trailing `>>` token. The `break_last_token` // field is used to track this token - it gets // appended to the captured stream when - // we evaluate a `LazyTokenStream` + // we evaluate a `LazyAttrTokenStream`. break_last_token: bool, } @@ -1464,11 +1464,11 @@ pub fn emit_unclosed_delims(unclosed_delims: &mut Vec, sess: &Pa } } -/// A helper struct used when building an `AttrAnnotatedTokenStream` from -/// a `LazyTokenStream`. Both delimiter and non-delimited tokens +/// A helper struct used when building an `AttrTokenStream` from +/// a `LazyAttrTokenStream`. Both delimiter and non-delimited tokens /// are stored as `FlatToken::Token`. A vector of `FlatToken`s -/// is then 'parsed' to build up an `AttrAnnotatedTokenStream` with nested -/// `AttrAnnotatedTokenTree::Delimited` tokens +/// is then 'parsed' to build up an `AttrTokenStream` with nested +/// `AttrTokenTree::Delimited` tokens. #[derive(Debug, Clone)] pub enum FlatToken { /// A token - this holds both delimiter (e.g. '{' and '}') @@ -1476,11 +1476,11 @@ pub enum FlatToken { Token(Token), /// Holds the `AttributesData` for an AST node. The /// `AttributesData` is inserted directly into the - /// constructed `AttrAnnotatedTokenStream` as - /// an `AttrAnnotatedTokenTree::Attributes` + /// constructed `AttrTokenStream` as + /// an `AttrTokenTree::Attributes`. AttrTarget(AttributesData), /// A special 'empty' token that is ignored during the conversion - /// to an `AttrAnnotatedTokenStream`. This is used to simplify the + /// to an `AttrTokenStream`. This is used to simplify the /// handling of replace ranges. Empty, }