mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Auto merge of #99887 - nnethercote:rm-TreeAndSpacing, r=petrochenkov

Browse Source 
Remove `TreeAndSpacing`.

A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.

This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.

The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`

These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.

This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.

These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.

r? `@petrochenkov`
This commit is contained in:
bors 2022-07-30 14:50:05 +00:00
commit 1202bbaf48
23 changed files with 317 additions and 307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
compiler/rustc_ast/src/attr/mod.rs
View File

@ -8,7 +8,7 @@
use crate::ptr::P; use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token}; use crate::token::{self, CommentKind, Delimiter, Token};
use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree}; use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing}; use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
use crate::tokenstream::{LazyTokenStream, TokenStream}; use crate::tokenstream::{LazyTokenStream, TokenStream};
use crate::util::comments; use crate::util::comments;
@ -388,7 +388,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
} }
impl MetaItem { impl MetaItem {
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> { fn token_trees(&self) -> Vec<TokenTree> {
let mut idents = vec![]; let mut idents = vec![];
let mut last_pos = BytePos(0_u32); let mut last_pos = BytePos(0_u32);
for (i, segment) in self.path.segments.iter().enumerate() { for (i, segment) in self.path.segments.iter().enumerate() {
@ -396,12 +396,12 @@ fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
if !is_first { if !is_first {
let mod_sep_span = let mod_sep_span =
Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt(), None); Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt(), None);
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into()); idents.push(TokenTree::token_alone(token::ModSep, mod_sep_span));
} }
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident), Spacing::Alone));
last_pos = segment.ident.span.hi(); last_pos = segment.ident.span.hi();
} }
idents.extend(self.kind.token_trees_and_spacings(self.span)); idents.extend(self.kind.token_trees(self.span));
idents idents
} }
@ -411,12 +411,13 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
{ {
// FIXME: Share code with `parse_path`. // FIXME: Share code with `parse_path`.
let path = match tokens.next().map(TokenTree::uninterpolate) { let path = match tokens.next().map(TokenTree::uninterpolate) {
Some(TokenTree::Token(Token { Some(TokenTree::Token(
kind: kind @ (token::Ident(..) | token::ModSep), Token { kind: kind @ (token::Ident(..) | token::ModSep), span },
span, _,
})) => 'arm: { )) => 'arm: {
let mut segments = if let token::Ident(name, _) = kind { let mut segments = if let token::Ident(name, _) = kind {
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
tokens.peek()
{ {
tokens.next(); tokens.next();
vec![PathSegment::from_ident(Ident::new(name, span))] vec![PathSegment::from_ident(Ident::new(name, span))]
@ -427,14 +428,15 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
vec![PathSegment::path_root(span)] vec![PathSegment::path_root(span)]
}; };
loop { loop {
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
tokens.next().map(TokenTree::uninterpolate) tokens.next().map(TokenTree::uninterpolate)
{ {
segments.push(PathSegment::from_ident(Ident::new(name, span))); segments.push(PathSegment::from_ident(Ident::new(name, span)));
} else { } else {
return None; return None;
} }
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
tokens.peek()
{ {
tokens.next(); tokens.next();
} else { } else {
@ -444,7 +446,7 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
let span = span.with_hi(segments.last().unwrap().ident.span.hi()); let span = span.with_hi(segments.last().unwrap().ident.span.hi());
Path { span, segments, tokens: None } Path { span, segments, tokens: None }
} }
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt { Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match *nt {
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span), token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
token::Nonterminal::NtPath(ref path) => (**path).clone(), token::Nonterminal::NtPath(ref path) => (**path).clone(),
_ => return None, _ => return None,
@ -491,9 +493,9 @@ pub fn mac_args(&self, span: Span) -> MacArgs {
let mut tts = Vec::new(); let mut tts = Vec::new();
for (i, item) in list.iter().enumerate() { for (i, item) in list.iter().enumerate() {
if i > 0 { if i > 0 {
tts.push(TokenTree::token(token::Comma, span).into()); tts.push(TokenTree::token_alone(token::Comma, span));
} }
tts.extend(item.token_trees_and_spacings()) tts.extend(item.token_trees())
} }
MacArgs::Delimited( MacArgs::Delimited(
DelimSpan::from_single(span), DelimSpan::from_single(span),
@ -504,31 +506,28 @@ pub fn mac_args(&self, span: Span) -> MacArgs {
} }
} }
fn token_trees_and_spacings(&self, span: Span) -> Vec<TreeAndSpacing> { fn token_trees(&self, span: Span) -> Vec<TokenTree> {
match *self { match *self {
MetaItemKind::Word => vec![], MetaItemKind::Word => vec![],
MetaItemKind::NameValue(ref lit) => { MetaItemKind::NameValue(ref lit) => {
vec![ vec![
TokenTree::token(token::Eq, span).into(), TokenTree::token_alone(token::Eq, span),
TokenTree::Token(lit.to_token()).into(), TokenTree::Token(lit.to_token(), Spacing::Alone),
] ]
} }
MetaItemKind::List(ref list) => { MetaItemKind::List(ref list) => {
let mut tokens = Vec::new(); let mut tokens = Vec::new();
for (i, item) in list.iter().enumerate() { for (i, item) in list.iter().enumerate() {
if i > 0 { if i > 0 {
tokens.push(TokenTree::token(token::Comma, span).into()); tokens.push(TokenTree::token_alone(token::Comma, span));
} }
tokens.extend(item.token_trees_and_spacings()) tokens.extend(item.token_trees())
} }
vec![ vec![TokenTree::Delimited(
TokenTree::Delimited( DelimSpan::from_single(span),
DelimSpan::from_single(span), Delimiter::Parenthesis,
Delimiter::Parenthesis, TokenStream::new(tokens),
TokenStream::new(tokens), )]
)
.into(),
]
} }
} }
} }
@ -540,7 +539,7 @@ fn list_from_tokens(tokens: TokenStream) -> Option<MetaItemKind> {
let item = NestedMetaItem::from_tokens(&mut tokens)?; let item = NestedMetaItem::from_tokens(&mut tokens)?;
result.push(item); result.push(item);
match tokens.next() { match tokens.next() {
None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {} None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
_ => return None, _ => return None,
} }
} }
@ -554,7 +553,7 @@ fn name_value_from_tokens(
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => { Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees()) MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
} }
Some(TokenTree::Token(token)) => { Some(TokenTree::Token(token, _)) => {
Lit::from_token(&token).ok().map(MetaItemKind::NameValue) Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
} }
_ => None, _ => None,
@ -586,7 +585,7 @@ fn from_tokens(
MetaItemKind::list_from_tokens(inner_tokens) MetaItemKind::list_from_tokens(inner_tokens)
} }
Some(TokenTree::Delimited(..)) => None, Some(TokenTree::Delimited(..)) => None,
Some(TokenTree::Token(Token { kind: token::Eq, .. })) => { Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
tokens.next(); tokens.next();
MetaItemKind::name_value_from_tokens(tokens) MetaItemKind::name_value_from_tokens(tokens)
} }
@ -603,10 +602,12 @@ pub fn span(&self) -> Span {
} }
} }
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> { fn token_trees(&self) -> Vec<TokenTree> {
match *self { match *self {
NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(), NestedMetaItem::MetaItem(ref item) => item.token_trees(),
NestedMetaItem::Literal(ref lit) => vec![TokenTree::Token(lit.to_token()).into()], NestedMetaItem::Literal(ref lit) => {
vec![TokenTree::Token(lit.to_token(), Spacing::Alone)]
}
} }
} }
@ -615,7 +616,7 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
I: Iterator<Item = TokenTree>, I: Iterator<Item = TokenTree>,
{ {
match tokens.peek() { match tokens.peek() {
Some(TokenTree::Token(token)) Some(TokenTree::Token(token, _))
if let Ok(lit) = Lit::from_token(token) => if let Ok(lit) = Lit::from_token(token) =>
{ {
tokens.next(); tokens.next();

4
compiler/rustc_ast/src/mut_visit.rs
View File

@ -675,7 +675,7 @@ pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, v
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`. // No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) { pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
match tt { match tt {
TokenTree::Token(token) => { TokenTree::Token(token, _) => {
visit_token(token, vis); visit_token(token, vis);
} }
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => { TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
@ -690,7 +690,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) { pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
if T::VISIT_TOKENS && !tts.is_empty() { if T::VISIT_TOKENS && !tts.is_empty() {
let tts = Lrc::make_mut(tts); let tts = Lrc::make_mut(tts);
visit_vec(tts, |(tree, _is_joint)| visit_tt(tree, vis)); visit_vec(tts, |tree| visit_tt(tree, vis));
} }
} }

152
compiler/rustc_ast/src/tokenstream.rs
View File

@ -42,11 +42,15 @@
#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] #[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum TokenTree { pub enum TokenTree {
/// A single token. /// A single token.
Token(Token), Token(Token, Spacing),
/// A delimited sequence of token trees. /// A delimited sequence of token trees.
Delimited(DelimSpan, Delimiter, TokenStream), Delimited(DelimSpan, Delimiter, TokenStream),
} }
// This type is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(TokenTree, 32);
// Ensure all fields of `TokenTree` is `Send` and `Sync`. // Ensure all fields of `TokenTree` is `Send` and `Sync`.
#[cfg(parallel_compiler)] #[cfg(parallel_compiler)]
fn _dummy() fn _dummy()
@ -62,7 +66,7 @@ impl TokenTree {
/// Checks if this `TokenTree` is equal to the other, regardless of span information. /// Checks if this `TokenTree` is equal to the other, regardless of span information.
pub fn eq_unspanned(&self, other: &TokenTree) -> bool { pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
match (self, other) { match (self, other) {
(TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind, (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => { (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
delim == delim2 && tts.eq_unspanned(&tts2) delim == delim2 && tts.eq_unspanned(&tts2)
} }
@ -73,7 +77,7 @@ pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
/// Retrieves the `TokenTree`'s span. /// Retrieves the `TokenTree`'s span.
pub fn span(&self) -> Span { pub fn span(&self) -> Span {
match self { match self {
TokenTree::Token(token) => token.span, TokenTree::Token(token, _) => token.span,
TokenTree::Delimited(sp, ..) => sp.entire(), TokenTree::Delimited(sp, ..) => sp.entire(),
} }
} }
@ -81,18 +85,26 @@ pub fn span(&self) -> Span {
/// Modify the `TokenTree`'s span in-place. /// Modify the `TokenTree`'s span in-place.
pub fn set_span(&mut self, span: Span) { pub fn set_span(&mut self, span: Span) {
match self { match self {
TokenTree::Token(token) => token.span = span, TokenTree::Token(token, _) => token.span = span,
TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span), TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span),
} }
} }
pub fn token(kind: TokenKind, span: Span) -> TokenTree { // Create a `TokenTree::Token` with alone spacing.
TokenTree::Token(Token::new(kind, span)) pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
}
// Create a `TokenTree::Token` with joint spacing.
pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
} }
pub fn uninterpolate(self) -> TokenTree { pub fn uninterpolate(self) -> TokenTree {
match self { match self {
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()), TokenTree::Token(token, spacing) => {
TokenTree::Token(token.uninterpolate().into_owned(), spacing)
}
tt => tt, tt => tt,
} }
} }
@ -194,13 +206,12 @@ pub fn to_tokenstream(&self) -> TokenStream {
.iter() .iter()
.flat_map(|tree| match &tree.0 { .flat_map(|tree| match &tree.0 {
AttrAnnotatedTokenTree::Token(inner) => { AttrAnnotatedTokenTree::Token(inner) => {
smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter() smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter()
}
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
.into_iter()
} }
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![(
TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),
tree.1,
)]
.into_iter(),
AttrAnnotatedTokenTree::Attributes(data) => { AttrAnnotatedTokenTree::Attributes(data) => {
let mut outer_attrs = Vec::new(); let mut outer_attrs = Vec::new();
let mut inner_attrs = Vec::new(); let mut inner_attrs = Vec::new();
@ -226,7 +237,7 @@ pub fn to_tokenstream(&self) -> TokenStream {
if !inner_attrs.is_empty() { if !inner_attrs.is_empty() {
let mut found = false; let mut found = false;
// Check the last two trees (to account for a trailing semi) // Check the last two trees (to account for a trailing semi)
for (tree, _) in target_tokens.iter_mut().rev().take(2) { for tree in target_tokens.iter_mut().rev().take(2) {
if let TokenTree::Delimited(span, delim, delim_tokens) = tree { if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
// Inner attributes are only supported on extern blocks, functions, impls, // Inner attributes are only supported on extern blocks, functions, impls,
// and modules. All of these have their inner attributes placed at // and modules. All of these have their inner attributes placed at
@ -299,15 +310,13 @@ pub struct AttributesData {
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for /// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
/// backwards compatibility. /// backwards compatibility.
#[derive(Clone, Debug, Default, Encodable, Decodable)] #[derive(Clone, Debug, Default, Encodable, Decodable)]
pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>); pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
pub type TreeAndSpacing = (TokenTree, Spacing);
// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger. // `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(TokenStream, 8); rustc_data_structures::static_assert_size!(TokenStream, 8);
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)] #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
pub enum Spacing { pub enum Spacing {
Alone, Alone,
Joint, Joint,
@ -323,10 +332,10 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
while let Some((pos, ts)) = iter.next() { while let Some((pos, ts)) = iter.next() {
if let Some((_, next)) = iter.peek() { if let Some((_, next)) = iter.peek() {
let sp = match (&ts, &next) { let sp = match (&ts, &next) {
(_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue, (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
( (
(TokenTree::Token(token_left), Spacing::Alone), TokenTree::Token(token_left, Spacing::Alone),
(TokenTree::Token(token_right), _), TokenTree::Token(token_right, _),
) if ((token_left.is_ident() && !token_left.is_reserved_ident()) ) if ((token_left.is_ident() && !token_left.is_reserved_ident())
|| token_left.is_lit()) || token_left.is_lit())
&& ((token_right.is_ident() && !token_right.is_reserved_ident()) && ((token_right.is_ident() && !token_right.is_reserved_ident())
@ -334,11 +343,11 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
{ {
token_left.span token_left.span
} }
((TokenTree::Delimited(sp, ..), Spacing::Alone), _) => sp.entire(), (TokenTree::Delimited(sp, ..), _) => sp.entire(),
_ => continue, _ => continue,
}; };
let sp = sp.shrink_to_hi(); let sp = sp.shrink_to_hi();
let comma = (TokenTree::token(token::Comma, sp), Spacing::Alone); let comma = TokenTree::token_alone(token::Comma, sp);
suggestion = Some((pos, comma, sp)); suggestion = Some((pos, comma, sp));
} }
} }
@ -360,21 +369,9 @@ fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedToke
} }
} }
impl From<TokenTree> for TokenStream {
fn from(tree: TokenTree) -> TokenStream {
TokenStream::new(vec![(tree, Spacing::Alone)])
}
}
impl From<TokenTree> for TreeAndSpacing {
fn from(tree: TokenTree) -> TreeAndSpacing {
(tree, Spacing::Alone)
}
}
impl iter::FromIterator<TokenTree> for TokenStream { impl iter::FromIterator<TokenTree> for TokenStream {
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self { fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndSpacing>>()) TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
} }
} }
@ -387,7 +384,7 @@ fn eq(&self, other: &TokenStream) -> bool {
} }
impl TokenStream { impl TokenStream {
pub fn new(streams: Vec<TreeAndSpacing>) -> TokenStream { pub fn new(streams: Vec<TokenTree>) -> TokenStream {
TokenStream(Lrc::new(streams)) TokenStream(Lrc::new(streams))
} }
@ -420,13 +417,7 @@ pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
} }
pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream { pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
TokenStream(Lrc::new( TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect()))
self.0
.iter()
.enumerate()
.map(|(i, (tree, is_joint))| (f(i, tree), *is_joint))
.collect(),
))
} }
fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> { fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> {
@ -444,6 +435,21 @@ fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> {
Some(attr_annotated.to_tokenstream()) Some(attr_annotated.to_tokenstream())
} }
// Create a token stream containing a single token with alone spacing.
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
}
// Create a token stream containing a single token with joint spacing.
pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
TokenStream::new(vec![TokenTree::token_joint(kind, span)])
}
// Create a token stream containing a single `Delimited`.
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
}
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream { pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
TokenStream::opt_from_ast(node) TokenStream::opt_from_ast(node)
.unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node)) .unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node))
@ -452,16 +458,16 @@ pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> To
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
match nt { match nt {
Nonterminal::NtIdent(ident, is_raw) => { Nonterminal::NtIdent(ident, is_raw) => {
TokenTree::token(token::Ident(ident.name, *is_raw), ident.span).into() TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
} }
Nonterminal::NtLifetime(ident) => { Nonterminal::NtLifetime(ident) => {
TokenTree::token(token::Lifetime(ident.name), ident.span).into() TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
} }
Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtItem(item) => TokenStream::from_ast(item),
Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
// FIXME: Properly collect tokens for empty statements. // FIXME: Properly collect tokens for empty statements.
TokenTree::token(token::Semi, stmt.span).into() TokenStream::token_alone(token::Semi, stmt.span)
} }
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt), Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
Nonterminal::NtPat(pat) => TokenStream::from_ast(pat), Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
@ -473,23 +479,23 @@ pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
} }
} }
fn flatten_token(token: &Token) -> TokenTree { fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
match &token.kind { match &token.kind {
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => { token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => {
TokenTree::token(token::Ident(ident.name, is_raw), ident.span) TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
} }
token::Interpolated(nt) => TokenTree::Delimited( token::Interpolated(nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span), DelimSpan::from_single(token.span),
Delimiter::Invisible, Delimiter::Invisible,
TokenStream::from_nonterminal_ast(&nt).flattened(), TokenStream::from_nonterminal_ast(&nt).flattened(),
), ),
_ => TokenTree::Token(token.clone()), _ => TokenTree::Token(token.clone(), spacing),
} }
} }
fn flatten_token_tree(tree: &TokenTree) -> TokenTree { fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
match tree { match tree {
TokenTree::Token(token) => TokenStream::flatten_token(token), TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
TokenTree::Delimited(span, delim, tts) => { TokenTree::Delimited(span, delim, tts) => {
TokenTree::Delimited(*span, *delim, tts.flattened()) TokenTree::Delimited(*span, *delim, tts.flattened())
} }
@ -500,7 +506,7 @@ fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
pub fn flattened(&self) -> TokenStream { pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool { fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree { stream.trees().all(|tree| match tree {
TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)), TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
TokenTree::Delimited(_, _, inner) => can_skip(inner), TokenTree::Delimited(_, _, inner) => can_skip(inner),
}) })
} }
@ -522,8 +528,8 @@ pub fn new() -> TokenStreamBuilder {
TokenStreamBuilder(SmallVec::new()) TokenStreamBuilder(SmallVec::new())
} }
pub fn push<T: Into<TokenStream>>(&mut self, stream: T) { pub fn push(&mut self, stream: TokenStream) {
self.0.push(stream.into()); self.0.push(stream);
} }
pub fn build(self) -> TokenStream { pub fn build(self) -> TokenStream {
@ -564,14 +570,14 @@ pub fn build(self) -> TokenStream {
// `stream` is not empty and the first tree within it is a // `stream` is not empty and the first tree within it is a
// token tree, and (c) the two tokens can be glued // token tree, and (c) the two tokens can be glued
// together... // together...
if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last() if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = res_vec_mut.last()
&& let Some((TokenTree::Token(tok), spacing)) = stream.0.first() && let Some(TokenTree::Token(tok, spacing)) = stream.0.first()
&& let Some(glued_tok) = last_tok.glue(&tok) && let Some(glued_tok) = last_tok.glue(&tok)
{ {
// ...then overwrite the last token tree in // ...then overwrite the last token tree in
// `res_vec_mut` with the glued token, and skip the // `res_vec_mut` with the glued token, and skip the
// first token tree from `stream`. // first token tree from `stream`.
*res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); *res_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
res_vec_mut.extend(stream_iter.skip(1)); res_vec_mut.extend(stream_iter.skip(1));
} else { } else {
// Append all of `stream`. // Append all of `stream`.
@ -597,16 +603,8 @@ fn new(stream: &'t TokenStream) -> Self {
CursorRef { stream, index: 0 } CursorRef { stream, index: 0 }
} }
#[inline]
fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> {
self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree
})
}
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.0[self.index..].get(n).map(|(tree, _)| tree) self.stream.0.get(self.index + n)
} }
} }
@ -614,7 +612,10 @@ impl<'t> Iterator for CursorRef<'t> {
type Item = &'t TokenTree; type Item = &'t TokenTree;
fn next(&mut self) -> Option<&'t TokenTree> { fn next(&mut self) -> Option<&'t TokenTree> {
self.next_with_spacing().map(|(tree, _)| tree) self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree
})
} }
} }
@ -630,7 +631,10 @@ impl Iterator for Cursor {
type Item = TokenTree; type Item = TokenTree;
fn next(&mut self) -> Option<TokenTree> { fn next(&mut self) -> Option<TokenTree> {
self.next_with_spacing().map(|(tree, _)| tree) self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree.clone()
})
} }
} }
@ -640,15 +644,7 @@ fn new(stream: TokenStream) -> Self {
} }
#[inline] #[inline]
pub fn next_with_spacing(&mut self) -> Option<TreeAndSpacing> { pub fn next_ref(&mut self) -> Option<&TokenTree> {
self.stream.0.get(self.index).map(|tree| {
self.index += 1;
tree.clone()
})
}
#[inline]
pub fn next_with_spacing_ref(&mut self) -> Option<&TreeAndSpacing> {
self.stream.0.get(self.index).map(|tree| { self.stream.0.get(self.index).map(|tree| {
self.index += 1; self.index += 1;
tree tree
@ -656,7 +652,7 @@ pub fn next_with_spacing_ref(&mut self) -> Option<&TreeAndSpacing> {
} }
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
self.stream.0[self.index..].get(n).map(|(tree, _)| tree) self.stream.0.get(self.index + n)
} }
} }

10
compiler/rustc_ast_pretty/src/pprust/state.rs
View File

@ -145,7 +145,7 @@ pub fn print_crate<'a>(
/// This makes printed token streams look slightly nicer, /// This makes printed token streams look slightly nicer,
/// and also addresses some specific regressions described in #63896 and #73345. /// and also addresses some specific regressions described in #63896 and #73345.
fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool { fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
if let TokenTree::Token(token) = prev { if let TokenTree::Token(token, _) = prev {
if matches!(token.kind, token::Dot | token::Dollar) { if matches!(token.kind, token::Dot | token::Dollar) {
return false; return false;
} }
@ -154,12 +154,12 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
} }
} }
match tt { match tt {
TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot), TokenTree::Token(token, _) => !matches!(token.kind, token::Comma | token::Not | token::Dot),
TokenTree::Delimited(_, Delimiter::Parenthesis, _) => { TokenTree::Delimited(_, Delimiter::Parenthesis, _) => {
!matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. })) !matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }, _))
} }
TokenTree::Delimited(_, Delimiter::Bracket, _) => { TokenTree::Delimited(_, Delimiter::Bracket, _) => {
!matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. })) !matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. }, _))
} }
TokenTree::Delimited(..) => true, TokenTree::Delimited(..) => true,
} }
@ -526,7 +526,7 @@ fn print_meta_item(&mut self, item: &ast::MetaItem) {
/// expression arguments as expressions). It can be done! I think. /// expression arguments as expressions). It can be done! I think.
fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) { fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) {
match tt { match tt {
TokenTree::Token(token) => { TokenTree::Token(token, _) => {
let token_str = self.token_to_string_ext(&token, convert_dollar_crate); let token_str = self.token_to_string_ext(&token, convert_dollar_crate);
self.word(token_str); self.word(token_str);
if let token::DocComment(..) = token.kind { if let token::DocComment(..) = token.kind {

8
compiler/rustc_builtin_macros/src/assert/context.rs
View File

@ -152,7 +152,7 @@ fn build_unlikely(&self, cond_expr: P<Expr>) -> P<Expr> {
fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> { fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
let escaped_expr_str = escape_to_fmt(expr_str); let escaped_expr_str = escape_to_fmt(expr_str);
let initial = [ let initial = [
TokenTree::token( TokenTree::token_alone(
token::Literal(token::Lit { token::Literal(token::Lit {
kind: token::LitKind::Str, kind: token::LitKind::Str,
symbol: Symbol::intern(&if self.fmt_string.is_empty() { symbol: Symbol::intern(&if self.fmt_string.is_empty() {
@ -167,12 +167,12 @@ fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
}), }),
self.span, self.span,
), ),
TokenTree::token(token::Comma, self.span), TokenTree::token_alone(token::Comma, self.span),
]; ];
let captures = self.capture_decls.iter().flat_map(|cap| { let captures = self.capture_decls.iter().flat_map(|cap| {
[ [
TokenTree::token(token::Ident(cap.ident.name, false), cap.ident.span), TokenTree::token_alone(token::Ident(cap.ident.name, false), cap.ident.span),
TokenTree::token(token::Comma, self.span), TokenTree::token_alone(token::Comma, self.span),
] ]
}); });
self.cx.expr( self.cx.expr(

4
compiler/rustc_builtin_macros/src/concat_idents.rs
View File

@ -20,14 +20,14 @@ pub fn expand_concat_idents<'cx>(
for (i, e) in tts.into_trees().enumerate() { for (i, e) in tts.into_trees().enumerate() {
if i & 1 == 1 { if i & 1 == 1 {
match e { match e {
TokenTree::Token(Token { kind: token::Comma, .. }) => {} TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}
_ => { _ => {
cx.span_err(sp, "concat_idents! expecting comma"); cx.span_err(sp, "concat_idents! expecting comma");
return DummyResult::any(sp); return DummyResult::any(sp);
} }
} }
} else { } else {
if let TokenTree::Token(token) = e { if let TokenTree::Token(token, _) = e {
if let Some((ident, _)) = token.ident() { if let Some((ident, _)) = token.ident() {
res_str.push_str(ident.name.as_str()); res_str.push_str(ident.name.as_str());
continue; continue;

4
compiler/rustc_builtin_macros/src/trace_macros.rs
View File

@ -11,8 +11,8 @@ pub fn expand_trace_macros(
let mut cursor = tt.into_trees(); let mut cursor = tt.into_trees();
let mut err = false; let mut err = false;
let value = match &cursor.next() { let value = match &cursor.next() {
Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true, Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true,
Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false, Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false,
_ => { _ => {
err = true; err = true;
false false

4
compiler/rustc_expand/src/config.rs
View File

@ -401,7 +401,7 @@ fn expand_cfg_attr_item(
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token // Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
// for `attr` when we expand it to `#[attr]` // for `attr` when we expand it to `#[attr]`
let mut orig_trees = orig_tokens.into_trees(); let mut orig_trees = orig_tokens.into_trees();
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }) = orig_trees.next().unwrap() else { let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr); panic!("Bad tokens for attribute {:?}", attr);
}; };
let pound_span = pound_token.span; let pound_span = pound_token.span;
@ -409,7 +409,7 @@ fn expand_cfg_attr_item(
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)]; let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
if attr.style == AttrStyle::Inner { if attr.style == AttrStyle::Inner {
// For inner attributes, we do the same thing for the `!` in `#![some_attr]` // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }) = orig_trees.next().unwrap() else { let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
panic!("Bad tokens for attribute {:?}", attr); panic!("Bad tokens for attribute {:?}", attr);
}; };
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone)); trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));

4
compiler/rustc_expand/src/mbe/macro_rules.rs
View File

@ -481,7 +481,7 @@ pub fn compile_declarative_macro(
.map(|m| { .map(|m| {
if let MatchedTokenTree(ref tt) = *m { if let MatchedTokenTree(ref tt) = *m {
let tt = mbe::quoted::parse( let tt = mbe::quoted::parse(
tt.clone().into(), TokenStream::new(vec![tt.clone()]),
true, true,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,
@ -505,7 +505,7 @@ pub fn compile_declarative_macro(
.map(|m| { .map(|m| {
if let MatchedTokenTree(ref tt) = *m { if let MatchedTokenTree(ref tt) = *m {
return mbe::quoted::parse( return mbe::quoted::parse(
tt.clone().into(), TokenStream::new(vec![tt.clone()]),
false, false,
&sess.parse_sess, &sess.parse_sess,
def.id, def.id,

6
compiler/rustc_expand/src/mbe/metavar_expr.rs
View File

@ -106,7 +106,7 @@ fn parse_depth<'sess>(
let Some(tt) = iter.next() else { return Ok(0) }; let Some(tt) = iter.next() else { return Ok(0) };
let TokenTree::Token(token::Token { let TokenTree::Token(token::Token {
kind: token::TokenKind::Literal(lit), .. kind: token::TokenKind::Literal(lit), ..
}) = tt else { }, _) = tt else {
return Err(sess.span_diagnostic.struct_span_err( return Err(sess.span_diagnostic.struct_span_err(
span, span,
"meta-variable expression depth must be a literal" "meta-variable expression depth must be a literal"
@ -130,7 +130,7 @@ fn parse_ident<'sess>(
sess: &'sess ParseSess, sess: &'sess ParseSess,
span: Span, span: Span,
) -> PResult<'sess, Ident> { ) -> PResult<'sess, Ident> {
if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt { if let Some(tt) = iter.next() && let TokenTree::Token(token, _) = tt {
if let Some((elem, false)) = token.ident() { if let Some((elem, false)) = token.ident() {
return Ok(elem); return Ok(elem);
} }
@ -153,7 +153,7 @@ fn parse_ident<'sess>(
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
/// iterator is not modified and the result is `false`. /// iterator is not modified and the result is `false`.
fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool { fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool {
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) { if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
let _ = iter.next(); let _ = iter.next();
return true; return true;
} }

16
compiler/rustc_expand/src/mbe/quoted.rs
View File

@ -56,9 +56,9 @@ pub(super) fn parse(
match tree { match tree {
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => { TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
let span = match trees.next() { let span = match trees.next() {
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
match trees.next() { match trees.next() {
Some(tokenstream::TokenTree::Token(token)) => match token.ident() { Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
Some((frag, _)) => { Some((frag, _)) => {
let span = token.span.with_lo(start_sp.lo()); let span = token.span.with_lo(start_sp.lo());
@ -146,7 +146,7 @@ fn parse_tree(
// Depending on what `tree` is, we could be parsing different parts of a macro // Depending on what `tree` is, we could be parsing different parts of a macro
match tree { match tree {
// `tree` is a `$` token. Look at the next token in `trees` // `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => { tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
// FIXME: Handle `Invisible`-delimited groups in a more systematic way // FIXME: Handle `Invisible`-delimited groups in a more systematic way
// during parsing. // during parsing.
let mut next = outer_trees.next(); let mut next = outer_trees.next();
@ -217,7 +217,7 @@ fn parse_tree(
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate`
// special metavariable that names the crate of the invocation. // special metavariable that names the crate of the invocation.
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
let (ident, is_raw) = token.ident().unwrap(); let (ident, is_raw) = token.ident().unwrap();
let span = ident.span.with_lo(span.lo()); let span = ident.span.with_lo(span.lo());
if ident.name == kw::Crate && !is_raw { if ident.name == kw::Crate && !is_raw {
@ -228,7 +228,7 @@ fn parse_tree(
} }
// `tree` is followed by another `$`. This is an escaped `$`. // `tree` is followed by another `$`. This is an escaped `$`.
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => { Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
if parsing_patterns { if parsing_patterns {
span_dollar_dollar_or_metavar_in_the_lhs_err( span_dollar_dollar_or_metavar_in_the_lhs_err(
sess, sess,
@ -241,7 +241,7 @@ fn parse_tree(
} }
// `tree` is followed by some other token. This is an error. // `tree` is followed by some other token. This is an error.
Some(tokenstream::TokenTree::Token(token)) => { Some(tokenstream::TokenTree::Token(token, _)) => {
let msg = format!( let msg = format!(
"expected identifier, found `{}`", "expected identifier, found `{}`",
pprust::token_to_string(&token), pprust::token_to_string(&token),
@ -256,7 +256,7 @@ fn parse_tree(
} }
// `tree` is an arbitrary token. Keep it. // `tree` is an arbitrary token. Keep it.
tokenstream::TokenTree::Token(token) => TokenTree::Token(token), tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token),
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
// descend into the delimited set and further parse it. // descend into the delimited set and further parse it.
@ -291,7 +291,7 @@ fn parse_kleene_op(
span: Span, span: Span,
) -> Result<Result<(KleeneOp, Span), Token>, Span> { ) -> Result<Result<(KleeneOp, Span), Token>, Span> {
match input.next() { match input.next() {
Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
Some(op) => Ok(Ok((op, token.span))), Some(op) => Ok(Ok((op, token.span))),
None => Ok(Err(token)), None => Ok(Err(token)),
}, },

53
compiler/rustc_expand/src/mbe/transcribe.rs
View File

@ -3,7 +3,7 @@
use crate::mbe::{self, MetaVarExpr}; use crate::mbe::{self, MetaVarExpr};
use rustc_ast::mut_visit::{self, MutVisitor}; use rustc_ast::mut_visit::{self, MutVisitor};
use rustc_ast::token::{self, Delimiter, Token, TokenKind}; use rustc_ast::token::{self, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing}; use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::fx::FxHashMap;
use rustc_errors::{pluralize, PResult}; use rustc_errors::{pluralize, PResult};
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
@ -105,7 +105,7 @@ pub(super) fn transcribe<'a>(
// //
// Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level // Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level
// again, and we are done transcribing. // again, and we are done transcribing.
let mut result: Vec<TreeAndSpacing> = Vec::new(); let mut result: Vec<TokenTree> = Vec::new();
let mut result_stack = Vec::new(); let mut result_stack = Vec::new();
let mut marker = Marker(cx.current_expansion.id, transparency); let mut marker = Marker(cx.current_expansion.id, transparency);
@ -123,7 +123,7 @@ pub(super) fn transcribe<'a>(
if repeat_idx < repeat_len { if repeat_idx < repeat_len {
*idx = 0; *idx = 0;
if let Some(sep) = sep { if let Some(sep) = sep {
result.push(TokenTree::Token(sep.clone()).into()); result.push(TokenTree::Token(sep.clone(), Spacing::Alone));
} }
continue; continue;
} }
@ -150,7 +150,7 @@ pub(super) fn transcribe<'a>(
// Step back into the parent Delimited. // Step back into the parent Delimited.
let tree = TokenTree::Delimited(span, delim, TokenStream::new(result)); let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
result = result_stack.pop().unwrap(); result = result_stack.pop().unwrap();
result.push(tree.into()); result.push(tree);
} }
} }
continue; continue;
@ -227,15 +227,15 @@ pub(super) fn transcribe<'a>(
// `tt`s are emitted into the output stream directly as "raw tokens", // `tt`s are emitted into the output stream directly as "raw tokens",
// without wrapping them into groups. // without wrapping them into groups.
let token = tt.clone(); let token = tt.clone();
result.push(token.into()); result.push(token);
} }
MatchedNonterminal(ref nt) => { MatchedNonterminal(ref nt) => {
// Other variables are emitted into the output stream as groups with // Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities. // `Delimiter::Invisible` to maintain parsing priorities.
// `Interpolated` is currently used for such groups in rustc parser. // `Interpolated` is currently used for such groups in rustc parser.
marker.visit_span(&mut sp); marker.visit_span(&mut sp);
let token = TokenTree::token(token::Interpolated(nt.clone()), sp); let token = TokenTree::token_alone(token::Interpolated(nt.clone()), sp);
result.push(token.into()); result.push(token);
} }
MatchedSeq(..) => { MatchedSeq(..) => {
// We were unable to descend far enough. This is an error. // We were unable to descend far enough. This is an error.
@ -250,8 +250,11 @@ pub(super) fn transcribe<'a>(
// with modified syntax context. (I believe this supports nested macros). // with modified syntax context. (I believe this supports nested macros).
marker.visit_span(&mut sp); marker.visit_span(&mut sp);
marker.visit_ident(&mut original_ident); marker.visit_ident(&mut original_ident);
result.push(TokenTree::token(token::Dollar, sp).into()); result.push(TokenTree::token_alone(token::Dollar, sp));
result.push(TokenTree::Token(Token::from_ast_ident(original_ident)).into()); result.push(TokenTree::Token(
Token::from_ast_ident(original_ident),
Spacing::Alone,
));
} }
} }
@ -281,8 +284,8 @@ pub(super) fn transcribe<'a>(
mbe::TokenTree::Token(token) => { mbe::TokenTree::Token(token) => {
let mut token = token.clone(); let mut token = token.clone();
mut_visit::visit_token(&mut token, &mut marker); mut_visit::visit_token(&mut token, &mut marker);
let tt = TokenTree::Token(token); let tt = TokenTree::Token(token, Spacing::Alone);
result.push(tt.into()); result.push(tt);
} }
// There should be no meta-var declarations in the invocation of a macro. // There should be no meta-var declarations in the invocation of a macro.
@ -532,7 +535,7 @@ fn transcribe_metavar_expr<'a>(
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>, interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
marker: &mut Marker, marker: &mut Marker,
repeats: &[(usize, usize)], repeats: &[(usize, usize)],
result: &mut Vec<TreeAndSpacing>, result: &mut Vec<TokenTree>,
sp: &DelimSpan, sp: &DelimSpan,
) -> PResult<'a, ()> { ) -> PResult<'a, ()> {
let mut visited_span = || { let mut visited_span = || {
@ -544,11 +547,11 @@ fn transcribe_metavar_expr<'a>(
MetaVarExpr::Count(original_ident, depth_opt) => { MetaVarExpr::Count(original_ident, depth_opt) => {
let matched = matched_from_ident(cx, original_ident, interp)?; let matched = matched_from_ident(cx, original_ident, interp)?;
let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?; let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?;
let tt = TokenTree::token( let tt = TokenTree::token_alone(
TokenKind::lit(token::Integer, sym::integer(count), None), TokenKind::lit(token::Integer, sym::integer(count), None),
visited_span(), visited_span(),
); );
result.push(tt.into()); result.push(tt);
} }
MetaVarExpr::Ignore(original_ident) => { MetaVarExpr::Ignore(original_ident) => {
// Used to ensure that `original_ident` is present in the LHS // Used to ensure that `original_ident` is present in the LHS
@ -556,25 +559,19 @@ fn transcribe_metavar_expr<'a>(
} }
MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) { MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) {
Some((index, _)) => { Some((index, _)) => {
result.push( result.push(TokenTree::token_alone(
TokenTree::token( TokenKind::lit(token::Integer, sym::integer(*index), None),
TokenKind::lit(token::Integer, sym::integer(*index), None), visited_span(),
visited_span(), ));
)
.into(),
);
} }
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")), None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")),
}, },
MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) { MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) {
Some((_, length)) => { Some((_, length)) => {
result.push( result.push(TokenTree::token_alone(
TokenTree::token( TokenKind::lit(token::Integer, sym::integer(*length), None),
TokenKind::lit(token::Integer, sym::integer(*length), None), visited_span(),
visited_span(), ));
)
.into(),
);
} }
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")), None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")),
}, },

36
compiler/rustc_expand/src/parse/tests.rs
View File

@ -66,23 +66,23 @@ fn string_to_tts_macro() {
match tts { match tts {
[ [
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }), TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
TokenTree::Token(Token { kind: token::Not, .. }), TokenTree::Token(Token { kind: token::Not, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }), TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
TokenTree::Delimited(_, macro_delim, macro_tts), TokenTree::Delimited(_, macro_delim, macro_tts),
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => { ] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
let tts = &macro_tts.trees().collect::<Vec<_>>(); let tts = &macro_tts.trees().collect::<Vec<_>>();
match &tts[..] { match &tts[..] {
[ [
TokenTree::Delimited(_, first_delim, first_tts), TokenTree::Delimited(_, first_delim, first_tts),
TokenTree::Token(Token { kind: token::FatArrow, .. }), TokenTree::Token(Token { kind: token::FatArrow, .. }, _),
TokenTree::Delimited(_, second_delim, second_tts), TokenTree::Delimited(_, second_delim, second_tts),
] if macro_delim == &Delimiter::Parenthesis => { ] if macro_delim == &Delimiter::Parenthesis => {
let tts = &first_tts.trees().collect::<Vec<_>>(); let tts = &first_tts.trees().collect::<Vec<_>>();
match &tts[..] { match &tts[..] {
[ [
TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => { ] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {
} }
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts), _ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
@ -90,8 +90,8 @@ fn string_to_tts_macro() {
let tts = &second_tts.trees().collect::<Vec<_>>(); let tts = &second_tts.trees().collect::<Vec<_>>();
match &tts[..] { match &tts[..] {
[ [
TokenTree::Token(Token { kind: token::Dollar, .. }), TokenTree::Token(Token { kind: token::Dollar, .. }, _),
TokenTree::Token(Token { kind: token::Ident(name, false), .. }), TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
] if second_delim == &Delimiter::Parenthesis ] if second_delim == &Delimiter::Parenthesis
&& name.as_str() == "a" => {} && name.as_str() == "a" => {}
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts), _ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
@ -111,29 +111,27 @@ fn string_to_tts_1() {
let tts = string_to_stream("fn a (b : i32) { b; }".to_string()); let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
let expected = TokenStream::new(vec![ let expected = TokenStream::new(vec![
TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(), TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)),
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(3, 4)).into(), TokenTree::token_alone(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
TokenTree::Delimited( TokenTree::Delimited(
DelimSpan::from_pair(sp(5, 6), sp(13, 14)), DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
Delimiter::Parenthesis, Delimiter::Parenthesis,
TokenStream::new(vec![ TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(6, 7)).into(), TokenTree::token_alone(token::Ident(Symbol::intern("b"), false), sp(6, 7)),
TokenTree::token(token::Colon, sp(8, 9)).into(), TokenTree::token_alone(token::Colon, sp(8, 9)),
TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(), TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)),
]) ])
.into(), .into(),
) ),
.into(),
TokenTree::Delimited( TokenTree::Delimited(
DelimSpan::from_pair(sp(15, 16), sp(20, 21)), DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
Delimiter::Brace, Delimiter::Brace,
TokenStream::new(vec![ TokenStream::new(vec![
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(), TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(17, 18)),
TokenTree::token(token::Semi, sp(18, 19)).into(), TokenTree::token_alone(token::Semi, sp(18, 19)),
]) ])
.into(), .into(),
) ),
.into(),
]); ]);
assert_eq!(tts, expected); assert_eq!(tts, expected);

4
compiler/rustc_expand/src/proc_macro.rs
View File

@ -4,7 +4,7 @@
use rustc_ast as ast; use rustc_ast as ast;
use rustc_ast::ptr::P; use rustc_ast::ptr::P;
use rustc_ast::token; use rustc_ast::token;
use rustc_ast::tokenstream::{TokenStream, TokenTree}; use rustc_ast::tokenstream::TokenStream;
use rustc_data_structures::sync::Lrc; use rustc_data_structures::sync::Lrc;
use rustc_errors::ErrorGuaranteed; use rustc_errors::ErrorGuaranteed;
use rustc_parse::parser::ForceCollect; use rustc_parse::parser::ForceCollect;
@ -123,7 +123,7 @@ fn expand(
Annotatable::Stmt(stmt) => token::NtStmt(stmt), Annotatable::Stmt(stmt) => token::NtStmt(stmt),
_ => unreachable!(), _ => unreachable!(),
}; };
TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into() TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
} else { } else {
item.to_tokens() item.to_tokens()
}; };

49
compiler/rustc_expand/src/proc_macro_server.rs
View File

@ -92,9 +92,8 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
let mut trees = Vec::with_capacity(stream.len().next_power_of_two()); let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
let mut cursor = stream.into_trees(); let mut cursor = stream.into_trees();
while let Some((tree, spacing)) = cursor.next_with_spacing() { while let Some(tree) = cursor.next() {
let joint = spacing == Joint; let (Token { kind, span }, joint) = match tree {
let Token { kind, span } = match tree {
tokenstream::TokenTree::Delimited(span, delim, tts) => { tokenstream::TokenTree::Delimited(span, delim, tts) => {
let delimiter = pm::Delimiter::from_internal(delim); let delimiter = pm::Delimiter::from_internal(delim);
trees.push(TokenTree::Group(Group { trees.push(TokenTree::Group(Group {
@ -108,7 +107,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
})); }));
continue; continue;
} }
tokenstream::TokenTree::Token(token) => token, tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
}; };
let mut op = |s: &str| { let mut op = |s: &str| {
@ -194,7 +193,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
TokenKind::lit(token::Str, Symbol::intern(&escaped), None), TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
] ]
.into_iter() .into_iter()
.map(|kind| tokenstream::TokenTree::token(kind, span)) .map(|kind| tokenstream::TokenTree::token_alone(kind, span))
.collect(); .collect();
trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span })); trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
if attr_style == ast::AttrStyle::Inner { if attr_style == ast::AttrStyle::Inner {
@ -246,16 +245,15 @@ fn to_internal(self) -> TokenStream {
let (ch, joint, span) = match tree { let (ch, joint, span) = match tree {
TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span), TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => { TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
return tokenstream::TokenTree::Delimited( return tokenstream::TokenStream::delimited(
tokenstream::DelimSpan { open, close }, tokenstream::DelimSpan { open, close },
delimiter.to_internal(), delimiter.to_internal(),
stream.unwrap_or_default(), stream.unwrap_or_default(),
) );
.into();
} }
TokenTree::Ident(self::Ident { sym, is_raw, span }) => { TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
rustc.sess().symbol_gallery.insert(sym, span); rustc.sess().symbol_gallery.insert(sym, span);
return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into(); return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span);
} }
TokenTree::Literal(self::Literal { TokenTree::Literal(self::Literal {
kind: self::LitKind::Integer, kind: self::LitKind::Integer,
@ -266,8 +264,8 @@ fn to_internal(self) -> TokenStream {
let minus = BinOp(BinOpToken::Minus); let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]); let symbol = Symbol::intern(&symbol.as_str()[1..]);
let integer = TokenKind::lit(token::Integer, symbol, suffix); let integer = TokenKind::lit(token::Integer, symbol, suffix);
let a = tokenstream::TokenTree::token(minus, span); let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token(integer, span); let b = tokenstream::TokenTree::token_alone(integer, span);
return [a, b].into_iter().collect(); return [a, b].into_iter().collect();
} }
TokenTree::Literal(self::Literal { TokenTree::Literal(self::Literal {
@ -279,16 +277,15 @@ fn to_internal(self) -> TokenStream {
let minus = BinOp(BinOpToken::Minus); let minus = BinOp(BinOpToken::Minus);
let symbol = Symbol::intern(&symbol.as_str()[1..]); let symbol = Symbol::intern(&symbol.as_str()[1..]);
let float = TokenKind::lit(token::Float, symbol, suffix); let float = TokenKind::lit(token::Float, symbol, suffix);
let a = tokenstream::TokenTree::token(minus, span); let a = tokenstream::TokenTree::token_alone(minus, span);
let b = tokenstream::TokenTree::token(float, span); let b = tokenstream::TokenTree::token_alone(float, span);
return [a, b].into_iter().collect(); return [a, b].into_iter().collect();
} }
TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => { TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
return tokenstream::TokenTree::token( return tokenstream::TokenStream::token_alone(
TokenKind::lit(kind.to_internal(), symbol, suffix), TokenKind::lit(kind.to_internal(), symbol, suffix),
span, span,
) );
.into();
} }
}; };
@ -318,8 +315,11 @@ fn to_internal(self) -> TokenStream {
_ => unreachable!(), _ => unreachable!(),
}; };
let tree = tokenstream::TokenTree::token(kind, span); if joint {
TokenStream::new(vec![(tree, if joint { Joint } else { Alone })]) tokenstream::TokenStream::token_joint(kind, span)
} else {
tokenstream::TokenStream::token_alone(kind, span)
}
} }
} }
@ -486,12 +486,11 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStrea
// We don't use `TokenStream::from_ast` as the tokenstream currently cannot // We don't use `TokenStream::from_ast` as the tokenstream currently cannot
// be recovered in the general case. // be recovered in the general case.
match &expr.kind { match &expr.kind {
ast::ExprKind::Lit(l) if l.token.kind == token::Bool => { ast::ExprKind::Lit(l) if l.token.kind == token::Bool => Ok(
Ok(tokenstream::TokenTree::token(token::Ident(l.token.symbol, false), l.span) tokenstream::TokenStream::token_alone(token::Ident(l.token.symbol, false), l.span),
.into()) ),
}
ast::ExprKind::Lit(l) => { ast::ExprKind::Lit(l) => {
Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into()) Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token), l.span))
} }
ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind { ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
ast::ExprKind::Lit(l) => match l.token { ast::ExprKind::Lit(l) => match l.token {
@ -499,8 +498,8 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStrea
Ok(Self::TokenStream::from_iter([ Ok(Self::TokenStream::from_iter([
// FIXME: The span of the `-` token is lost when // FIXME: The span of the `-` token is lost when
// parsing, so we cannot faithfully recover it here. // parsing, so we cannot faithfully recover it here.
tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span), tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
tokenstream::TokenTree::token(token::Literal(l.token), l.span), tokenstream::TokenTree::token_alone(token::Literal(l.token), l.span),
])) ]))
} }
_ => Err(()), _ => Err(()),

17
compiler/rustc_expand/src/tokenstream/tests.rs
View File

@ -1,7 +1,7 @@
use crate::tests::string_to_stream; use crate::tests::string_to_stream;
use rustc_ast::token; use rustc_ast::token;
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree}; use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
use rustc_span::create_default_session_globals_then; use rustc_span::create_default_session_globals_then;
use rustc_span::{BytePos, Span, Symbol}; use rustc_span::{BytePos, Span, Symbol};
@ -13,10 +13,6 @@ fn sp(a: u32, b: u32) -> Span {
Span::with_root_ctxt(BytePos(a), BytePos(b)) Span::with_root_ctxt(BytePos(a), BytePos(b))
} }
fn joint(tree: TokenTree) -> TokenStream {
TokenStream::new(vec![(tree, Spacing::Joint)])
}
#[test] #[test]
fn test_concat() { fn test_concat() {
create_default_session_globals_then(|| { create_default_session_globals_then(|| {
@ -90,9 +86,8 @@ fn test_diseq_1() {
#[test] #[test]
fn test_is_empty() { fn test_is_empty() {
create_default_session_globals_then(|| { create_default_session_globals_then(|| {
let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect(); let test0 = TokenStream::default();
let test1: TokenStream = let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1));
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(0, 1)).into();
let test2 = string_to_ts("foo(bar::baz)"); let test2 = string_to_ts("foo(bar::baz)");
assert_eq!(test0.is_empty(), true); assert_eq!(test0.is_empty(), true);
@ -105,9 +100,9 @@ fn test_is_empty() {
fn test_dotdotdot() { fn test_dotdotdot() {
create_default_session_globals_then(|| { create_default_session_globals_then(|| {
let mut builder = TokenStreamBuilder::new(); let mut builder = TokenStreamBuilder::new();
builder.push(joint(TokenTree::token(token::Dot, sp(0, 1)))); builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
builder.push(joint(TokenTree::token(token::Dot, sp(1, 2)))); builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
builder.push(TokenTree::token(token::Dot, sp(2, 3))); builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
let stream = builder.build(); let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("..."))); assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1); assert_eq!(stream.trees().count(), 1);

2
compiler/rustc_lint/src/builtin.rs
View File

@ -1974,7 +1974,7 @@ fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: TokenStream) {
for tt in tokens.into_trees() { for tt in tokens.into_trees() {
match tt { match tt {
// Only report non-raw idents. // Only report non-raw idents.
TokenTree::Token(token) => { TokenTree::Token(token, _) => {
if let Some((ident, false)) = token.ident() { if let Some((ident, false)) = token.ident() {
self.check_ident_token(cx, UnderMacro(true), ident); self.check_ident_token(cx, UnderMacro(true), ident);
} }

28
compiler/rustc_parse/src/lexer/tokentrees.rs
View File

@ -1,11 +1,7 @@
use super::{StringReader, UnmatchedBrace}; use super::{StringReader, UnmatchedBrace};
use rustc_ast::token::{self, Delimiter, Token}; use rustc_ast::token::{self, Delimiter, Token};
use rustc_ast::tokenstream::{ use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
DelimSpan,
Spacing::{self, *},
TokenStream, TokenTree, TreeAndSpacing,
};
use rustc_ast_pretty::pprust::token_to_string; use rustc_ast_pretty::pprust::token_to_string;
use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::fx::FxHashMap;
use rustc_errors::PResult; use rustc_errors::PResult;
@ -77,7 +73,7 @@ fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
} }
} }
fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> { fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
let sm = self.string_reader.sess.source_map(); let sm = self.string_reader.sess.source_map();
match self.token.kind { match self.token.kind {
@ -223,7 +219,7 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> {
_ => {} _ => {}
} }
Ok(TokenTree::Delimited(delim_span, delim, tts).into()) Ok(TokenTree::Delimited(delim_span, delim, tts))
} }
token::CloseDelim(delim) => { token::CloseDelim(delim) => {
// An unexpected closing delimiter (i.e., there is no // An unexpected closing delimiter (i.e., there is no
@ -258,12 +254,12 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> {
Err(err) Err(err)
} }
_ => { _ => {
let tt = TokenTree::Token(self.token.take()); let tok = self.token.take();
let mut spacing = self.bump(); let mut spacing = self.bump();
if !self.token.is_op() { if !self.token.is_op() {
spacing = Alone; spacing = Spacing::Alone;
} }
Ok((tt, spacing)) Ok(TokenTree::Token(tok, spacing))
} }
} }
} }
@ -277,20 +273,20 @@ fn bump(&mut self) -> Spacing {
#[derive(Default)] #[derive(Default)]
struct TokenStreamBuilder { struct TokenStreamBuilder {
buf: Vec<TreeAndSpacing>, buf: Vec<TokenTree>,
} }
impl TokenStreamBuilder { impl TokenStreamBuilder {
fn push(&mut self, (tree, joint): TreeAndSpacing) { fn push(&mut self, tree: TokenTree) {
if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last() if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
&& let TokenTree::Token(token) = &tree && let TokenTree::Token(token, joint) = &tree
&& let Some(glued) = prev_token.glue(token) && let Some(glued) = prev_token.glue(token)
{ {
self.buf.pop(); self.buf.pop();
self.buf.push((TokenTree::Token(glued), joint)); self.buf.push(TokenTree::Token(glued, *joint));
return; return;
} }
self.buf.push((tree, joint)) self.buf.push(tree);
} }
fn into_token_stream(self) -> TokenStream { fn into_token_stream(self) -> TokenStream {

4
compiler/rustc_parse/src/parser/item.rs
View File

@ -1664,8 +1664,8 @@ fn parse_item_decl_macro(&mut self, lo: Span) -> PResult<'a, ItemInfo> {
let body = self.parse_token_tree(); // `MacBody` let body = self.parse_token_tree(); // `MacBody`
// Convert `MacParams MacBody` into `{ MacParams => MacBody }`. // Convert `MacParams MacBody` into `{ MacParams => MacBody }`.
let bspan = body.span(); let bspan = body.span();
let arrow = TokenTree::token(token::FatArrow, pspan.between(bspan)); // `=>` let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>`
let tokens = TokenStream::new(vec![params.into(), arrow.into(), body.into()]); let tokens = TokenStream::new(vec![params, arrow, body]);
let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi()); let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens)) P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
} else { } else {

40
compiler/rustc_parse/src/parser/mod.rs
View File

@ -268,13 +268,13 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
// FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will // FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
// need to, whereupon the `delim != Delimiter::Invisible` conditions below can be // need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
// removed. // removed.
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() { if let Some(tree) = self.frame.tree_cursor.next_ref() {
match tree { match tree {
&TokenTree::Token(ref token) => match (desugar_doc_comments, token) { &TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) {
(true, &Token { kind: token::DocComment(_, attr_style, data), span }) => { (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
return self.desugar(attr_style, data, span); return self.desugar(attr_style, data, span);
} }
_ => return (token.clone(), *spacing), _ => return (token.clone(), spacing),
}, },
&TokenTree::Delimited(sp, delim, ref tts) => { &TokenTree::Delimited(sp, delim, ref tts) => {
// Set `open_delim` to true here because we deal with it immediately. // Set `open_delim` to true here because we deal with it immediately.
@ -318,12 +318,14 @@ fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token
delim_span, delim_span,
Delimiter::Bracket, Delimiter::Bracket,
[ [
TokenTree::token(token::Ident(sym::doc, false), span), TokenTree::token_alone(token::Ident(sym::doc, false), span),
TokenTree::token(token::Eq, span), TokenTree::token_alone(token::Eq, span),
TokenTree::token(TokenKind::lit(token::StrRaw(num_of_hashes), data, None), span), TokenTree::token_alone(
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
span,
),
] ]
.iter() .into_iter()
.cloned()
.collect::<TokenStream>(), .collect::<TokenStream>(),
); );
@ -332,14 +334,16 @@ fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token
TokenCursorFrame::new( TokenCursorFrame::new(
None, None,
if attr_style == AttrStyle::Inner { if attr_style == AttrStyle::Inner {
[TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body] [
.iter() TokenTree::token_alone(token::Pound, span),
.cloned() TokenTree::token_alone(token::Not, span),
.collect::<TokenStream>() body,
]
.into_iter()
.collect::<TokenStream>()
} else { } else {
[TokenTree::token(token::Pound, span), body] [TokenTree::token_alone(token::Pound, span), body]
.iter() .into_iter()
.cloned()
.collect::<TokenStream>() .collect::<TokenStream>()
}, },
), ),
@ -1042,7 +1046,7 @@ pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R
if all_normal { if all_normal {
return match frame.tree_cursor.look_ahead(dist - 1) { return match frame.tree_cursor.look_ahead(dist - 1) {
Some(tree) => match tree { Some(tree) => match tree {
TokenTree::Token(token) => looker(token), TokenTree::Token(token, _) => looker(token),
TokenTree::Delimited(dspan, delim, _) => { TokenTree::Delimited(dspan, delim, _) => {
looker(&Token::new(token::OpenDelim(*delim), dspan.open)) looker(&Token::new(token::OpenDelim(*delim), dspan.open))
} }
@ -1226,7 +1230,7 @@ pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
token::CloseDelim(_) | token::Eof => unreachable!(), token::CloseDelim(_) | token::Eof => unreachable!(),
_ => { _ => {
self.bump(); self.bump();
TokenTree::Token(self.prev_token.clone()) TokenTree::Token(self.prev_token.clone(), Spacing::Alone)
} }
} }
} }
@ -1245,7 +1249,7 @@ pub fn parse_tokens(&mut self) -> TokenStream {
loop { loop {
match self.token.kind { match self.token.kind {
token::Eof | token::CloseDelim(..) => break, token::Eof | token::CloseDelim(..) => break,
_ => result.push(self.parse_token_tree().into()), _ => result.push(self.parse_token_tree()),
} }
} }
TokenStream::new(result) TokenStream::new(result)

6
src/librustdoc/clean/render_macro_matchers.rs
View File

@ -43,7 +43,7 @@ pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Stri
TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts), TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts),
// Matcher which is not a Delimited is unexpected and should've failed // Matcher which is not a Delimited is unexpected and should've failed
// to compile, but we render whatever it is wrapped in parens. // to compile, but we render whatever it is wrapped in parens.
TokenTree::Token(_) => print_tt(&mut printer, matcher), TokenTree::Token(..) => print_tt(&mut printer, matcher),
} }
printer.end(); printer.end();
printer.break_offset_if_not_bol(0, -4); printer.break_offset_if_not_bol(0, -4);
@ -93,7 +93,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String
fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) { fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) {
match tt { match tt {
TokenTree::Token(token) => { TokenTree::Token(token, _) => {
let token_str = printer.token_to_string(token); let token_str = printer.token_to_string(token);
printer.word(token_str); printer.word(token_str);
if let token::DocComment(..) = token.kind { if let token::DocComment(..) = token.kind {
@ -138,7 +138,7 @@ enum State {
let mut state = Start; let mut state = Start;
for tt in tts.trees() { for tt in tts.trees() {
let (needs_space, next_state) = match &tt { let (needs_space, next_state) = match &tt {
TokenTree::Token(tt) => match (state, &tt.kind) { TokenTree::Token(tt, _) => match (state, &tt.kind) {
(Dollar, token::Ident(..)) => (false, DollarIdent), (Dollar, token::Ident(..)) => (false, DollarIdent),
(DollarIdent, token::Colon) => (false, DollarIdentColon), (DollarIdent, token::Colon) => (false, DollarIdentColon),
(DollarIdentColon, token::Ident(..)) => (false, Other), (DollarIdentColon, token::Ident(..)) => (false, Other),

4
src/tools/clippy/clippy_lints/src/crate_in_macro_def.rs
View File

@ -110,14 +110,14 @@ fn contains_unhygienic_crate_reference(tts: &TokenStream) -> Option<Span> {
fn is_crate_keyword(tt: &TokenTree) -> Option<Span> { fn is_crate_keyword(tt: &TokenTree) -> Option<Span> {
if_chain! { if_chain! {
if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }) = tt; if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }, _) = tt;
if symbol.as_str() == "crate"; if symbol.as_str() == "crate";
then { Some(*span) } else { None } then { Some(*span) } else { None }
} }
} }
fn is_token(tt: &TokenTree, kind: &TokenKind) -> bool { fn is_token(tt: &TokenTree, kind: &TokenKind) -> bool {
if let TokenTree::Token(Token { kind: other, .. }) = tt { if let TokenTree::Token(Token { kind: other, .. }, _) = tt {
kind == other kind == other
} else { } else {
false false

98
src/tools/rustfmt/src/macros.rs
View File

@ -13,7 +13,7 @@
use std::panic::{catch_unwind, AssertUnwindSafe}; use std::panic::{catch_unwind, AssertUnwindSafe};
use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind}; use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind};
use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree}; use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
use rustc_ast::{ast, ptr}; use rustc_ast::{ast, ptr};
use rustc_ast_pretty::pprust; use rustc_ast_pretty::pprust;
use rustc_span::{ use rustc_span::{
@ -682,7 +682,7 @@ struct MacroArgParser {
fn last_tok(tt: &TokenTree) -> Token { fn last_tok(tt: &TokenTree) -> Token {
match *tt { match *tt {
TokenTree::Token(ref t) => t.clone(), TokenTree::Token(ref t, _) => t.clone(),
TokenTree::Delimited(delim_span, delim, _) => Token { TokenTree::Delimited(delim_span, delim, _) => Token {
kind: TokenKind::CloseDelim(delim), kind: TokenKind::CloseDelim(delim),
span: delim_span.close, span: delim_span.close,
@ -737,10 +737,13 @@ fn add_other(&mut self) {
fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> { fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> {
match iter.next() { match iter.next() {
Some(TokenTree::Token(Token { Some(TokenTree::Token(
kind: TokenKind::Ident(name, _), Token {
.. kind: TokenKind::Ident(name, _),
})) => { ..
},
_,
)) => {
self.result.push(ParsedMacroArg { self.result.push(ParsedMacroArg {
kind: MacroArgKind::MetaVariable(name, self.buf.clone()), kind: MacroArgKind::MetaVariable(name, self.buf.clone()),
}); });
@ -777,21 +780,30 @@ fn add_repeat(
} }
match tok { match tok {
TokenTree::Token(Token { TokenTree::Token(
kind: TokenKind::BinOp(BinOpToken::Plus), Token {
.. kind: TokenKind::BinOp(BinOpToken::Plus),
}) ..
| TokenTree::Token(Token { },
kind: TokenKind::Question, _,
.. )
}) | TokenTree::Token(
| TokenTree::Token(Token { Token {
kind: TokenKind::BinOp(BinOpToken::Star), kind: TokenKind::Question,
.. ..
}) => { },
_,
)
| TokenTree::Token(
Token {
kind: TokenKind::BinOp(BinOpToken::Star),
..
},
_,
) => {
break; break;
} }
TokenTree::Token(ref t) => { TokenTree::Token(ref t, _) => {
buffer.push_str(&pprust::token_to_string(t)); buffer.push_str(&pprust::token_to_string(t));
} }
_ => return None, _ => return None,
@ -859,10 +871,13 @@ fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
while let Some(tok) = iter.next() { while let Some(tok) = iter.next() {
match tok { match tok {
TokenTree::Token(Token { TokenTree::Token(
kind: TokenKind::Dollar, Token {
span, kind: TokenKind::Dollar,
}) => { span,
},
_,
) => {
// We always want to add a separator before meta variables. // We always want to add a separator before meta variables.
if !self.buf.is_empty() { if !self.buf.is_empty() {
self.add_separator(); self.add_separator();
@ -875,13 +890,16 @@ fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
span, span,
}; };
} }
TokenTree::Token(Token { TokenTree::Token(
kind: TokenKind::Colon, Token {
.. kind: TokenKind::Colon,
}) if self.is_meta_var => { ..
},
_,
) if self.is_meta_var => {
self.add_meta_variable(&mut iter)?; self.add_meta_variable(&mut iter)?;
} }
TokenTree::Token(ref t) => self.update_buffer(t), TokenTree::Token(ref t, _) => self.update_buffer(t),
TokenTree::Delimited(_delimited_span, delimited, ref tts) => { TokenTree::Delimited(_delimited_span, delimited, ref tts) => {
if !self.buf.is_empty() { if !self.buf.is_empty() {
if next_space(&self.last_tok.kind) == SpaceState::Always { if next_space(&self.last_tok.kind) == SpaceState::Always {
@ -1123,12 +1141,15 @@ fn parse_branch(&mut self) -> Option<MacroBranch> {
TokenTree::Token(..) => return None, TokenTree::Token(..) => return None,
TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d), TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d),
}; };
let args = TokenStream::new(vec![(tok, Spacing::Joint)]); let args = TokenStream::new(vec![tok]);
match self.toks.next()? { match self.toks.next()? {
TokenTree::Token(Token { TokenTree::Token(
kind: TokenKind::FatArrow, Token {
.. kind: TokenKind::FatArrow,
}) => {} ..
},
_,
) => {}
_ => return None, _ => return None,
} }
let (mut hi, body, whole_body) = match self.toks.next()? { let (mut hi, body, whole_body) = match self.toks.next()? {
@ -1147,10 +1168,13 @@ fn parse_branch(&mut self) -> Option<MacroBranch> {
) )
} }
}; };
if let Some(TokenTree::Token(Token { if let Some(TokenTree::Token(
kind: TokenKind::Semi, Token {
span, kind: TokenKind::Semi,
})) = self.toks.look_ahead(0) span,
},
_,
)) = self.toks.look_ahead(0)
{ {
hi = span.hi(); hi = span.hi();
self.toks.next(); self.toks.next();