Auto merge of #99887 - nnethercote:rm-TreeAndSpacing, r=petrochenkov
Remove `TreeAndSpacing`.
A `TokenStream` contains a `Lrc<Vec<(TokenTree, Spacing)>>`. But this is
not quite right. `Spacing` makes sense for `TokenTree::Token`, but does
not make sense for `TokenTree::Delimited`, because a
`TokenTree::Delimited` cannot be joined with another `TokenTree`.
This commit fixes this problem, by adding `Spacing` to `TokenTree::Token`,
changing `TokenStream` to contain a `Lrc<Vec<TokenTree>>`, and removing the
`TreeAndSpacing` typedef.
The commit removes these two impls:
- `impl From<TokenTree> for TokenStream`
- `impl From<TokenTree> for TreeAndSpacing`
These were useful, but also resulted in code with many `.into()` calls
that was hard to read, particularly for anyone not highly familiar with
the relevant types. This commit makes some other changes to compensate:
- `TokenTree::token()` becomes `TokenTree::token_{alone,joint}()`.
- `TokenStream::token_{alone,joint}()` are added.
- `TokenStream::delimited` is added.
This results in things like this:
```rust
TokenTree::token(token::Semi, stmt.span).into()
```
changing to this:
```rust
TokenStream::token_alone(token::Semi, stmt.span)
```
This makes the type of the result, and its spacing, clearer.
These changes also simplifies `Cursor` and `CursorRef`, because they no longer
need to distinguish between `next` and `next_with_spacing`.
r? `@petrochenkov`
This commit is contained in:
bors
commit
1202bbaf48
@ -8,7 +8,7 @@
|
||||
use crate::ptr::P;
|
||||
use crate::token::{self, CommentKind, Delimiter, Token};
|
||||
use crate::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree};
|
||||
use crate::tokenstream::{DelimSpan, Spacing, TokenTree, TreeAndSpacing};
|
||||
use crate::tokenstream::{DelimSpan, Spacing, TokenTree};
|
||||
use crate::tokenstream::{LazyTokenStream, TokenStream};
|
||||
use crate::util::comments;
|
||||
|
||||
@ -388,7 +388,7 @@ pub fn list_contains_name(items: &[NestedMetaItem], name: Symbol) -> bool {
|
||||
}
|
||||
|
||||
impl MetaItem {
|
||||
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
|
||||
fn token_trees(&self) -> Vec<TokenTree> {
|
||||
let mut idents = vec![];
|
||||
let mut last_pos = BytePos(0_u32);
|
||||
for (i, segment) in self.path.segments.iter().enumerate() {
|
||||
@ -396,12 +396,12 @@ fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
|
||||
if !is_first {
|
||||
let mod_sep_span =
|
||||
Span::new(last_pos, segment.ident.span.lo(), segment.ident.span.ctxt(), None);
|
||||
idents.push(TokenTree::token(token::ModSep, mod_sep_span).into());
|
||||
idents.push(TokenTree::token_alone(token::ModSep, mod_sep_span));
|
||||
}
|
||||
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
|
||||
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident), Spacing::Alone));
|
||||
last_pos = segment.ident.span.hi();
|
||||
}
|
||||
idents.extend(self.kind.token_trees_and_spacings(self.span));
|
||||
idents.extend(self.kind.token_trees(self.span));
|
||||
idents
|
||||
}
|
||||
|
||||
@ -411,12 +411,13 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
|
||||
{
|
||||
// FIXME: Share code with `parse_path`.
|
||||
let path = match tokens.next().map(TokenTree::uninterpolate) {
|
||||
Some(TokenTree::Token(Token {
|
||||
kind: kind @ (token::Ident(..) | token::ModSep),
|
||||
span,
|
||||
})) => 'arm: {
|
||||
Some(TokenTree::Token(
|
||||
Token { kind: kind @ (token::Ident(..) | token::ModSep), span },
|
||||
_,
|
||||
)) => 'arm: {
|
||||
let mut segments = if let token::Ident(name, _) = kind {
|
||||
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
|
||||
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
|
||||
tokens.peek()
|
||||
{
|
||||
tokens.next();
|
||||
vec![PathSegment::from_ident(Ident::new(name, span))]
|
||||
@ -427,14 +428,15 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
|
||||
vec![PathSegment::path_root(span)]
|
||||
};
|
||||
loop {
|
||||
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) =
|
||||
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }, _)) =
|
||||
tokens.next().map(TokenTree::uninterpolate)
|
||||
{
|
||||
segments.push(PathSegment::from_ident(Ident::new(name, span)));
|
||||
} else {
|
||||
return None;
|
||||
}
|
||||
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek()
|
||||
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. }, _)) =
|
||||
tokens.peek()
|
||||
{
|
||||
tokens.next();
|
||||
} else {
|
||||
@ -444,7 +446,7 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
|
||||
let span = span.with_hi(segments.last().unwrap().ident.span.hi());
|
||||
Path { span, segments, tokens: None }
|
||||
}
|
||||
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
|
||||
Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match *nt {
|
||||
token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
|
||||
token::Nonterminal::NtPath(ref path) => (**path).clone(),
|
||||
_ => return None,
|
||||
@ -491,9 +493,9 @@ pub fn mac_args(&self, span: Span) -> MacArgs {
|
||||
let mut tts = Vec::new();
|
||||
for (i, item) in list.iter().enumerate() {
|
||||
if i > 0 {
|
||||
tts.push(TokenTree::token(token::Comma, span).into());
|
||||
tts.push(TokenTree::token_alone(token::Comma, span));
|
||||
}
|
||||
tts.extend(item.token_trees_and_spacings())
|
||||
tts.extend(item.token_trees())
|
||||
}
|
||||
MacArgs::Delimited(
|
||||
DelimSpan::from_single(span),
|
||||
@ -504,31 +506,28 @@ pub fn mac_args(&self, span: Span) -> MacArgs {
|
||||
}
|
||||
}
|
||||
|
||||
fn token_trees_and_spacings(&self, span: Span) -> Vec<TreeAndSpacing> {
|
||||
fn token_trees(&self, span: Span) -> Vec<TokenTree> {
|
||||
match *self {
|
||||
MetaItemKind::Word => vec![],
|
||||
MetaItemKind::NameValue(ref lit) => {
|
||||
vec![
|
||||
TokenTree::token(token::Eq, span).into(),
|
||||
TokenTree::Token(lit.to_token()).into(),
|
||||
TokenTree::token_alone(token::Eq, span),
|
||||
TokenTree::Token(lit.to_token(), Spacing::Alone),
|
||||
]
|
||||
}
|
||||
MetaItemKind::List(ref list) => {
|
||||
let mut tokens = Vec::new();
|
||||
for (i, item) in list.iter().enumerate() {
|
||||
if i > 0 {
|
||||
tokens.push(TokenTree::token(token::Comma, span).into());
|
||||
tokens.push(TokenTree::token_alone(token::Comma, span));
|
||||
}
|
||||
tokens.extend(item.token_trees_and_spacings())
|
||||
tokens.extend(item.token_trees())
|
||||
}
|
||||
vec![
|
||||
TokenTree::Delimited(
|
||||
DelimSpan::from_single(span),
|
||||
Delimiter::Parenthesis,
|
||||
TokenStream::new(tokens),
|
||||
)
|
||||
.into(),
|
||||
]
|
||||
vec![TokenTree::Delimited(
|
||||
DelimSpan::from_single(span),
|
||||
Delimiter::Parenthesis,
|
||||
TokenStream::new(tokens),
|
||||
)]
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -540,7 +539,7 @@ fn list_from_tokens(tokens: TokenStream) -> Option<MetaItemKind> {
|
||||
let item = NestedMetaItem::from_tokens(&mut tokens)?;
|
||||
result.push(item);
|
||||
match tokens.next() {
|
||||
None | Some(TokenTree::Token(Token { kind: token::Comma, .. })) => {}
|
||||
None | Some(TokenTree::Token(Token { kind: token::Comma, .. }, _)) => {}
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
@ -554,7 +553,7 @@ fn name_value_from_tokens(
|
||||
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
|
||||
MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
|
||||
}
|
||||
Some(TokenTree::Token(token)) => {
|
||||
Some(TokenTree::Token(token, _)) => {
|
||||
Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
|
||||
}
|
||||
_ => None,
|
||||
@ -586,7 +585,7 @@ fn from_tokens(
|
||||
MetaItemKind::list_from_tokens(inner_tokens)
|
||||
}
|
||||
Some(TokenTree::Delimited(..)) => None,
|
||||
Some(TokenTree::Token(Token { kind: token::Eq, .. })) => {
|
||||
Some(TokenTree::Token(Token { kind: token::Eq, .. }, _)) => {
|
||||
tokens.next();
|
||||
MetaItemKind::name_value_from_tokens(tokens)
|
||||
}
|
||||
@ -603,10 +602,12 @@ pub fn span(&self) -> Span {
|
||||
}
|
||||
}
|
||||
|
||||
fn token_trees_and_spacings(&self) -> Vec<TreeAndSpacing> {
|
||||
fn token_trees(&self) -> Vec<TokenTree> {
|
||||
match *self {
|
||||
NestedMetaItem::MetaItem(ref item) => item.token_trees_and_spacings(),
|
||||
NestedMetaItem::Literal(ref lit) => vec![TokenTree::Token(lit.to_token()).into()],
|
||||
NestedMetaItem::MetaItem(ref item) => item.token_trees(),
|
||||
NestedMetaItem::Literal(ref lit) => {
|
||||
vec![TokenTree::Token(lit.to_token(), Spacing::Alone)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -615,7 +616,7 @@ fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<NestedMetaItem>
|
||||
I: Iterator<Item = TokenTree>,
|
||||
{
|
||||
match tokens.peek() {
|
||||
Some(TokenTree::Token(token))
|
||||
Some(TokenTree::Token(token, _))
|
||||
if let Ok(lit) = Lit::from_token(token) =>
|
||||
{
|
||||
tokens.next();
|
||||
|
||||
@ -675,7 +675,7 @@ pub fn visit_attr_annotated_tt<T: MutVisitor>(tt: &mut AttrAnnotatedTokenTree, v
|
||||
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
|
||||
pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
|
||||
match tt {
|
||||
TokenTree::Token(token) => {
|
||||
TokenTree::Token(token, _) => {
|
||||
visit_token(token, vis);
|
||||
}
|
||||
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
|
||||
@ -690,7 +690,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
|
||||
pub fn visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &mut T) {
|
||||
if T::VISIT_TOKENS && !tts.is_empty() {
|
||||
let tts = Lrc::make_mut(tts);
|
||||
visit_vec(tts, |(tree, _is_joint)| visit_tt(tree, vis));
|
||||
visit_vec(tts, |tree| visit_tt(tree, vis));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -42,11 +42,15 @@
|
||||
#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
|
||||
pub enum TokenTree {
|
||||
/// A single token.
|
||||
Token(Token),
|
||||
Token(Token, Spacing),
|
||||
/// A delimited sequence of token trees.
|
||||
Delimited(DelimSpan, Delimiter, TokenStream),
|
||||
}
|
||||
|
||||
// This type is used a lot. Make sure it doesn't unintentionally get bigger.
|
||||
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
|
||||
rustc_data_structures::static_assert_size!(TokenTree, 32);
|
||||
|
||||
// Ensure all fields of `TokenTree` is `Send` and `Sync`.
|
||||
#[cfg(parallel_compiler)]
|
||||
fn _dummy()
|
||||
@ -62,7 +66,7 @@ impl TokenTree {
|
||||
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
|
||||
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
|
||||
match (self, other) {
|
||||
(TokenTree::Token(token), TokenTree::Token(token2)) => token.kind == token2.kind,
|
||||
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
|
||||
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
|
||||
delim == delim2 && tts.eq_unspanned(&tts2)
|
||||
}
|
||||
@ -73,7 +77,7 @@ pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
|
||||
/// Retrieves the `TokenTree`'s span.
|
||||
pub fn span(&self) -> Span {
|
||||
match self {
|
||||
TokenTree::Token(token) => token.span,
|
||||
TokenTree::Token(token, _) => token.span,
|
||||
TokenTree::Delimited(sp, ..) => sp.entire(),
|
||||
}
|
||||
}
|
||||
@ -81,18 +85,26 @@ pub fn span(&self) -> Span {
|
||||
/// Modify the `TokenTree`'s span in-place.
|
||||
pub fn set_span(&mut self, span: Span) {
|
||||
match self {
|
||||
TokenTree::Token(token) => token.span = span,
|
||||
TokenTree::Token(token, _) => token.span = span,
|
||||
TokenTree::Delimited(dspan, ..) => *dspan = DelimSpan::from_single(span),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn token(kind: TokenKind, span: Span) -> TokenTree {
|
||||
TokenTree::Token(Token::new(kind, span))
|
||||
// Create a `TokenTree::Token` with alone spacing.
|
||||
pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
|
||||
TokenTree::Token(Token::new(kind, span), Spacing::Alone)
|
||||
}
|
||||
|
||||
// Create a `TokenTree::Token` with joint spacing.
|
||||
pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
|
||||
TokenTree::Token(Token::new(kind, span), Spacing::Joint)
|
||||
}
|
||||
|
||||
pub fn uninterpolate(self) -> TokenTree {
|
||||
match self {
|
||||
TokenTree::Token(token) => TokenTree::Token(token.uninterpolate().into_owned()),
|
||||
TokenTree::Token(token, spacing) => {
|
||||
TokenTree::Token(token.uninterpolate().into_owned(), spacing)
|
||||
}
|
||||
tt => tt,
|
||||
}
|
||||
}
|
||||
@ -194,13 +206,12 @@ pub fn to_tokenstream(&self) -> TokenStream {
|
||||
.iter()
|
||||
.flat_map(|tree| match &tree.0 {
|
||||
AttrAnnotatedTokenTree::Token(inner) => {
|
||||
smallvec![(TokenTree::Token(inner.clone()), tree.1)].into_iter()
|
||||
smallvec![TokenTree::Token(inner.clone(), tree.1)].into_iter()
|
||||
}
|
||||
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => {
|
||||
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
|
||||
.into_iter()
|
||||
}
|
||||
AttrAnnotatedTokenTree::Delimited(span, delim, stream) => smallvec![(
|
||||
TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),
|
||||
tree.1,
|
||||
)]
|
||||
.into_iter(),
|
||||
AttrAnnotatedTokenTree::Attributes(data) => {
|
||||
let mut outer_attrs = Vec::new();
|
||||
let mut inner_attrs = Vec::new();
|
||||
@ -226,7 +237,7 @@ pub fn to_tokenstream(&self) -> TokenStream {
|
||||
if !inner_attrs.is_empty() {
|
||||
let mut found = false;
|
||||
// Check the last two trees (to account for a trailing semi)
|
||||
for (tree, _) in target_tokens.iter_mut().rev().take(2) {
|
||||
for tree in target_tokens.iter_mut().rev().take(2) {
|
||||
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
|
||||
// Inner attributes are only supported on extern blocks, functions, impls,
|
||||
// and modules. All of these have their inner attributes placed at
|
||||
@ -299,15 +310,13 @@ pub struct AttributesData {
|
||||
/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
|
||||
/// backwards compatibility.
|
||||
#[derive(Clone, Debug, Default, Encodable, Decodable)]
|
||||
pub struct TokenStream(pub(crate) Lrc<Vec<TreeAndSpacing>>);
|
||||
|
||||
pub type TreeAndSpacing = (TokenTree, Spacing);
|
||||
pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
|
||||
|
||||
// `TokenStream` is used a lot. Make sure it doesn't unintentionally get bigger.
|
||||
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
|
||||
rustc_data_structures::static_assert_size!(TokenStream, 8);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
|
||||
pub enum Spacing {
|
||||
Alone,
|
||||
Joint,
|
||||
@ -323,10 +332,10 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
|
||||
while let Some((pos, ts)) = iter.next() {
|
||||
if let Some((_, next)) = iter.peek() {
|
||||
let sp = match (&ts, &next) {
|
||||
(_, (TokenTree::Token(Token { kind: token::Comma, .. }), _)) => continue,
|
||||
(_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
|
||||
(
|
||||
(TokenTree::Token(token_left), Spacing::Alone),
|
||||
(TokenTree::Token(token_right), _),
|
||||
TokenTree::Token(token_left, Spacing::Alone),
|
||||
TokenTree::Token(token_right, _),
|
||||
) if ((token_left.is_ident() && !token_left.is_reserved_ident())
|
||||
|| token_left.is_lit())
|
||||
&& ((token_right.is_ident() && !token_right.is_reserved_ident())
|
||||
@ -334,11 +343,11 @@ pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
|
||||
{
|
||||
token_left.span
|
||||
}
|
||||
((TokenTree::Delimited(sp, ..), Spacing::Alone), _) => sp.entire(),
|
||||
(TokenTree::Delimited(sp, ..), _) => sp.entire(),
|
||||
_ => continue,
|
||||
};
|
||||
let sp = sp.shrink_to_hi();
|
||||
let comma = (TokenTree::token(token::Comma, sp), Spacing::Alone);
|
||||
let comma = TokenTree::token_alone(token::Comma, sp);
|
||||
suggestion = Some((pos, comma, sp));
|
||||
}
|
||||
}
|
||||
@ -360,21 +369,9 @@ fn from((tree, spacing): (AttrAnnotatedTokenTree, Spacing)) -> AttrAnnotatedToke
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TokenTree> for TokenStream {
|
||||
fn from(tree: TokenTree) -> TokenStream {
|
||||
TokenStream::new(vec![(tree, Spacing::Alone)])
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TokenTree> for TreeAndSpacing {
|
||||
fn from(tree: TokenTree) -> TreeAndSpacing {
|
||||
(tree, Spacing::Alone)
|
||||
}
|
||||
}
|
||||
|
||||
impl iter::FromIterator<TokenTree> for TokenStream {
|
||||
fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
|
||||
TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndSpacing>>())
|
||||
TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
|
||||
}
|
||||
}
|
||||
|
||||
@ -387,7 +384,7 @@ fn eq(&self, other: &TokenStream) -> bool {
|
||||
}
|
||||
|
||||
impl TokenStream {
|
||||
pub fn new(streams: Vec<TreeAndSpacing>) -> TokenStream {
|
||||
pub fn new(streams: Vec<TokenTree>) -> TokenStream {
|
||||
TokenStream(Lrc::new(streams))
|
||||
}
|
||||
|
||||
@ -420,13 +417,7 @@ pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
|
||||
}
|
||||
|
||||
pub fn map_enumerated<F: FnMut(usize, &TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
|
||||
TokenStream(Lrc::new(
|
||||
self.0
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, (tree, is_joint))| (f(i, tree), *is_joint))
|
||||
.collect(),
|
||||
))
|
||||
TokenStream(Lrc::new(self.0.iter().enumerate().map(|(i, tree)| f(i, tree)).collect()))
|
||||
}
|
||||
|
||||
fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> {
|
||||
@ -444,6 +435,21 @@ fn opt_from_ast(node: &(impl HasAttrs + HasTokens)) -> Option<TokenStream> {
|
||||
Some(attr_annotated.to_tokenstream())
|
||||
}
|
||||
|
||||
// Create a token stream containing a single token with alone spacing.
|
||||
pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
|
||||
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
|
||||
}
|
||||
|
||||
// Create a token stream containing a single token with joint spacing.
|
||||
pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
|
||||
TokenStream::new(vec![TokenTree::token_joint(kind, span)])
|
||||
}
|
||||
|
||||
// Create a token stream containing a single `Delimited`.
|
||||
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
|
||||
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
|
||||
}
|
||||
|
||||
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
|
||||
TokenStream::opt_from_ast(node)
|
||||
.unwrap_or_else(|| panic!("missing tokens for node at {:?}: {:?}", node.span(), node))
|
||||
@ -452,16 +458,16 @@ pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> To
|
||||
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
|
||||
match nt {
|
||||
Nonterminal::NtIdent(ident, is_raw) => {
|
||||
TokenTree::token(token::Ident(ident.name, *is_raw), ident.span).into()
|
||||
TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
|
||||
}
|
||||
Nonterminal::NtLifetime(ident) => {
|
||||
TokenTree::token(token::Lifetime(ident.name), ident.span).into()
|
||||
TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
|
||||
}
|
||||
Nonterminal::NtItem(item) => TokenStream::from_ast(item),
|
||||
Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
|
||||
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
|
||||
// FIXME: Properly collect tokens for empty statements.
|
||||
TokenTree::token(token::Semi, stmt.span).into()
|
||||
TokenStream::token_alone(token::Semi, stmt.span)
|
||||
}
|
||||
Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
|
||||
Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
|
||||
@ -473,23 +479,23 @@ pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
|
||||
}
|
||||
}
|
||||
|
||||
fn flatten_token(token: &Token) -> TokenTree {
|
||||
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
|
||||
match &token.kind {
|
||||
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => {
|
||||
TokenTree::token(token::Ident(ident.name, is_raw), ident.span)
|
||||
TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
|
||||
}
|
||||
token::Interpolated(nt) => TokenTree::Delimited(
|
||||
DelimSpan::from_single(token.span),
|
||||
Delimiter::Invisible,
|
||||
TokenStream::from_nonterminal_ast(&nt).flattened(),
|
||||
),
|
||||
_ => TokenTree::Token(token.clone()),
|
||||
_ => TokenTree::Token(token.clone(), spacing),
|
||||
}
|
||||
}
|
||||
|
||||
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
|
||||
match tree {
|
||||
TokenTree::Token(token) => TokenStream::flatten_token(token),
|
||||
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
|
||||
TokenTree::Delimited(span, delim, tts) => {
|
||||
TokenTree::Delimited(*span, *delim, tts.flattened())
|
||||
}
|
||||
@ -500,7 +506,7 @@ fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
|
||||
pub fn flattened(&self) -> TokenStream {
|
||||
fn can_skip(stream: &TokenStream) -> bool {
|
||||
stream.trees().all(|tree| match tree {
|
||||
TokenTree::Token(token) => !matches!(token.kind, token::Interpolated(_)),
|
||||
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
|
||||
TokenTree::Delimited(_, _, inner) => can_skip(inner),
|
||||
})
|
||||
}
|
||||
@ -522,8 +528,8 @@ pub fn new() -> TokenStreamBuilder {
|
||||
TokenStreamBuilder(SmallVec::new())
|
||||
}
|
||||
|
||||
pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
|
||||
self.0.push(stream.into());
|
||||
pub fn push(&mut self, stream: TokenStream) {
|
||||
self.0.push(stream);
|
||||
}
|
||||
|
||||
pub fn build(self) -> TokenStream {
|
||||
@ -564,14 +570,14 @@ pub fn build(self) -> TokenStream {
|
||||
// `stream` is not empty and the first tree within it is a
|
||||
// token tree, and (c) the two tokens can be glued
|
||||
// together...
|
||||
if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last()
|
||||
&& let Some((TokenTree::Token(tok), spacing)) = stream.0.first()
|
||||
if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = res_vec_mut.last()
|
||||
&& let Some(TokenTree::Token(tok, spacing)) = stream.0.first()
|
||||
&& let Some(glued_tok) = last_tok.glue(&tok)
|
||||
{
|
||||
// ...then overwrite the last token tree in
|
||||
// `res_vec_mut` with the glued token, and skip the
|
||||
// first token tree from `stream`.
|
||||
*res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing);
|
||||
*res_vec_mut.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
|
||||
res_vec_mut.extend(stream_iter.skip(1));
|
||||
} else {
|
||||
// Append all of `stream`.
|
||||
@ -597,16 +603,8 @@ fn new(stream: &'t TokenStream) -> Self {
|
||||
CursorRef { stream, index: 0 }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn next_with_spacing(&mut self) -> Option<&'t TreeAndSpacing> {
|
||||
self.stream.0.get(self.index).map(|tree| {
|
||||
self.index += 1;
|
||||
tree
|
||||
})
|
||||
}
|
||||
|
||||
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
|
||||
self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
|
||||
self.stream.0.get(self.index + n)
|
||||
}
|
||||
}
|
||||
|
||||
@ -614,7 +612,10 @@ impl<'t> Iterator for CursorRef<'t> {
|
||||
type Item = &'t TokenTree;
|
||||
|
||||
fn next(&mut self) -> Option<&'t TokenTree> {
|
||||
self.next_with_spacing().map(|(tree, _)| tree)
|
||||
self.stream.0.get(self.index).map(|tree| {
|
||||
self.index += 1;
|
||||
tree
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -630,7 +631,10 @@ impl Iterator for Cursor {
|
||||
type Item = TokenTree;
|
||||
|
||||
fn next(&mut self) -> Option<TokenTree> {
|
||||
self.next_with_spacing().map(|(tree, _)| tree)
|
||||
self.stream.0.get(self.index).map(|tree| {
|
||||
self.index += 1;
|
||||
tree.clone()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@ -640,15 +644,7 @@ fn new(stream: TokenStream) -> Self {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn next_with_spacing(&mut self) -> Option<TreeAndSpacing> {
|
||||
self.stream.0.get(self.index).map(|tree| {
|
||||
self.index += 1;
|
||||
tree.clone()
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn next_with_spacing_ref(&mut self) -> Option<&TreeAndSpacing> {
|
||||
pub fn next_ref(&mut self) -> Option<&TokenTree> {
|
||||
self.stream.0.get(self.index).map(|tree| {
|
||||
self.index += 1;
|
||||
tree
|
||||
@ -656,7 +652,7 @@ pub fn next_with_spacing_ref(&mut self) -> Option<&TreeAndSpacing> {
|
||||
}
|
||||
|
||||
pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
|
||||
self.stream.0[self.index..].get(n).map(|(tree, _)| tree)
|
||||
self.stream.0.get(self.index + n)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -145,7 +145,7 @@ pub fn print_crate<'a>(
|
||||
/// This makes printed token streams look slightly nicer,
|
||||
/// and also addresses some specific regressions described in #63896 and #73345.
|
||||
fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
|
||||
if let TokenTree::Token(token) = prev {
|
||||
if let TokenTree::Token(token, _) = prev {
|
||||
if matches!(token.kind, token::Dot | token::Dollar) {
|
||||
return false;
|
||||
}
|
||||
@ -154,12 +154,12 @@ fn tt_prepend_space(tt: &TokenTree, prev: &TokenTree) -> bool {
|
||||
}
|
||||
}
|
||||
match tt {
|
||||
TokenTree::Token(token) => !matches!(token.kind, token::Comma | token::Not | token::Dot),
|
||||
TokenTree::Token(token, _) => !matches!(token.kind, token::Comma | token::Not | token::Dot),
|
||||
TokenTree::Delimited(_, Delimiter::Parenthesis, _) => {
|
||||
!matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }))
|
||||
!matches!(prev, TokenTree::Token(Token { kind: token::Ident(..), .. }, _))
|
||||
}
|
||||
TokenTree::Delimited(_, Delimiter::Bracket, _) => {
|
||||
!matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. }))
|
||||
!matches!(prev, TokenTree::Token(Token { kind: token::Pound, .. }, _))
|
||||
}
|
||||
TokenTree::Delimited(..) => true,
|
||||
}
|
||||
@ -526,7 +526,7 @@ fn print_meta_item(&mut self, item: &ast::MetaItem) {
|
||||
/// expression arguments as expressions). It can be done! I think.
|
||||
fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) {
|
||||
match tt {
|
||||
TokenTree::Token(token) => {
|
||||
TokenTree::Token(token, _) => {
|
||||
let token_str = self.token_to_string_ext(&token, convert_dollar_crate);
|
||||
self.word(token_str);
|
||||
if let token::DocComment(..) = token.kind {
|
||||
|
||||
@ -152,7 +152,7 @@ fn build_unlikely(&self, cond_expr: P<Expr>) -> P<Expr> {
|
||||
fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
|
||||
let escaped_expr_str = escape_to_fmt(expr_str);
|
||||
let initial = [
|
||||
TokenTree::token(
|
||||
TokenTree::token_alone(
|
||||
token::Literal(token::Lit {
|
||||
kind: token::LitKind::Str,
|
||||
symbol: Symbol::intern(&if self.fmt_string.is_empty() {
|
||||
@ -167,12 +167,12 @@ fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
|
||||
}),
|
||||
self.span,
|
||||
),
|
||||
TokenTree::token(token::Comma, self.span),
|
||||
TokenTree::token_alone(token::Comma, self.span),
|
||||
];
|
||||
let captures = self.capture_decls.iter().flat_map(|cap| {
|
||||
[
|
||||
TokenTree::token(token::Ident(cap.ident.name, false), cap.ident.span),
|
||||
TokenTree::token(token::Comma, self.span),
|
||||
TokenTree::token_alone(token::Ident(cap.ident.name, false), cap.ident.span),
|
||||
TokenTree::token_alone(token::Comma, self.span),
|
||||
]
|
||||
});
|
||||
self.cx.expr(
|
||||
|
||||
@ -20,14 +20,14 @@ pub fn expand_concat_idents<'cx>(
|
||||
for (i, e) in tts.into_trees().enumerate() {
|
||||
if i & 1 == 1 {
|
||||
match e {
|
||||
TokenTree::Token(Token { kind: token::Comma, .. }) => {}
|
||||
TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}
|
||||
_ => {
|
||||
cx.span_err(sp, "concat_idents! expecting comma");
|
||||
return DummyResult::any(sp);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if let TokenTree::Token(token) = e {
|
||||
if let TokenTree::Token(token, _) = e {
|
||||
if let Some((ident, _)) = token.ident() {
|
||||
res_str.push_str(ident.name.as_str());
|
||||
continue;
|
||||
|
||||
@ -11,8 +11,8 @@ pub fn expand_trace_macros(
|
||||
let mut cursor = tt.into_trees();
|
||||
let mut err = false;
|
||||
let value = match &cursor.next() {
|
||||
Some(TokenTree::Token(token)) if token.is_keyword(kw::True) => true,
|
||||
Some(TokenTree::Token(token)) if token.is_keyword(kw::False) => false,
|
||||
Some(TokenTree::Token(token, _)) if token.is_keyword(kw::True) => true,
|
||||
Some(TokenTree::Token(token, _)) if token.is_keyword(kw::False) => false,
|
||||
_ => {
|
||||
err = true;
|
||||
false
|
||||
|
||||
@ -401,7 +401,7 @@ fn expand_cfg_attr_item(
|
||||
// Use the `#` in `#[cfg_attr(pred, attr)]` as the `#` token
|
||||
// for `attr` when we expand it to `#[attr]`
|
||||
let mut orig_trees = orig_tokens.into_trees();
|
||||
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }) = orig_trees.next().unwrap() else {
|
||||
let TokenTree::Token(pound_token @ Token { kind: TokenKind::Pound, .. }, _) = orig_trees.next().unwrap() else {
|
||||
panic!("Bad tokens for attribute {:?}", attr);
|
||||
};
|
||||
let pound_span = pound_token.span;
|
||||
@ -409,7 +409,7 @@ fn expand_cfg_attr_item(
|
||||
let mut trees = vec![(AttrAnnotatedTokenTree::Token(pound_token), Spacing::Alone)];
|
||||
if attr.style == AttrStyle::Inner {
|
||||
// For inner attributes, we do the same thing for the `!` in `#![some_attr]`
|
||||
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }) = orig_trees.next().unwrap() else {
|
||||
let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) = orig_trees.next().unwrap() else {
|
||||
panic!("Bad tokens for attribute {:?}", attr);
|
||||
};
|
||||
trees.push((AttrAnnotatedTokenTree::Token(bang_token), Spacing::Alone));
|
||||
|
||||
@ -481,7 +481,7 @@ pub fn compile_declarative_macro(
|
||||
.map(|m| {
|
||||
if let MatchedTokenTree(ref tt) = *m {
|
||||
let tt = mbe::quoted::parse(
|
||||
tt.clone().into(),
|
||||
TokenStream::new(vec![tt.clone()]),
|
||||
true,
|
||||
&sess.parse_sess,
|
||||
def.id,
|
||||
@ -505,7 +505,7 @@ pub fn compile_declarative_macro(
|
||||
.map(|m| {
|
||||
if let MatchedTokenTree(ref tt) = *m {
|
||||
return mbe::quoted::parse(
|
||||
tt.clone().into(),
|
||||
TokenStream::new(vec![tt.clone()]),
|
||||
false,
|
||||
&sess.parse_sess,
|
||||
def.id,
|
||||
|
||||
@ -106,7 +106,7 @@ fn parse_depth<'sess>(
|
||||
let Some(tt) = iter.next() else { return Ok(0) };
|
||||
let TokenTree::Token(token::Token {
|
||||
kind: token::TokenKind::Literal(lit), ..
|
||||
}) = tt else {
|
||||
}, _) = tt else {
|
||||
return Err(sess.span_diagnostic.struct_span_err(
|
||||
span,
|
||||
"meta-variable expression depth must be a literal"
|
||||
@ -130,7 +130,7 @@ fn parse_ident<'sess>(
|
||||
sess: &'sess ParseSess,
|
||||
span: Span,
|
||||
) -> PResult<'sess, Ident> {
|
||||
if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt {
|
||||
if let Some(tt) = iter.next() && let TokenTree::Token(token, _) = tt {
|
||||
if let Some((elem, false)) = token.ident() {
|
||||
return Ok(elem);
|
||||
}
|
||||
@ -153,7 +153,7 @@ fn parse_ident<'sess>(
|
||||
/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
|
||||
/// iterator is not modified and the result is `false`.
|
||||
fn try_eat_comma(iter: &mut CursorRef<'_>) -> bool {
|
||||
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) {
|
||||
if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. }, _)) = iter.look_ahead(0) {
|
||||
let _ = iter.next();
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -56,9 +56,9 @@ pub(super) fn parse(
|
||||
match tree {
|
||||
TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
|
||||
let span = match trees.next() {
|
||||
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => {
|
||||
Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span }, _)) => {
|
||||
match trees.next() {
|
||||
Some(tokenstream::TokenTree::Token(token)) => match token.ident() {
|
||||
Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
|
||||
Some((frag, _)) => {
|
||||
let span = token.span.with_lo(start_sp.lo());
|
||||
|
||||
@ -146,7 +146,7 @@ fn parse_tree(
|
||||
// Depending on what `tree` is, we could be parsing different parts of a macro
|
||||
match tree {
|
||||
// `tree` is a `$` token. Look at the next token in `trees`
|
||||
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
|
||||
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _) => {
|
||||
// FIXME: Handle `Invisible`-delimited groups in a more systematic way
|
||||
// during parsing.
|
||||
let mut next = outer_trees.next();
|
||||
@ -217,7 +217,7 @@ fn parse_tree(
|
||||
|
||||
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate`
|
||||
// special metavariable that names the crate of the invocation.
|
||||
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
|
||||
Some(tokenstream::TokenTree::Token(token, _)) if token.is_ident() => {
|
||||
let (ident, is_raw) = token.ident().unwrap();
|
||||
let span = ident.span.with_lo(span.lo());
|
||||
if ident.name == kw::Crate && !is_raw {
|
||||
@ -228,7 +228,7 @@ fn parse_tree(
|
||||
}
|
||||
|
||||
// `tree` is followed by another `$`. This is an escaped `$`.
|
||||
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span })) => {
|
||||
Some(tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }, _)) => {
|
||||
if parsing_patterns {
|
||||
span_dollar_dollar_or_metavar_in_the_lhs_err(
|
||||
sess,
|
||||
@ -241,7 +241,7 @@ fn parse_tree(
|
||||
}
|
||||
|
||||
// `tree` is followed by some other token. This is an error.
|
||||
Some(tokenstream::TokenTree::Token(token)) => {
|
||||
Some(tokenstream::TokenTree::Token(token, _)) => {
|
||||
let msg = format!(
|
||||
"expected identifier, found `{}`",
|
||||
pprust::token_to_string(&token),
|
||||
@ -256,7 +256,7 @@ fn parse_tree(
|
||||
}
|
||||
|
||||
// `tree` is an arbitrary token. Keep it.
|
||||
tokenstream::TokenTree::Token(token) => TokenTree::Token(token),
|
||||
tokenstream::TokenTree::Token(token, _) => TokenTree::Token(token),
|
||||
|
||||
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
|
||||
// descend into the delimited set and further parse it.
|
||||
@ -291,7 +291,7 @@ fn parse_kleene_op(
|
||||
span: Span,
|
||||
) -> Result<Result<(KleeneOp, Span), Token>, Span> {
|
||||
match input.next() {
|
||||
Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) {
|
||||
Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
|
||||
Some(op) => Ok(Ok((op, token.span))),
|
||||
None => Ok(Err(token)),
|
||||
},
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
use crate::mbe::{self, MetaVarExpr};
|
||||
use rustc_ast::mut_visit::{self, MutVisitor};
|
||||
use rustc_ast::token::{self, Delimiter, Token, TokenKind};
|
||||
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndSpacing};
|
||||
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use rustc_errors::{pluralize, PResult};
|
||||
use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
|
||||
@ -105,7 +105,7 @@ pub(super) fn transcribe<'a>(
|
||||
//
|
||||
// Thus, if we try to pop the `result_stack` and it is empty, we have reached the top-level
|
||||
// again, and we are done transcribing.
|
||||
let mut result: Vec<TreeAndSpacing> = Vec::new();
|
||||
let mut result: Vec<TokenTree> = Vec::new();
|
||||
let mut result_stack = Vec::new();
|
||||
let mut marker = Marker(cx.current_expansion.id, transparency);
|
||||
|
||||
@ -123,7 +123,7 @@ pub(super) fn transcribe<'a>(
|
||||
if repeat_idx < repeat_len {
|
||||
*idx = 0;
|
||||
if let Some(sep) = sep {
|
||||
result.push(TokenTree::Token(sep.clone()).into());
|
||||
result.push(TokenTree::Token(sep.clone(), Spacing::Alone));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@ -150,7 +150,7 @@ pub(super) fn transcribe<'a>(
|
||||
// Step back into the parent Delimited.
|
||||
let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
|
||||
result = result_stack.pop().unwrap();
|
||||
result.push(tree.into());
|
||||
result.push(tree);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
@ -227,15 +227,15 @@ pub(super) fn transcribe<'a>(
|
||||
// `tt`s are emitted into the output stream directly as "raw tokens",
|
||||
// without wrapping them into groups.
|
||||
let token = tt.clone();
|
||||
result.push(token.into());
|
||||
result.push(token);
|
||||
}
|
||||
MatchedNonterminal(ref nt) => {
|
||||
// Other variables are emitted into the output stream as groups with
|
||||
// `Delimiter::Invisible` to maintain parsing priorities.
|
||||
// `Interpolated` is currently used for such groups in rustc parser.
|
||||
marker.visit_span(&mut sp);
|
||||
let token = TokenTree::token(token::Interpolated(nt.clone()), sp);
|
||||
result.push(token.into());
|
||||
let token = TokenTree::token_alone(token::Interpolated(nt.clone()), sp);
|
||||
result.push(token);
|
||||
}
|
||||
MatchedSeq(..) => {
|
||||
// We were unable to descend far enough. This is an error.
|
||||
@ -250,8 +250,11 @@ pub(super) fn transcribe<'a>(
|
||||
// with modified syntax context. (I believe this supports nested macros).
|
||||
marker.visit_span(&mut sp);
|
||||
marker.visit_ident(&mut original_ident);
|
||||
result.push(TokenTree::token(token::Dollar, sp).into());
|
||||
result.push(TokenTree::Token(Token::from_ast_ident(original_ident)).into());
|
||||
result.push(TokenTree::token_alone(token::Dollar, sp));
|
||||
result.push(TokenTree::Token(
|
||||
Token::from_ast_ident(original_ident),
|
||||
Spacing::Alone,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@ -281,8 +284,8 @@ pub(super) fn transcribe<'a>(
|
||||
mbe::TokenTree::Token(token) => {
|
||||
let mut token = token.clone();
|
||||
mut_visit::visit_token(&mut token, &mut marker);
|
||||
let tt = TokenTree::Token(token);
|
||||
result.push(tt.into());
|
||||
let tt = TokenTree::Token(token, Spacing::Alone);
|
||||
result.push(tt);
|
||||
}
|
||||
|
||||
// There should be no meta-var declarations in the invocation of a macro.
|
||||
@ -532,7 +535,7 @@ fn transcribe_metavar_expr<'a>(
|
||||
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
|
||||
marker: &mut Marker,
|
||||
repeats: &[(usize, usize)],
|
||||
result: &mut Vec<TreeAndSpacing>,
|
||||
result: &mut Vec<TokenTree>,
|
||||
sp: &DelimSpan,
|
||||
) -> PResult<'a, ()> {
|
||||
let mut visited_span = || {
|
||||
@ -544,11 +547,11 @@ fn transcribe_metavar_expr<'a>(
|
||||
MetaVarExpr::Count(original_ident, depth_opt) => {
|
||||
let matched = matched_from_ident(cx, original_ident, interp)?;
|
||||
let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?;
|
||||
let tt = TokenTree::token(
|
||||
let tt = TokenTree::token_alone(
|
||||
TokenKind::lit(token::Integer, sym::integer(count), None),
|
||||
visited_span(),
|
||||
);
|
||||
result.push(tt.into());
|
||||
result.push(tt);
|
||||
}
|
||||
MetaVarExpr::Ignore(original_ident) => {
|
||||
// Used to ensure that `original_ident` is present in the LHS
|
||||
@ -556,25 +559,19 @@ fn transcribe_metavar_expr<'a>(
|
||||
}
|
||||
MetaVarExpr::Index(depth) => match repeats.iter().nth_back(depth) {
|
||||
Some((index, _)) => {
|
||||
result.push(
|
||||
TokenTree::token(
|
||||
TokenKind::lit(token::Integer, sym::integer(*index), None),
|
||||
visited_span(),
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
result.push(TokenTree::token_alone(
|
||||
TokenKind::lit(token::Integer, sym::integer(*index), None),
|
||||
visited_span(),
|
||||
));
|
||||
}
|
||||
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "index")),
|
||||
},
|
||||
MetaVarExpr::Length(depth) => match repeats.iter().nth_back(depth) {
|
||||
Some((_, length)) => {
|
||||
result.push(
|
||||
TokenTree::token(
|
||||
TokenKind::lit(token::Integer, sym::integer(*length), None),
|
||||
visited_span(),
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
result.push(TokenTree::token_alone(
|
||||
TokenKind::lit(token::Integer, sym::integer(*length), None),
|
||||
visited_span(),
|
||||
));
|
||||
}
|
||||
None => return Err(out_of_bounds_err(cx, repeats.len(), sp.entire(), "length")),
|
||||
},
|
||||
|
||||
@ -66,23 +66,23 @@ fn string_to_tts_macro() {
|
||||
|
||||
match tts {
|
||||
[
|
||||
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }),
|
||||
TokenTree::Token(Token { kind: token::Not, .. }),
|
||||
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }),
|
||||
TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
|
||||
TokenTree::Token(Token { kind: token::Not, .. }, _),
|
||||
TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
|
||||
TokenTree::Delimited(_, macro_delim, macro_tts),
|
||||
] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
|
||||
let tts = ¯o_tts.trees().collect::<Vec<_>>();
|
||||
match &tts[..] {
|
||||
[
|
||||
TokenTree::Delimited(_, first_delim, first_tts),
|
||||
TokenTree::Token(Token { kind: token::FatArrow, .. }),
|
||||
TokenTree::Token(Token { kind: token::FatArrow, .. }, _),
|
||||
TokenTree::Delimited(_, second_delim, second_tts),
|
||||
] if macro_delim == &Delimiter::Parenthesis => {
|
||||
let tts = &first_tts.trees().collect::<Vec<_>>();
|
||||
match &tts[..] {
|
||||
[
|
||||
TokenTree::Token(Token { kind: token::Dollar, .. }),
|
||||
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
|
||||
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
|
||||
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
|
||||
] if first_delim == &Delimiter::Parenthesis && name.as_str() == "a" => {
|
||||
}
|
||||
_ => panic!("value 3: {:?} {:?}", first_delim, first_tts),
|
||||
@ -90,8 +90,8 @@ fn string_to_tts_macro() {
|
||||
let tts = &second_tts.trees().collect::<Vec<_>>();
|
||||
match &tts[..] {
|
||||
[
|
||||
TokenTree::Token(Token { kind: token::Dollar, .. }),
|
||||
TokenTree::Token(Token { kind: token::Ident(name, false), .. }),
|
||||
TokenTree::Token(Token { kind: token::Dollar, .. }, _),
|
||||
TokenTree::Token(Token { kind: token::Ident(name, false), .. }, _),
|
||||
] if second_delim == &Delimiter::Parenthesis
|
||||
&& name.as_str() == "a" => {}
|
||||
_ => panic!("value 4: {:?} {:?}", second_delim, second_tts),
|
||||
@ -111,29 +111,27 @@ fn string_to_tts_1() {
|
||||
let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
|
||||
|
||||
let expected = TokenStream::new(vec![
|
||||
TokenTree::token(token::Ident(kw::Fn, false), sp(0, 2)).into(),
|
||||
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(3, 4)).into(),
|
||||
TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)),
|
||||
TokenTree::token_alone(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
|
||||
TokenTree::Delimited(
|
||||
DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
|
||||
Delimiter::Parenthesis,
|
||||
TokenStream::new(vec![
|
||||
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(6, 7)).into(),
|
||||
TokenTree::token(token::Colon, sp(8, 9)).into(),
|
||||
TokenTree::token(token::Ident(sym::i32, false), sp(10, 13)).into(),
|
||||
TokenTree::token_alone(token::Ident(Symbol::intern("b"), false), sp(6, 7)),
|
||||
TokenTree::token_alone(token::Colon, sp(8, 9)),
|
||||
TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)),
|
||||
])
|
||||
.into(),
|
||||
)
|
||||
.into(),
|
||||
),
|
||||
TokenTree::Delimited(
|
||||
DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
|
||||
Delimiter::Brace,
|
||||
TokenStream::new(vec![
|
||||
TokenTree::token(token::Ident(Symbol::intern("b"), false), sp(17, 18)).into(),
|
||||
TokenTree::token(token::Semi, sp(18, 19)).into(),
|
||||
TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(17, 18)),
|
||||
TokenTree::token_alone(token::Semi, sp(18, 19)),
|
||||
])
|
||||
.into(),
|
||||
)
|
||||
.into(),
|
||||
),
|
||||
]);
|
||||
|
||||
assert_eq!(tts, expected);
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
use rustc_ast as ast;
|
||||
use rustc_ast::ptr::P;
|
||||
use rustc_ast::token;
|
||||
use rustc_ast::tokenstream::{TokenStream, TokenTree};
|
||||
use rustc_ast::tokenstream::TokenStream;
|
||||
use rustc_data_structures::sync::Lrc;
|
||||
use rustc_errors::ErrorGuaranteed;
|
||||
use rustc_parse::parser::ForceCollect;
|
||||
@ -123,7 +123,7 @@ fn expand(
|
||||
Annotatable::Stmt(stmt) => token::NtStmt(stmt),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
TokenTree::token(token::Interpolated(Lrc::new(nt)), DUMMY_SP).into()
|
||||
TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
|
||||
} else {
|
||||
item.to_tokens()
|
||||
};
|
||||
|
||||
@ -92,9 +92,8 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
|
||||
let mut trees = Vec::with_capacity(stream.len().next_power_of_two());
|
||||
let mut cursor = stream.into_trees();
|
||||
|
||||
while let Some((tree, spacing)) = cursor.next_with_spacing() {
|
||||
let joint = spacing == Joint;
|
||||
let Token { kind, span } = match tree {
|
||||
while let Some(tree) = cursor.next() {
|
||||
let (Token { kind, span }, joint) = match tree {
|
||||
tokenstream::TokenTree::Delimited(span, delim, tts) => {
|
||||
let delimiter = pm::Delimiter::from_internal(delim);
|
||||
trees.push(TokenTree::Group(Group {
|
||||
@ -108,7 +107,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
|
||||
}));
|
||||
continue;
|
||||
}
|
||||
tokenstream::TokenTree::Token(token) => token,
|
||||
tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
|
||||
};
|
||||
|
||||
let mut op = |s: &str| {
|
||||
@ -194,7 +193,7 @@ fn from_internal((stream, rustc): (TokenStream, &mut Rustc<'_, '_>)) -> Self {
|
||||
TokenKind::lit(token::Str, Symbol::intern(&escaped), None),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|kind| tokenstream::TokenTree::token(kind, span))
|
||||
.map(|kind| tokenstream::TokenTree::token_alone(kind, span))
|
||||
.collect();
|
||||
trees.push(TokenTree::Punct(Punct { ch: b'#', joint: false, span }));
|
||||
if attr_style == ast::AttrStyle::Inner {
|
||||
@ -246,16 +245,15 @@ fn to_internal(self) -> TokenStream {
|
||||
let (ch, joint, span) = match tree {
|
||||
TokenTree::Punct(Punct { ch, joint, span }) => (ch, joint, span),
|
||||
TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
|
||||
return tokenstream::TokenTree::Delimited(
|
||||
return tokenstream::TokenStream::delimited(
|
||||
tokenstream::DelimSpan { open, close },
|
||||
delimiter.to_internal(),
|
||||
stream.unwrap_or_default(),
|
||||
)
|
||||
.into();
|
||||
);
|
||||
}
|
||||
TokenTree::Ident(self::Ident { sym, is_raw, span }) => {
|
||||
rustc.sess().symbol_gallery.insert(sym, span);
|
||||
return tokenstream::TokenTree::token(Ident(sym, is_raw), span).into();
|
||||
return tokenstream::TokenStream::token_alone(Ident(sym, is_raw), span);
|
||||
}
|
||||
TokenTree::Literal(self::Literal {
|
||||
kind: self::LitKind::Integer,
|
||||
@ -266,8 +264,8 @@ fn to_internal(self) -> TokenStream {
|
||||
let minus = BinOp(BinOpToken::Minus);
|
||||
let symbol = Symbol::intern(&symbol.as_str()[1..]);
|
||||
let integer = TokenKind::lit(token::Integer, symbol, suffix);
|
||||
let a = tokenstream::TokenTree::token(minus, span);
|
||||
let b = tokenstream::TokenTree::token(integer, span);
|
||||
let a = tokenstream::TokenTree::token_alone(minus, span);
|
||||
let b = tokenstream::TokenTree::token_alone(integer, span);
|
||||
return [a, b].into_iter().collect();
|
||||
}
|
||||
TokenTree::Literal(self::Literal {
|
||||
@ -279,16 +277,15 @@ fn to_internal(self) -> TokenStream {
|
||||
let minus = BinOp(BinOpToken::Minus);
|
||||
let symbol = Symbol::intern(&symbol.as_str()[1..]);
|
||||
let float = TokenKind::lit(token::Float, symbol, suffix);
|
||||
let a = tokenstream::TokenTree::token(minus, span);
|
||||
let b = tokenstream::TokenTree::token(float, span);
|
||||
let a = tokenstream::TokenTree::token_alone(minus, span);
|
||||
let b = tokenstream::TokenTree::token_alone(float, span);
|
||||
return [a, b].into_iter().collect();
|
||||
}
|
||||
TokenTree::Literal(self::Literal { kind, symbol, suffix, span }) => {
|
||||
return tokenstream::TokenTree::token(
|
||||
return tokenstream::TokenStream::token_alone(
|
||||
TokenKind::lit(kind.to_internal(), symbol, suffix),
|
||||
span,
|
||||
)
|
||||
.into();
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@ -318,8 +315,11 @@ fn to_internal(self) -> TokenStream {
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
let tree = tokenstream::TokenTree::token(kind, span);
|
||||
TokenStream::new(vec![(tree, if joint { Joint } else { Alone })])
|
||||
if joint {
|
||||
tokenstream::TokenStream::token_joint(kind, span)
|
||||
} else {
|
||||
tokenstream::TokenStream::token_alone(kind, span)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -486,12 +486,11 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStrea
|
||||
// We don't use `TokenStream::from_ast` as the tokenstream currently cannot
|
||||
// be recovered in the general case.
|
||||
match &expr.kind {
|
||||
ast::ExprKind::Lit(l) if l.token.kind == token::Bool => {
|
||||
Ok(tokenstream::TokenTree::token(token::Ident(l.token.symbol, false), l.span)
|
||||
.into())
|
||||
}
|
||||
ast::ExprKind::Lit(l) if l.token.kind == token::Bool => Ok(
|
||||
tokenstream::TokenStream::token_alone(token::Ident(l.token.symbol, false), l.span),
|
||||
),
|
||||
ast::ExprKind::Lit(l) => {
|
||||
Ok(tokenstream::TokenTree::token(token::Literal(l.token), l.span).into())
|
||||
Ok(tokenstream::TokenStream::token_alone(token::Literal(l.token), l.span))
|
||||
}
|
||||
ast::ExprKind::Unary(ast::UnOp::Neg, e) => match &e.kind {
|
||||
ast::ExprKind::Lit(l) => match l.token {
|
||||
@ -499,8 +498,8 @@ fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result<Self::TokenStrea
|
||||
Ok(Self::TokenStream::from_iter([
|
||||
// FIXME: The span of the `-` token is lost when
|
||||
// parsing, so we cannot faithfully recover it here.
|
||||
tokenstream::TokenTree::token(token::BinOp(token::Minus), e.span),
|
||||
tokenstream::TokenTree::token(token::Literal(l.token), l.span),
|
||||
tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
|
||||
tokenstream::TokenTree::token_alone(token::Literal(l.token), l.span),
|
||||
]))
|
||||
}
|
||||
_ => Err(()),
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
use crate::tests::string_to_stream;
|
||||
|
||||
use rustc_ast::token;
|
||||
use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree};
|
||||
use rustc_ast::tokenstream::{TokenStream, TokenStreamBuilder};
|
||||
use rustc_span::create_default_session_globals_then;
|
||||
use rustc_span::{BytePos, Span, Symbol};
|
||||
|
||||
@ -13,10 +13,6 @@ fn sp(a: u32, b: u32) -> Span {
|
||||
Span::with_root_ctxt(BytePos(a), BytePos(b))
|
||||
}
|
||||
|
||||
fn joint(tree: TokenTree) -> TokenStream {
|
||||
TokenStream::new(vec![(tree, Spacing::Joint)])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_concat() {
|
||||
create_default_session_globals_then(|| {
|
||||
@ -90,9 +86,8 @@ fn test_diseq_1() {
|
||||
#[test]
|
||||
fn test_is_empty() {
|
||||
create_default_session_globals_then(|| {
|
||||
let test0: TokenStream = Vec::<TokenTree>::new().into_iter().collect();
|
||||
let test1: TokenStream =
|
||||
TokenTree::token(token::Ident(Symbol::intern("a"), false), sp(0, 1)).into();
|
||||
let test0 = TokenStream::default();
|
||||
let test1 = TokenStream::token_alone(token::Ident(Symbol::intern("a"), false), sp(0, 1));
|
||||
let test2 = string_to_ts("foo(bar::baz)");
|
||||
|
||||
assert_eq!(test0.is_empty(), true);
|
||||
@ -105,9 +100,9 @@ fn test_is_empty() {
|
||||
fn test_dotdotdot() {
|
||||
create_default_session_globals_then(|| {
|
||||
let mut builder = TokenStreamBuilder::new();
|
||||
builder.push(joint(TokenTree::token(token::Dot, sp(0, 1))));
|
||||
builder.push(joint(TokenTree::token(token::Dot, sp(1, 2))));
|
||||
builder.push(TokenTree::token(token::Dot, sp(2, 3)));
|
||||
builder.push(TokenStream::token_joint(token::Dot, sp(0, 1)));
|
||||
builder.push(TokenStream::token_joint(token::Dot, sp(1, 2)));
|
||||
builder.push(TokenStream::token_alone(token::Dot, sp(2, 3)));
|
||||
let stream = builder.build();
|
||||
assert!(stream.eq_unspanned(&string_to_ts("...")));
|
||||
assert_eq!(stream.trees().count(), 1);
|
||||
|
||||
@ -1974,7 +1974,7 @@ fn check_tokens(&mut self, cx: &EarlyContext<'_>, tokens: TokenStream) {
|
||||
for tt in tokens.into_trees() {
|
||||
match tt {
|
||||
// Only report non-raw idents.
|
||||
TokenTree::Token(token) => {
|
||||
TokenTree::Token(token, _) => {
|
||||
if let Some((ident, false)) = token.ident() {
|
||||
self.check_ident_token(cx, UnderMacro(true), ident);
|
||||
}
|
||||
|
||||
@ -1,11 +1,7 @@
|
||||
use super::{StringReader, UnmatchedBrace};
|
||||
|
||||
use rustc_ast::token::{self, Delimiter, Token};
|
||||
use rustc_ast::tokenstream::{
|
||||
DelimSpan,
|
||||
Spacing::{self, *},
|
||||
TokenStream, TokenTree, TreeAndSpacing,
|
||||
};
|
||||
use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
|
||||
use rustc_ast_pretty::pprust::token_to_string;
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use rustc_errors::PResult;
|
||||
@ -77,7 +73,7 @@ fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> {
|
||||
fn parse_token_tree(&mut self) -> PResult<'a, TokenTree> {
|
||||
let sm = self.string_reader.sess.source_map();
|
||||
|
||||
match self.token.kind {
|
||||
@ -223,7 +219,7 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> {
|
||||
_ => {}
|
||||
}
|
||||
|
||||
Ok(TokenTree::Delimited(delim_span, delim, tts).into())
|
||||
Ok(TokenTree::Delimited(delim_span, delim, tts))
|
||||
}
|
||||
token::CloseDelim(delim) => {
|
||||
// An unexpected closing delimiter (i.e., there is no
|
||||
@ -258,12 +254,12 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndSpacing> {
|
||||
Err(err)
|
||||
}
|
||||
_ => {
|
||||
let tt = TokenTree::Token(self.token.take());
|
||||
let tok = self.token.take();
|
||||
let mut spacing = self.bump();
|
||||
if !self.token.is_op() {
|
||||
spacing = Alone;
|
||||
spacing = Spacing::Alone;
|
||||
}
|
||||
Ok((tt, spacing))
|
||||
Ok(TokenTree::Token(tok, spacing))
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -277,20 +273,20 @@ fn bump(&mut self) -> Spacing {
|
||||
|
||||
#[derive(Default)]
|
||||
struct TokenStreamBuilder {
|
||||
buf: Vec<TreeAndSpacing>,
|
||||
buf: Vec<TokenTree>,
|
||||
}
|
||||
|
||||
impl TokenStreamBuilder {
|
||||
fn push(&mut self, (tree, joint): TreeAndSpacing) {
|
||||
if let Some((TokenTree::Token(prev_token), Joint)) = self.buf.last()
|
||||
&& let TokenTree::Token(token) = &tree
|
||||
fn push(&mut self, tree: TokenTree) {
|
||||
if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
|
||||
&& let TokenTree::Token(token, joint) = &tree
|
||||
&& let Some(glued) = prev_token.glue(token)
|
||||
{
|
||||
self.buf.pop();
|
||||
self.buf.push((TokenTree::Token(glued), joint));
|
||||
self.buf.push(TokenTree::Token(glued, *joint));
|
||||
return;
|
||||
}
|
||||
self.buf.push((tree, joint))
|
||||
self.buf.push(tree);
|
||||
}
|
||||
|
||||
fn into_token_stream(self) -> TokenStream {
|
||||
|
||||
@ -1664,8 +1664,8 @@ fn parse_item_decl_macro(&mut self, lo: Span) -> PResult<'a, ItemInfo> {
|
||||
let body = self.parse_token_tree(); // `MacBody`
|
||||
// Convert `MacParams MacBody` into `{ MacParams => MacBody }`.
|
||||
let bspan = body.span();
|
||||
let arrow = TokenTree::token(token::FatArrow, pspan.between(bspan)); // `=>`
|
||||
let tokens = TokenStream::new(vec![params.into(), arrow.into(), body.into()]);
|
||||
let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>`
|
||||
let tokens = TokenStream::new(vec![params, arrow, body]);
|
||||
let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
|
||||
P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
|
||||
} else {
|
||||
|
||||
@ -268,13 +268,13 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
|
||||
// FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
|
||||
// need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
|
||||
// removed.
|
||||
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() {
|
||||
if let Some(tree) = self.frame.tree_cursor.next_ref() {
|
||||
match tree {
|
||||
&TokenTree::Token(ref token) => match (desugar_doc_comments, token) {
|
||||
&TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) {
|
||||
(true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
|
||||
return self.desugar(attr_style, data, span);
|
||||
}
|
||||
_ => return (token.clone(), *spacing),
|
||||
_ => return (token.clone(), spacing),
|
||||
},
|
||||
&TokenTree::Delimited(sp, delim, ref tts) => {
|
||||
// Set `open_delim` to true here because we deal with it immediately.
|
||||
@ -318,12 +318,14 @@ fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token
|
||||
delim_span,
|
||||
Delimiter::Bracket,
|
||||
[
|
||||
TokenTree::token(token::Ident(sym::doc, false), span),
|
||||
TokenTree::token(token::Eq, span),
|
||||
TokenTree::token(TokenKind::lit(token::StrRaw(num_of_hashes), data, None), span),
|
||||
TokenTree::token_alone(token::Ident(sym::doc, false), span),
|
||||
TokenTree::token_alone(token::Eq, span),
|
||||
TokenTree::token_alone(
|
||||
TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
|
||||
span,
|
||||
),
|
||||
]
|
||||
.iter()
|
||||
.cloned()
|
||||
.into_iter()
|
||||
.collect::<TokenStream>(),
|
||||
);
|
||||
|
||||
@ -332,14 +334,16 @@ fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token
|
||||
TokenCursorFrame::new(
|
||||
None,
|
||||
if attr_style == AttrStyle::Inner {
|
||||
[TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body]
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect::<TokenStream>()
|
||||
[
|
||||
TokenTree::token_alone(token::Pound, span),
|
||||
TokenTree::token_alone(token::Not, span),
|
||||
body,
|
||||
]
|
||||
.into_iter()
|
||||
.collect::<TokenStream>()
|
||||
} else {
|
||||
[TokenTree::token(token::Pound, span), body]
|
||||
.iter()
|
||||
.cloned()
|
||||
[TokenTree::token_alone(token::Pound, span), body]
|
||||
.into_iter()
|
||||
.collect::<TokenStream>()
|
||||
},
|
||||
),
|
||||
@ -1042,7 +1046,7 @@ pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R
|
||||
if all_normal {
|
||||
return match frame.tree_cursor.look_ahead(dist - 1) {
|
||||
Some(tree) => match tree {
|
||||
TokenTree::Token(token) => looker(token),
|
||||
TokenTree::Token(token, _) => looker(token),
|
||||
TokenTree::Delimited(dspan, delim, _) => {
|
||||
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
|
||||
}
|
||||
@ -1226,7 +1230,7 @@ pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
|
||||
token::CloseDelim(_) | token::Eof => unreachable!(),
|
||||
_ => {
|
||||
self.bump();
|
||||
TokenTree::Token(self.prev_token.clone())
|
||||
TokenTree::Token(self.prev_token.clone(), Spacing::Alone)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1245,7 +1249,7 @@ pub fn parse_tokens(&mut self) -> TokenStream {
|
||||
loop {
|
||||
match self.token.kind {
|
||||
token::Eof | token::CloseDelim(..) => break,
|
||||
_ => result.push(self.parse_token_tree().into()),
|
||||
_ => result.push(self.parse_token_tree()),
|
||||
}
|
||||
}
|
||||
TokenStream::new(result)
|
||||
|
||||
@ -43,7 +43,7 @@ pub(super) fn render_macro_matcher(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Stri
|
||||
TokenTree::Delimited(_span, _delim, tts) => print_tts(&mut printer, tts),
|
||||
// Matcher which is not a Delimited is unexpected and should've failed
|
||||
// to compile, but we render whatever it is wrapped in parens.
|
||||
TokenTree::Token(_) => print_tt(&mut printer, matcher),
|
||||
TokenTree::Token(..) => print_tt(&mut printer, matcher),
|
||||
}
|
||||
printer.end();
|
||||
printer.break_offset_if_not_bol(0, -4);
|
||||
@ -93,7 +93,7 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String
|
||||
|
||||
fn print_tt(printer: &mut Printer<'_>, tt: &TokenTree) {
|
||||
match tt {
|
||||
TokenTree::Token(token) => {
|
||||
TokenTree::Token(token, _) => {
|
||||
let token_str = printer.token_to_string(token);
|
||||
printer.word(token_str);
|
||||
if let token::DocComment(..) = token.kind {
|
||||
@ -138,7 +138,7 @@ enum State {
|
||||
let mut state = Start;
|
||||
for tt in tts.trees() {
|
||||
let (needs_space, next_state) = match &tt {
|
||||
TokenTree::Token(tt) => match (state, &tt.kind) {
|
||||
TokenTree::Token(tt, _) => match (state, &tt.kind) {
|
||||
(Dollar, token::Ident(..)) => (false, DollarIdent),
|
||||
(DollarIdent, token::Colon) => (false, DollarIdentColon),
|
||||
(DollarIdentColon, token::Ident(..)) => (false, Other),
|
||||
|
||||
@ -110,14 +110,14 @@ fn contains_unhygienic_crate_reference(tts: &TokenStream) -> Option<Span> {
|
||||
|
||||
fn is_crate_keyword(tt: &TokenTree) -> Option<Span> {
|
||||
if_chain! {
|
||||
if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }) = tt;
|
||||
if let TokenTree::Token(Token { kind: TokenKind::Ident(symbol, _), span }, _) = tt;
|
||||
if symbol.as_str() == "crate";
|
||||
then { Some(*span) } else { None }
|
||||
}
|
||||
}
|
||||
|
||||
fn is_token(tt: &TokenTree, kind: &TokenKind) -> bool {
|
||||
if let TokenTree::Token(Token { kind: other, .. }) = tt {
|
||||
if let TokenTree::Token(Token { kind: other, .. }, _) = tt {
|
||||
kind == other
|
||||
} else {
|
||||
false
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
use std::panic::{catch_unwind, AssertUnwindSafe};
|
||||
|
||||
use rustc_ast::token::{BinOpToken, Delimiter, Token, TokenKind};
|
||||
use rustc_ast::tokenstream::{Cursor, Spacing, TokenStream, TokenTree};
|
||||
use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
|
||||
use rustc_ast::{ast, ptr};
|
||||
use rustc_ast_pretty::pprust;
|
||||
use rustc_span::{
|
||||
@ -682,7 +682,7 @@ struct MacroArgParser {
|
||||
|
||||
fn last_tok(tt: &TokenTree) -> Token {
|
||||
match *tt {
|
||||
TokenTree::Token(ref t) => t.clone(),
|
||||
TokenTree::Token(ref t, _) => t.clone(),
|
||||
TokenTree::Delimited(delim_span, delim, _) => Token {
|
||||
kind: TokenKind::CloseDelim(delim),
|
||||
span: delim_span.close,
|
||||
@ -737,10 +737,13 @@ fn add_other(&mut self) {
|
||||
|
||||
fn add_meta_variable(&mut self, iter: &mut Cursor) -> Option<()> {
|
||||
match iter.next() {
|
||||
Some(TokenTree::Token(Token {
|
||||
kind: TokenKind::Ident(name, _),
|
||||
..
|
||||
})) => {
|
||||
Some(TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::Ident(name, _),
|
||||
..
|
||||
},
|
||||
_,
|
||||
)) => {
|
||||
self.result.push(ParsedMacroArg {
|
||||
kind: MacroArgKind::MetaVariable(name, self.buf.clone()),
|
||||
});
|
||||
@ -777,21 +780,30 @@ fn add_repeat(
|
||||
}
|
||||
|
||||
match tok {
|
||||
TokenTree::Token(Token {
|
||||
kind: TokenKind::BinOp(BinOpToken::Plus),
|
||||
..
|
||||
})
|
||||
| TokenTree::Token(Token {
|
||||
kind: TokenKind::Question,
|
||||
..
|
||||
})
|
||||
| TokenTree::Token(Token {
|
||||
kind: TokenKind::BinOp(BinOpToken::Star),
|
||||
..
|
||||
}) => {
|
||||
TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::BinOp(BinOpToken::Plus),
|
||||
..
|
||||
},
|
||||
_,
|
||||
)
|
||||
| TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::Question,
|
||||
..
|
||||
},
|
||||
_,
|
||||
)
|
||||
| TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::BinOp(BinOpToken::Star),
|
||||
..
|
||||
},
|
||||
_,
|
||||
) => {
|
||||
break;
|
||||
}
|
||||
TokenTree::Token(ref t) => {
|
||||
TokenTree::Token(ref t, _) => {
|
||||
buffer.push_str(&pprust::token_to_string(t));
|
||||
}
|
||||
_ => return None,
|
||||
@ -859,10 +871,13 @@ fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
|
||||
|
||||
while let Some(tok) = iter.next() {
|
||||
match tok {
|
||||
TokenTree::Token(Token {
|
||||
kind: TokenKind::Dollar,
|
||||
span,
|
||||
}) => {
|
||||
TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::Dollar,
|
||||
span,
|
||||
},
|
||||
_,
|
||||
) => {
|
||||
// We always want to add a separator before meta variables.
|
||||
if !self.buf.is_empty() {
|
||||
self.add_separator();
|
||||
@ -875,13 +890,16 @@ fn parse(mut self, tokens: TokenStream) -> Option<Vec<ParsedMacroArg>> {
|
||||
span,
|
||||
};
|
||||
}
|
||||
TokenTree::Token(Token {
|
||||
kind: TokenKind::Colon,
|
||||
..
|
||||
}) if self.is_meta_var => {
|
||||
TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::Colon,
|
||||
..
|
||||
},
|
||||
_,
|
||||
) if self.is_meta_var => {
|
||||
self.add_meta_variable(&mut iter)?;
|
||||
}
|
||||
TokenTree::Token(ref t) => self.update_buffer(t),
|
||||
TokenTree::Token(ref t, _) => self.update_buffer(t),
|
||||
TokenTree::Delimited(_delimited_span, delimited, ref tts) => {
|
||||
if !self.buf.is_empty() {
|
||||
if next_space(&self.last_tok.kind) == SpaceState::Always {
|
||||
@ -1123,12 +1141,15 @@ fn parse_branch(&mut self) -> Option<MacroBranch> {
|
||||
TokenTree::Token(..) => return None,
|
||||
TokenTree::Delimited(delimited_span, d, _) => (delimited_span.open.lo(), d),
|
||||
};
|
||||
let args = TokenStream::new(vec![(tok, Spacing::Joint)]);
|
||||
let args = TokenStream::new(vec![tok]);
|
||||
match self.toks.next()? {
|
||||
TokenTree::Token(Token {
|
||||
kind: TokenKind::FatArrow,
|
||||
..
|
||||
}) => {}
|
||||
TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::FatArrow,
|
||||
..
|
||||
},
|
||||
_,
|
||||
) => {}
|
||||
_ => return None,
|
||||
}
|
||||
let (mut hi, body, whole_body) = match self.toks.next()? {
|
||||
@ -1147,10 +1168,13 @@ fn parse_branch(&mut self) -> Option<MacroBranch> {
|
||||
)
|
||||
}
|
||||
};
|
||||
if let Some(TokenTree::Token(Token {
|
||||
kind: TokenKind::Semi,
|
||||
span,
|
||||
})) = self.toks.look_ahead(0)
|
||||
if let Some(TokenTree::Token(
|
||||
Token {
|
||||
kind: TokenKind::Semi,
|
||||
span,
|
||||
},
|
||||
_,
|
||||
)) = self.toks.look_ahead(0)
|
||||
{
|
||||
hi = span.hi();
|
||||
self.toks.next();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user