Merge #7884
7884: Simplify TokenStream FromStr r=edwin0cheng a=edwin0cheng Make sure `FromStr` ignore all `TokenMap` in all cases. bors r+ Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>
This commit is contained in:
commit
94012e24e0
@ -184,6 +184,7 @@ pub mod token_stream {
|
||||
let (subtree, _token_map) =
|
||||
mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
|
||||
|
||||
let subtree = subtree_replace_token_ids_with_unspecified(subtree);
|
||||
Ok(TokenStream { subtree })
|
||||
}
|
||||
}
|
||||
@ -226,6 +227,44 @@ pub mod token_stream {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
|
||||
tt::Subtree {
|
||||
delimiter: subtree
|
||||
.delimiter
|
||||
.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
|
||||
token_trees: subtree
|
||||
.token_trees
|
||||
.into_iter()
|
||||
.map(|t| token_tree_replace_token_ids_with_unspecified(t))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
|
||||
match tt {
|
||||
tt::TokenTree::Leaf(leaf) => {
|
||||
tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
|
||||
}
|
||||
tt::TokenTree::Subtree(subtree) => {
|
||||
tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
|
||||
match leaf {
|
||||
tt::Leaf::Literal(lit) => {
|
||||
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
|
||||
}
|
||||
tt::Leaf::Punct(punct) => {
|
||||
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
|
||||
}
|
||||
tt::Leaf::Ident(ident) => {
|
||||
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenStreamBuilder {
|
||||
@ -277,42 +316,6 @@ impl server::FreeFunctions for Rustc {
|
||||
}
|
||||
}
|
||||
|
||||
fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
|
||||
tt::Subtree {
|
||||
delimiter: subtree.delimiter.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
|
||||
token_trees: subtree
|
||||
.token_trees
|
||||
.into_iter()
|
||||
.map(|t| token_tree_replace_token_ids_with_unspecified(t))
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
|
||||
match tt {
|
||||
tt::TokenTree::Leaf(leaf) => {
|
||||
tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
|
||||
}
|
||||
tt::TokenTree::Subtree(subtree) => {
|
||||
tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
|
||||
match leaf {
|
||||
tt::Leaf::Literal(lit) => {
|
||||
tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
|
||||
}
|
||||
tt::Leaf::Punct(punct) => {
|
||||
tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
|
||||
}
|
||||
tt::Leaf::Ident(ident) => {
|
||||
tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl server::TokenStream for Rustc {
|
||||
fn new(&mut self) -> Self::TokenStream {
|
||||
Self::TokenStream::new()
|
||||
@ -322,8 +325,9 @@ impl server::TokenStream for Rustc {
|
||||
stream.is_empty()
|
||||
}
|
||||
fn from_str(&mut self, src: &str) -> Self::TokenStream {
|
||||
let (subtree, _) = mbe::parse_to_token_tree(src).expect("cannot parse string");
|
||||
TokenStream::with_subtree(subtree_replace_token_ids_with_unspecified(subtree))
|
||||
use std::str::FromStr;
|
||||
|
||||
Self::TokenStream::from_str(src).expect("cannot parse string")
|
||||
}
|
||||
fn to_string(&mut self, stream: &Self::TokenStream) -> String {
|
||||
stream.to_string()
|
||||
|
@ -101,7 +101,7 @@ SUBTREE $
|
||||
PUNCH : [alone] 4294967295
|
||||
IDENT Serialize 4294967295
|
||||
IDENT for 4294967295
|
||||
IDENT Foo 1
|
||||
IDENT Foo 4294967295
|
||||
SUBTREE {} 4294967295
|
||||
IDENT fn 4294967295
|
||||
IDENT serialize 4294967295
|
||||
|
Loading…
x
Reference in New Issue
Block a user