From 0405d758a5d9db06994c4bc4ad6683d3afffd44a Mon Sep 17 00:00:00 2001
From: Edwin Cheng <edwin0cheng@gmail.com>
Date: Sat, 6 Mar 2021 03:17:36 +0800
Subject: [PATCH 1/2] Simplify TokenStream FromStr
---
crates/proc_macro_srv/src/rustc_server.rs | 80 ++++++++++++-----------
1 file changed, 42 insertions(+), 38 deletions(-)
diff --git a/crates/proc_macro_srv/src/rustc_server.rs b/crates/proc_macro_srv/src/rustc_server.rs
index 952b4a97fbb..14c853c7752 100644
--- a/crates/proc_macro_srv/src/rustc_server.rs
+++ b/crates/proc_macro_srv/src/rustc_server.rs
@@ -184,6 +184,7 @@ pub mod token_stream {
let (subtree, _token_map) =
mbe::parse_to_token_tree(src).ok_or("Failed to parse from mbe")?;
+ let subtree = subtree_replace_token_ids_with_unspecified(subtree);
Ok(TokenStream { subtree })
}
}
@@ -226,6 +227,44 @@ pub mod token_stream {
}
}
}
+
+ fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
+ tt::Subtree {
+ delimiter: subtree
+ .delimiter
+ .map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
+ token_trees: subtree
+ .token_trees
+ .into_iter()
+ .map(|t| token_tree_replace_token_ids_with_unspecified(t))
+ .collect(),
+ }
+ }
+
+ fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
+ match tt {
+ tt::TokenTree::Leaf(leaf) => {
+ tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
+ }
+ tt::TokenTree::Subtree(subtree) => {
+ tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
+ }
+ }
+ }
+
+ fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
+ match leaf {
+ tt::Leaf::Literal(lit) => {
+ tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
+ }
+ tt::Leaf::Punct(punct) => {
+ tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
+ }
+ tt::Leaf::Ident(ident) => {
+ tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
+ }
+ }
+ }
}
impl TokenStreamBuilder {
@@ -277,42 +316,6 @@ impl server::FreeFunctions for Rustc {
}
}
-fn subtree_replace_token_ids_with_unspecified(subtree: tt::Subtree) -> tt::Subtree {
- tt::Subtree {
- delimiter: subtree.delimiter.map(|d| tt::Delimiter { id: tt::TokenId::unspecified(), ..d }),
- token_trees: subtree
- .token_trees
- .into_iter()
- .map(|t| token_tree_replace_token_ids_with_unspecified(t))
- .collect(),
- }
-}
-
-fn token_tree_replace_token_ids_with_unspecified(tt: tt::TokenTree) -> tt::TokenTree {
- match tt {
- tt::TokenTree::Leaf(leaf) => {
- tt::TokenTree::Leaf(leaf_replace_token_ids_with_unspecified(leaf))
- }
- tt::TokenTree::Subtree(subtree) => {
- tt::TokenTree::Subtree(subtree_replace_token_ids_with_unspecified(subtree))
- }
- }
-}
-
-fn leaf_replace_token_ids_with_unspecified(leaf: tt::Leaf) -> tt::Leaf {
- match leaf {
- tt::Leaf::Literal(lit) => {
- tt::Leaf::Literal(tt::Literal { id: tt::TokenId::unspecified(), ..lit })
- }
- tt::Leaf::Punct(punct) => {
- tt::Leaf::Punct(tt::Punct { id: tt::TokenId::unspecified(), ..punct })
- }
- tt::Leaf::Ident(ident) => {
- tt::Leaf::Ident(tt::Ident { id: tt::TokenId::unspecified(), ..ident })
- }
- }
-}
-
impl server::TokenStream for Rustc {
fn new(&mut self) -> Self::TokenStream {
Self::TokenStream::new()
@@ -322,8 +325,9 @@ impl server::TokenStream for Rustc {
stream.is_empty()
}
fn from_str(&mut self, src: &str) -> Self::TokenStream {
- let (subtree, _) = mbe::parse_to_token_tree(src).expect("cannot parse string");
- TokenStream::with_subtree(subtree_replace_token_ids_with_unspecified(subtree))
+ use std::str::FromStr;
+
+ Self::TokenStream::from_str(src).expect("cannot parse string")
}
fn to_string(&mut self, stream: &Self::TokenStream) -> String {
stream.to_string()
From 83230b270455c82636147ee631c425fefdce26ff Mon Sep 17 00:00:00 2001
From: Edwin Cheng <edwin0cheng@gmail.com>
Date: Sat, 6 Mar 2021 03:30:22 +0800
Subject: [PATCH 2/2] Fix test
---
.../src/tests/fixtures/test_serialize_proc_macro.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt b/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
index ea34e688f73..fa581f11020 100644
--- a/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
+++ b/crates/proc_macro_srv/src/tests/fixtures/test_serialize_proc_macro.txt
@@ -101,7 +101,7 @@ SUBTREE $
PUNCH : [alone] 4294967295
IDENT Serialize 4294967295
IDENT for 4294967295
- IDENT Foo 1
+ IDENT Foo 4294967295
SUBTREE {} 4294967295
IDENT fn 4294967295
IDENT serialize 4294967295