mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #47343 - goffrie:master, r=jseyfried

Browse Source 
Glued tokens can themselves be joint.

When gluing two tokens, the second of which is joint, the result should also be
joint.
This fixes an issue with joining three `Dot` tokens to make a `DotDotDot` - the
intermediate `DotDot` would not be joint and therefore we would not attempt to
glue the last `Dot` token, yielding `.. .` instead of `...`.

r? @jseyfried
This commit is contained in:
kennytm 2018-01-13 02:26:39 +08:00
commit b480366445
No known key found for this signature in database
GPG Key ID: FEF6C8051D0E013C
1 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/libsyntax/tokenstream.rs
View File

@ -286,12 +286,12 @@ pub fn map<F: FnMut(TokenTree) -> TokenTree>(self, mut f: F) -> TokenStream {
TokenStream::concat(result)
}
fn first_tree(&self) -> Option<TokenTree> {
fn first_tree_and_joint(&self) -> Option<(TokenTree, bool)> {
match self.kind {
TokenStreamKind::Empty => None,
TokenStreamKind::Tree(ref tree) |
TokenStreamKind::JointTree(ref tree) => Some(tree.clone()),
TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree(),
TokenStreamKind::Tree(ref tree) => Some((tree.clone(), false)),
TokenStreamKind::JointTree(ref tree) => Some((tree.clone(), true)),
TokenStreamKind::Stream(ref stream) => stream.first().unwrap().first_tree_and_joint(),
}
}
@ -315,12 +315,18 @@ pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
let stream = stream.into();
let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);
if let Some(TokenTree::Token(last_span, last_tok)) = last_tree_if_joint {
if let Some(TokenTree::Token(span, tok)) = stream.first_tree() {
if let Some((TokenTree::Token(span, tok), is_joint)) = stream.first_tree_and_joint() {
if let Some(glued_tok) = last_tok.glue(tok) {
let last_stream = self.0.pop().unwrap();
self.push_all_but_last_tree(&last_stream);
let glued_span = last_span.to(span);
self.0.push(TokenTree::Token(glued_span, glued_tok).into());
let glued_tt = TokenTree::Token(glued_span, glued_tok);
let glued_tokenstream = if is_joint {
glued_tt.joint()
} else {
glued_tt.into()
};
self.0.push(glued_tokenstream);
self.push_all_but_first_tree(&stream);
return
}
@ -669,4 +675,16 @@ fn test_is_empty() {
assert_eq!(test1.is_empty(), false);
assert_eq!(test2.is_empty(), false);
}
#[test]
fn test_dotdotdot() {
let mut builder = TokenStreamBuilder::new();
builder.push(TokenTree::Token(sp(0, 1), Token::Dot).joint());
builder.push(TokenTree::Token(sp(1, 2), Token::Dot).joint());
builder.push(TokenTree::Token(sp(2, 3), Token::Dot));
let stream = builder.build();
assert!(stream.eq_unspanned(&string_to_ts("...")));
assert_eq!(stream.trees().count(), 1);
}
}