mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove TokenStreamBuilder.

Browse Source 
It's now only used in one function. Also, the "should we glue the
tokens?" check is only necessary when pushing a `TokenTree::Token`, not
when pushing a `TokenTree::Delimited`.

As part of this, we now do the "should we glue the tokens?" check
immediately, which avoids having look back at the previous token. It
also puts all the logic dealing with token gluing in a single place.
This commit is contained in:
Nicholas Nethercote 2022-09-30 16:51:35 +10:00
parent 8d0754d602
commit a822d08bd1
1 changed files with 20 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
compiler/rustc_parse/src/lexer/tokentrees.rs
View File

@ -47,13 +47,13 @@ impl<'a> TokenTreesReader<'a> {
// Parse a stream of tokens into a list of `TokenTree`s.
fn parse_token_trees(&mut self, is_top_level: bool) -> PResult<'a, TokenStream> {
self.token = self.string_reader.next_token().0;
let mut buf = TokenStreamBuilder::default();
let mut buf = Vec::new();
loop {
match self.token.kind {
token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)),
token::CloseDelim(delim) => {
return if !is_top_level {
Ok(buf.into_token_stream())
Ok(TokenStream::new(buf))
} else {
Err(self.close_delim_err(delim))
};
@ -62,21 +62,28 @@ impl<'a> TokenTreesReader<'a> {
if !is_top_level {
self.eof_err().emit();
}
return Ok(buf.into_token_stream());
return Ok(TokenStream::new(buf));
}
_ => {
// `this_spacing` for the returned token refers to whether the token is
// immediately followed by another op token. It is determined by the
// next token: its kind and its `preceded_by_whitespace` status.
let (next_tok, is_next_tok_preceded_by_whitespace) =
self.string_reader.next_token();
let this_spacing = if is_next_tok_preceded_by_whitespace || !next_tok.is_op() {
Spacing::Alone
} else {
Spacing::Joint
// Get the next normal token. This might require getting multiple adjacent
// single-char tokens and joining them together.
let (this_spacing, next_tok) = loop {
let (next_tok, is_next_tok_preceded_by_whitespace) =
self.string_reader.next_token();
if !is_next_tok_preceded_by_whitespace {
if let Some(glued) = self.token.glue(&next_tok) {
self.token = glued;
} else {
let this_spacing =
if next_tok.is_op() { Spacing::Joint } else { Spacing::Alone };
break (this_spacing, next_tok);
}
} else {
break (Spacing::Alone, next_tok);
}
};
let this_tok = std::mem::replace(&mut self.token, next_tok);
buf.push(TokenTree::Token(this_tok, this_spacing))
buf.push(TokenTree::Token(this_tok, this_spacing));
}
}
}
@ -249,27 +256,3 @@ impl<'a> TokenTreesReader<'a> {
err
}
}
#[derive(Default)]
struct TokenStreamBuilder {
buf: Vec<TokenTree>,
}
impl TokenStreamBuilder {
#[inline(always)]
fn push(&mut self, tree: TokenTree) {
if let Some(TokenTree::Token(prev_token, Spacing::Joint)) = self.buf.last()
&& let TokenTree::Token(token, joint) = &tree
&& let Some(glued) = prev_token.glue(token)
{
self.buf.pop();
self.buf.push(TokenTree::Token(glued, *joint));
} else {
self.buf.push(tree)
}
}
fn into_token_stream(self) -> TokenStream {
TokenStream::new(self.buf)
}
}