mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

syntax: Use Token in StringReader and TokenTreesReader

Browse Source
This commit is contained in:
Vadim Petrochenkov 2019-06-05 00:02:59 +03:00
parent e0127dbf81
commit c0c57acd7b
3 changed files with 37 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/librustdoc/html/highlight.rs
View File

@ -234,7 +234,7 @@ fn write_token<W: Writer>(&mut self,
// reference or dereference operator or a reference or pointer type, instead of the // reference or dereference operator or a reference or pointer type, instead of the
// bit-and or multiplication operator. // bit-and or multiplication operator.
token::BinOp(token::And) | token::BinOp(token::Star) token::BinOp(token::And) | token::BinOp(token::Star)
if self.lexer.peek() != token::Whitespace => Class::RefKeyWord, if self.lexer.peek() != &token::Whitespace => Class::RefKeyWord,
// Consider this as part of a macro invocation if there was a // Consider this as part of a macro invocation if there was a
// leading identifier. // leading identifier.
@ -280,9 +280,9 @@ fn write_token<W: Writer>(&mut self,
// as an attribute. // as an attribute.
// Case 1: #![inner_attribute] // Case 1: #![inner_attribute]
if self.lexer.peek() == token::Not { if self.lexer.peek() == &token::Not {
self.try_next_token()?; // NOTE: consumes `!` token! self.try_next_token()?; // NOTE: consumes `!` token!
if self.lexer.peek() == token::OpenDelim(token::Bracket) { if self.lexer.peek() == &token::OpenDelim(token::Bracket) {
self.in_attribute = true; self.in_attribute = true;
out.enter_span(Class::Attribute)?; out.enter_span(Class::Attribute)?;
} }
@ -292,7 +292,7 @@ fn write_token<W: Writer>(&mut self,
} }
// Case 2: #[outer_attribute] // Case 2: #[outer_attribute]
if self.lexer.peek() == token::OpenDelim(token::Bracket) { if self.lexer.peek() == &token::OpenDelim(token::Bracket) {
self.in_attribute = true; self.in_attribute = true;
out.enter_span(Class::Attribute)?; out.enter_span(Class::Attribute)?;
} }
@ -341,7 +341,7 @@ fn write_token<W: Writer>(&mut self,
if self.in_macro_nonterminal { if self.in_macro_nonterminal {
self.in_macro_nonterminal = false; self.in_macro_nonterminal = false;
Class::MacroNonTerminal Class::MacroNonTerminal
} else if self.lexer.peek() == token::Not { } else if self.lexer.peek() == &token::Not {
self.in_macro = true; self.in_macro = true;
Class::Macro Class::Macro
} else { } else {

35
src/libsyntax/parse/lexer/mod.rs
View File

@ -12,7 +12,6 @@
use std::borrow::Cow; use std::borrow::Cow;
use std::char; use std::char;
use std::iter; use std::iter;
use std::mem::replace;
use rustc_data_structures::sync::Lrc; use rustc_data_structures::sync::Lrc;
use log::debug; use log::debug;
@ -41,8 +40,7 @@ pub struct StringReader<'a> {
/// Stop reading src at this index. /// Stop reading src at this index.
crate end_src_index: usize, crate end_src_index: usize,
// cached: // cached:
peek_tok: TokenKind, peek_token: Token,
peek_span: Span,
peek_span_src_raw: Span, peek_span_src_raw: Span,
fatal_errs: Vec<DiagnosticBuilder<'a>>, fatal_errs: Vec<DiagnosticBuilder<'a>>,
// cache a direct reference to the source text, so that we don't have to // cache a direct reference to the source text, so that we don't have to
@ -90,10 +88,7 @@ fn next_token(&mut self) -> Token where Self: Sized {
/// Returns the next token. EFFECT: advances the string_reader. /// Returns the next token. EFFECT: advances the string_reader.
pub fn try_next_token(&mut self) -> Result<Token, ()> { pub fn try_next_token(&mut self) -> Result<Token, ()> {
assert!(self.fatal_errs.is_empty()); assert!(self.fatal_errs.is_empty());
let ret_val = Token { let ret_val = self.peek_token.clone();
kind: replace(&mut self.peek_tok, token::Whitespace),
span: self.peek_span,
};
self.advance_token()?; self.advance_token()?;
Ok(ret_val) Ok(ret_val)
} }
@ -158,7 +153,7 @@ fn fail_unterminated_raw_string(&self, pos: BytePos, hash_count: u16) {
} }
fn fatal(&self, m: &str) -> FatalError { fn fatal(&self, m: &str) -> FatalError {
self.fatal_span(self.peek_span, m) self.fatal_span(self.peek_token.span, m)
} }
crate fn emit_fatal_errors(&mut self) { crate fn emit_fatal_errors(&mut self) {
@ -179,12 +174,8 @@ pub fn buffer_fatal_errors(&mut self) -> Vec<Diagnostic> {
buffer buffer
} }
pub fn peek(&self) -> Token { pub fn peek(&self) -> &Token {
// FIXME(pcwalton): Bad copy! &self.peek_token
Token {
kind: self.peek_tok.clone(),
span: self.peek_span,
}
} }
/// For comments.rs, which hackily pokes into next_pos and ch /// For comments.rs, which hackily pokes into next_pos and ch
@ -215,8 +206,7 @@ fn new_raw_internal(sess: &'a ParseSess, source_file: Lrc<syntax_pos::SourceFile
source_file, source_file,
end_src_index: src.len(), end_src_index: src.len(),
// dummy values; not read // dummy values; not read
peek_tok: token::Eof, peek_token: Token { kind: token::Eof, span: syntax_pos::DUMMY_SP },
peek_span: syntax_pos::DUMMY_SP,
peek_span_src_raw: syntax_pos::DUMMY_SP, peek_span_src_raw: syntax_pos::DUMMY_SP,
src, src,
fatal_errs: Vec::new(), fatal_errs: Vec::new(),
@ -321,29 +311,28 @@ fn err_span_char(&self, from_pos: BytePos, to_pos: BytePos, m: &str, c: char) {
self.err_span_(from_pos, to_pos, &m[..]); self.err_span_(from_pos, to_pos, &m[..]);
} }
/// Advance peek_tok and peek_span to refer to the next token, and /// Advance peek_token to refer to the next token, and
/// possibly update the interner. /// possibly update the interner.
fn advance_token(&mut self) -> Result<(), ()> { fn advance_token(&mut self) -> Result<(), ()> {
match self.scan_whitespace_or_comment() { match self.scan_whitespace_or_comment() {
Some(comment) => { Some(comment) => {
self.peek_span_src_raw = comment.span; self.peek_span_src_raw = comment.span;
self.peek_span = comment.span; self.peek_token = comment;
self.peek_tok = comment.kind;
} }
None => { None => {
if self.is_eof() { if self.is_eof() {
self.peek_tok = token::Eof;
let (real, raw) = self.mk_sp_and_raw( let (real, raw) = self.mk_sp_and_raw(
self.source_file.end_pos, self.source_file.end_pos,
self.source_file.end_pos, self.source_file.end_pos,
); );
self.peek_span = real; self.peek_token = Token { kind: token::Eof, span: real };
self.peek_span_src_raw = raw; self.peek_span_src_raw = raw;
} else { } else {
let start_bytepos = self.pos; let start_bytepos = self.pos;
self.peek_tok = self.next_token_inner()?; let kind = self.next_token_inner()?;
let (real, raw) = self.mk_sp_and_raw(start_bytepos, self.pos); let (real, raw) = self.mk_sp_and_raw(start_bytepos, self.pos);
self.peek_span = real; self.peek_token = Token { kind, span: real };
self.peek_span_src_raw = raw; self.peek_span_src_raw = raw;
}; };
} }

43
src/libsyntax/parse/lexer/tokentrees.rs
View File

@ -2,15 +2,15 @@
use crate::print::pprust::token_to_string; use crate::print::pprust::token_to_string;
use crate::parse::lexer::{StringReader, UnmatchedBrace}; use crate::parse::lexer::{StringReader, UnmatchedBrace};
use crate::parse::{token, PResult}; use crate::parse::token::{self, Token};
use crate::parse::PResult;
use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint}; use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
impl<'a> StringReader<'a> { impl<'a> StringReader<'a> {
crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) { crate fn into_token_trees(self) -> (PResult<'a, TokenStream>, Vec<UnmatchedBrace>) {
let mut tt_reader = TokenTreesReader { let mut tt_reader = TokenTreesReader {
string_reader: self, string_reader: self,
token: token::Eof, token: token::Token { kind: token::Eof, span: syntax_pos::DUMMY_SP },
span: syntax_pos::DUMMY_SP,
open_braces: Vec::new(), open_braces: Vec::new(),
unmatched_braces: Vec::new(), unmatched_braces: Vec::new(),
matching_delim_spans: Vec::new(), matching_delim_spans: Vec::new(),
@ -23,8 +23,7 @@ impl<'a> StringReader<'a> {
struct TokenTreesReader<'a> { struct TokenTreesReader<'a> {
string_reader: StringReader<'a>, string_reader: StringReader<'a>,
token: token::TokenKind, token: Token,
span: Span,
/// Stack of open delimiters and their spans. Used for error message. /// Stack of open delimiters and their spans. Used for error message.
open_braces: Vec<(token::DelimToken, Span)>, open_braces: Vec<(token::DelimToken, Span)>,
unmatched_braces: Vec<UnmatchedBrace>, unmatched_braces: Vec<UnmatchedBrace>,
@ -52,7 +51,7 @@ fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
fn parse_token_trees_until_close_delim(&mut self) -> TokenStream { fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
let mut tts = vec![]; let mut tts = vec![];
loop { loop {
if let token::CloseDelim(..) = self.token { if let token::CloseDelim(..) = self.token.kind {
return TokenStream::new(tts); return TokenStream::new(tts);
} }
@ -68,11 +67,11 @@ fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> { fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
let sm = self.string_reader.sess.source_map(); let sm = self.string_reader.sess.source_map();
match self.token { match self.token.kind {
token::Eof => { token::Eof => {
let msg = "this file contains an un-closed delimiter"; let msg = "this file contains an un-closed delimiter";
let mut err = self.string_reader.sess.span_diagnostic let mut err = self.string_reader.sess.span_diagnostic
.struct_span_err(self.span, msg); .struct_span_err(self.token.span, msg);
for &(_, sp) in &self.open_braces { for &(_, sp) in &self.open_braces {
err.span_label(sp, "un-closed delimiter"); err.span_label(sp, "un-closed delimiter");
} }
@ -102,10 +101,10 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
}, },
token::OpenDelim(delim) => { token::OpenDelim(delim) => {
// The span for beginning of the delimited section // The span for beginning of the delimited section
let pre_span = self.span; let pre_span = self.token.span;
// Parse the open delimiter. // Parse the open delimiter.
self.open_braces.push((delim, self.span)); self.open_braces.push((delim, self.token.span));
self.real_token(); self.real_token();
// Parse the token trees within the delimiters. // Parse the token trees within the delimiters.
@ -114,9 +113,9 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
let tts = self.parse_token_trees_until_close_delim(); let tts = self.parse_token_trees_until_close_delim();
// Expand to cover the entire delimited token tree // Expand to cover the entire delimited token tree
let delim_span = DelimSpan::from_pair(pre_span, self.span); let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
match self.token { match self.token.kind {
// Correct delimiter. // Correct delimiter.
token::CloseDelim(d) if d == delim => { token::CloseDelim(d) if d == delim => {
let (open_brace, open_brace_span) = self.open_braces.pop().unwrap(); let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
@ -126,7 +125,7 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
self.matching_delim_spans.clear(); self.matching_delim_spans.clear();
} else { } else {
self.matching_delim_spans.push( self.matching_delim_spans.push(
(open_brace, open_brace_span, self.span), (open_brace, open_brace_span, self.token.span),
); );
} }
// Parse the close delimiter. // Parse the close delimiter.
@ -136,16 +135,16 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
token::CloseDelim(other) => { token::CloseDelim(other) => {
let mut unclosed_delimiter = None; let mut unclosed_delimiter = None;
let mut candidate = None; let mut candidate = None;
if self.last_unclosed_found_span != Some(self.span) { if self.last_unclosed_found_span != Some(self.token.span) {
// do not complain about the same unclosed delimiter multiple times // do not complain about the same unclosed delimiter multiple times
self.last_unclosed_found_span = Some(self.span); self.last_unclosed_found_span = Some(self.token.span);
// This is a conservative error: only report the last unclosed // This is a conservative error: only report the last unclosed
// delimiter. The previous unclosed delimiters could actually be // delimiter. The previous unclosed delimiters could actually be
// closed! The parser just hasn't gotten to them yet. // closed! The parser just hasn't gotten to them yet.
if let Some(&(_, sp)) = self.open_braces.last() { if let Some(&(_, sp)) = self.open_braces.last() {
unclosed_delimiter = Some(sp); unclosed_delimiter = Some(sp);
}; };
if let Some(current_padding) = sm.span_to_margin(self.span) { if let Some(current_padding) = sm.span_to_margin(self.token.span) {
for (brace, brace_span) in &self.open_braces { for (brace, brace_span) in &self.open_braces {
if let Some(padding) = sm.span_to_margin(*brace_span) { if let Some(padding) = sm.span_to_margin(*brace_span) {
// high likelihood of these two corresponding // high likelihood of these two corresponding
@ -159,7 +158,7 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
self.unmatched_braces.push(UnmatchedBrace { self.unmatched_braces.push(UnmatchedBrace {
expected_delim: tok, expected_delim: tok,
found_delim: other, found_delim: other,
found_span: self.span, found_span: self.token.span,
unclosed_span: unclosed_delimiter, unclosed_span: unclosed_delimiter,
candidate_span: candidate, candidate_span: candidate,
}); });
@ -198,12 +197,12 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
let token_str = token_to_string(&self.token); let token_str = token_to_string(&self.token);
let msg = format!("unexpected close delimiter: `{}`", token_str); let msg = format!("unexpected close delimiter: `{}`", token_str);
let mut err = self.string_reader.sess.span_diagnostic let mut err = self.string_reader.sess.span_diagnostic
.struct_span_err(self.span, &msg); .struct_span_err(self.token.span, &msg);
err.span_label(self.span, "unexpected close delimiter"); err.span_label(self.token.span, "unexpected close delimiter");
Err(err) Err(err)
}, },
_ => { _ => {
let tt = TokenTree::token(self.span, self.token.clone()); let tt = TokenTree::Token(self.token.clone());
// Note that testing for joint-ness here is done via the raw // Note that testing for joint-ness here is done via the raw
// source span as the joint-ness is a property of the raw source // source span as the joint-ness is a property of the raw source
// rather than wanting to take `override_span` into account. // rather than wanting to take `override_span` into account.
@ -219,8 +218,6 @@ fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
} }
fn real_token(&mut self) { fn real_token(&mut self) {
let t = self.string_reader.real_token(); self.token = self.string_reader.real_token();
self.token = t.kind;
self.span = t.span;
} }
} }