mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge #11117

Browse Source 
11117: internal: replace TreeSink with a data structure  r=matklad a=matklad

The general theme of this is to make parser a better independent
library.

The specific thing we do here is replacing callback based TreeSink with
a data structure. That is, rather than calling user-provided tree
construction methods, the parser now spits out a very bare-bones tree,
effectively a log of a DFS traversal.

This makes the parser usable without any *specifc* tree sink, and allows
us to, eg, move tests into this crate.

Now, it's also true that this is a distinction without a difference, as
the old and the new interface are equivalent in expressiveness. Still,
this new thing seems somewhat simpler. But yeah, I admit I don't have a
suuper strong motivation here, just a hunch that this is better.

cc #10765 

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>
This commit is contained in:
bors[bot] 2021-12-25 19:13:56 +00:00 committed by GitHub
commit c456b217d8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 272 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crates/mbe/src/lib.rs
View File

@ -10,7 +10,7 @@
mod expander;
mod syntax_bridge;
mod tt_iter;
mod to_parser_tokens;
mod to_parser_input;
#[cfg(test)]
mod benchmark;

21
crates/mbe/src/syntax_bridge.rs
View File

@ -1,6 +1,5 @@
//! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
use parser::{ParseError, TreeSink};
use rustc_hash::{FxHashMap, FxHashSet};
use syntax::{
ast::{self, make::tokens::doc_comment},
@ -11,7 +10,7 @@
use tt::buffer::{Cursor, TokenBuffer};
use crate::{
to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
to_parser_input::to_parser_input, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
};
/// Convert the syntax node to a `TokenTree` (what macro
@ -55,9 +54,19 @@ pub fn token_tree_to_syntax_node(
}
_ => TokenBuffer::from_subtree(tt),
};
let parser_tokens = to_parser_tokens(&buffer);
let parser_input = to_parser_input(&buffer);
let parser_output = parser::parse(&parser_input, entry_point);
let mut tree_sink = TtTreeSink::new(buffer.begin());
parser::parse(&parser_tokens, &mut tree_sink, entry_point);
for event in parser_output.iter() {
match event {
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
tree_sink.token(kind, n_raw_tokens)
}
parser::Step::Enter { kind } => tree_sink.start_node(kind),
parser::Step::Exit => tree_sink.finish_node(),
parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
}
}
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -643,7 +652,7 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
&texts[idx..texts.len() - (1 - idx)]
}
impl<'a> TreeSink for TtTreeSink<'a> {
impl<'a> TtTreeSink<'a> {
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == LIFETIME_IDENT {
n_tokens = 2;
@ -741,7 +750,7 @@ fn finish_node(&mut self) {
*self.roots.last_mut().unwrap() -= 1;
}
fn error(&mut self, error: ParseError) {
fn error(&mut self, error: String) {
self.inner.error(error, self.text_pos)
}
}

4
crates/mbe/src/to_parser_tokens.rs → crates/mbe/src/to_parser_input.rs
View File

@ -4,8 +4,8 @@
use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
let mut res = parser::Tokens::default();
pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
let mut res = parser::Input::default();
let mut current = buffer.begin();

53
crates/mbe/src/tt_iter.rs
View File

@ -1,11 +1,10 @@
//! A "Parser" structure for token trees. We use this when parsing a declarative
//! macro definition into a list of patterns and templates.
use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult, ParserEntryPoint};
use parser::TreeSink;
use syntax::SyntaxKind;
use tt::buffer::{Cursor, TokenBuffer};
use tt::buffer::TokenBuffer;
macro_rules! err {
() => {
@ -94,34 +93,28 @@ pub(crate) fn expect_fragment(
&mut self,
entry_point: ParserEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> {
struct OffsetTokenSink<'a> {
cursor: Cursor<'a>,
error: bool,
}
impl<'a> TreeSink for OffsetTokenSink<'a> {
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == SyntaxKind::LIFETIME_IDENT {
n_tokens = 2;
}
for _ in 0..n_tokens {
self.cursor = self.cursor.bump_subtree();
}
}
fn start_node(&mut self, _kind: SyntaxKind) {}
fn finish_node(&mut self) {}
fn error(&mut self, _error: parser::ParseError) {
self.error = true;
}
}
let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
let parser_tokens = to_parser_tokens(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
let parser_input = to_parser_input(&buffer);
let tree_traversal = parser::parse(&parser_input, entry_point);
parser::parse(&parser_tokens, &mut sink, entry_point);
let mut cursor = buffer.begin();
let mut error = false;
for step in tree_traversal.iter() {
match step {
parser::Step::Token { kind, mut n_input_tokens } => {
if kind == SyntaxKind::LIFETIME_IDENT {
n_input_tokens = 2;
}
for _ in 0..n_input_tokens {
cursor = cursor.bump_subtree();
}
}
parser::Step::Enter { .. } | parser::Step::Exit => (),
parser::Step::Error { .. } => error = true,
}
}
let mut err = if !sink.cursor.is_root() || sink.error {
let mut err = if !cursor.is_root() || error {
Some(err!("expected {:?}", entry_point))
} else {
None
@ -130,8 +123,8 @@ fn error(&mut self, _error: parser::ParseError) {
let mut curr = buffer.begin();
let mut res = vec![];
if sink.cursor.is_root() {
while curr != sink.cursor {
if cursor.is_root() {
while curr != cursor {
if let Some(token) = curr.token_tree() {
res.push(token);
}

18
crates/parser/src/event.rs
View File

@ -10,9 +10,8 @@
use std::mem;
use crate::{
ParseError,
output::Output,
SyntaxKind::{self, *},
TreeSink,
};
/// `Parser` produces a flat list of `Event`s.
@ -77,7 +76,7 @@ pub(crate) enum Event {
},
Error {
msg: ParseError,
msg: String,
},
}
@ -88,7 +87,8 @@ pub(crate) fn tombstone() -> Self {
}
/// Generate the syntax tree with the control of events.
pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
pub(super) fn process(mut events: Vec<Event>) -> Output {
let mut res = Output::default();
let mut forward_parents = Vec::new();
for i in 0..events.len() {
@ -117,15 +117,17 @@ pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
for kind in forward_parents.drain(..).rev() {
if kind != TOMBSTONE {
sink.start_node(kind);
res.enter_node(kind);
}
}
}
Event::Finish => sink.finish_node(),
Event::Finish => res.leave_node(),
Event::Token { kind, n_raw_tokens } => {
sink.token(kind, n_raw_tokens);
res.token(kind, n_raw_tokens);
}
Event::Error { msg } => sink.error(msg),
Event::Error { msg } => res.error(msg),
}
}
res
}

22
crates/parser/src/tokens.rs → crates/parser/src/input.rs
View File

@ -1,26 +1,26 @@
//! Input for the parser -- a sequence of tokens.
//!
//! As of now, parser doesn't have access to the *text* of the tokens, and makes
//! decisions based solely on their classification. Unlike `LexerToken`, the
//! `Tokens` doesn't include whitespace and comments.
//! See [`Input`].
use crate::SyntaxKind;
#[allow(non_camel_case_types)]
type bits = u64;
/// Main input to the parser.
/// Input for the parser -- a sequence of tokens.
///
/// A sequence of tokens represented internally as a struct of arrays.
/// As of now, parser doesn't have access to the *text* of the tokens, and makes
/// decisions based solely on their classification. Unlike `LexerToken`, the
/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
///
/// Struct of arrays internally, but this shouldn't really matter.
#[derive(Default)]
pub struct Tokens {
pub struct Input {
kind: Vec<SyntaxKind>,
joint: Vec<bits>,
contextual_kind: Vec<SyntaxKind>,
}
/// `pub` impl used by callers to create `Tokens`.
impl Tokens {
impl Input {
#[inline]
pub fn push(&mut self, kind: SyntaxKind) {
self.push_impl(kind, SyntaxKind::EOF)
@ -63,7 +63,7 @@ fn push_impl(&mut self, kind: SyntaxKind, contextual_kind: SyntaxKind) {
}
/// pub(crate) impl used by the parser to consume `Tokens`.
impl Tokens {
impl Input {
pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
}
@ -76,7 +76,7 @@ pub(crate) fn is_joint(&self, n: usize) -> bool {
}
}
impl Tokens {
impl Input {
fn bit_index(&self, n: usize) -> (usize, usize) {
let idx = n / (bits::BITS as usize);
let b_idx = n % (bits::BITS as usize);

4
crates/parser/src/lexed_str.rs
View File

@ -122,8 +122,8 @@ pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
}
pub fn to_tokens(&self) -> crate::Tokens {
let mut res = crate::Tokens::default();
pub fn to_input(&self) -> crate::Input {
let mut res = crate::Input::default();
let mut was_joint = false;
for i in 0..self.len() {
let kind = self.kind(i);

50
crates/parser/src/lib.rs
View File

@ -24,32 +24,20 @@
mod event;
mod parser;
mod grammar;
mod tokens;
mod input;
mod output;
#[cfg(test)]
mod tests;
pub(crate) use token_set::TokenSet;
pub use crate::{lexed_str::LexedStr, syntax_kind::SyntaxKind, tokens::Tokens};
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ParseError(pub Box<String>);
/// `TreeSink` abstracts details of a particular syntax tree implementation.
pub trait TreeSink {
/// Adds new token to the current branch.
fn token(&mut self, kind: SyntaxKind, n_tokens: u8);
/// Start new branch and make it current.
fn start_node(&mut self, kind: SyntaxKind);
/// Finish current branch and restore previous
/// branch as current.
fn finish_node(&mut self);
fn error(&mut self, error: ParseError);
}
pub use crate::{
input::Input,
lexed_str::LexedStr,
output::{Output, Step},
syntax_kind::SyntaxKind,
};
/// rust-analyzer parser allows you to choose one of the possible entry points.
///
@ -74,11 +62,19 @@ pub enum ParserEntryPoint {
}
/// Parse given tokens into the given sink as a rust file.
pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
parse(tokens, tree_sink, ParserEntryPoint::SourceFile);
pub fn parse_source_file(inp: &Input) -> Output {
parse(inp, ParserEntryPoint::SourceFile)
}
pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) {
/// Parses the given [`Input`] into [`Output`] assuming that the top-level
/// syntactic construct is the given [`ParserEntryPoint`].
///
/// Both input and output here are fairly abstract. The overall flow is that the
/// caller has some "real" tokens, converts them to [`Input`], parses them to
/// [`Output`], and then converts that into a "real" tree. The "real" tree is
/// made of "real" tokens, so this all hinges on rather tight coordination of
/// indices between the four stages.
pub fn parse(inp: &Input, entry_point: ParserEntryPoint) -> Output {
let entry_point: fn(&'_ mut parser::Parser) = match entry_point {
ParserEntryPoint::SourceFile => grammar::entry_points::source_file,
ParserEntryPoint::Path => grammar::entry_points::path,
@ -96,10 +92,10 @@ pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserE
ParserEntryPoint::Attr => grammar::entry_points::attr,
};
let mut p = parser::Parser::new(tokens);
let mut p = parser::Parser::new(inp);
entry_point(&mut p);
let events = p.finish();
event::process(tree_sink, events);
event::process(events)
}
/// A parsing function for a specific braced-block.
@ -119,11 +115,11 @@ pub fn for_node(
///
/// Tokens must start with `{`, end with `}` and form a valid brace
/// sequence.
pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
pub fn parse(self, tokens: &Input) -> Output {
let Reparser(r) = self;
let mut p = parser::Parser::new(tokens);
r(&mut p);
let events = p.finish();
event::process(tree_sink, events);
event::process(events)
}
}

76
crates/parser/src/output.rs Normal file
View File

@ -0,0 +1,76 @@
//! See [`Output`]
use crate::SyntaxKind;
/// Output of the parser -- a DFS traversal of a concrete syntax tree.
///
/// Use the [`Output::iter`] method to iterate over traversal steps and consume
/// a syntax tree.
///
/// In a sense, this is just a sequence of [`SyntaxKind`]-colored parenthesis
/// interspersed into the original [`crate::Input`]. The output is fundamentally
/// coordinated with the input and `n_input_tokens` refers to the number of
/// times [`crate::Input::push`] was called.
#[derive(Default)]
pub struct Output {
/// 32-bit encoding of events. If LSB is zero, then that's an index into the
/// error vector. Otherwise, it's one of the thee other variants, with data encoded as
///
/// |16 bit kind|8 bit n_input_tokens|4 bit tag|4 bit leftover|
///
event: Vec<u32>,
error: Vec<String>,
}
pub enum Step<'a> {
Token { kind: SyntaxKind, n_input_tokens: u8 },
Enter { kind: SyntaxKind },
Exit,
Error { msg: &'a str },
}
impl Output {
pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {
self.event.iter().map(|&event| {
if event & 0b1 == 0 {
return Step::Error { msg: self.error[(event as usize) >> 1].as_str() };
}
let tag = ((event & 0x0000_00F0) >> 4) as u8;
match tag {
0 => {
let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8;
Step::Token { kind, n_input_tokens }
}
1 => {
let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
Step::Enter { kind }
}
2 => Step::Exit,
_ => unreachable!(),
}
})
}
pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
let e = ((kind as u16 as u32) << 16) | ((n_tokens as u32) << 8) | (0 << 4) | 1;
self.event.push(e)
}
pub(crate) fn enter_node(&mut self, kind: SyntaxKind) {
let e = ((kind as u16 as u32) << 16) | (1 << 4) | 1;
self.event.push(e)
}
pub(crate) fn leave_node(&mut self) {
let e = 2 << 4 | 1;
self.event.push(e)
}
pub(crate) fn error(&mut self, error: String) {
let idx = self.error.len();
self.error.push(error);
let e = (idx as u32) << 1;
self.event.push(e);
}
}

33
crates/parser/src/parser.rs
View File

@ -7,8 +7,7 @@
use crate::{
event::Event,
tokens::Tokens,
ParseError,
input::Input,
SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
TokenSet, T,
};
@ -23,7 +22,7 @@
/// "start expression, consume number literal,
/// finish expression". See `Event` docs for more.
pub(crate) struct Parser<'t> {
tokens: &'t Tokens,
inp: &'t Input,
pos: usize,
events: Vec<Event>,
steps: Cell<u32>,
@ -32,8 +31,8 @@ pub(crate) struct Parser<'t> {
static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
impl<'t> Parser<'t> {
pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {
Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }
pub(super) fn new(inp: &'t Input) -> Parser<'t> {
Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0) }
}
pub(crate) fn finish(self) -> Vec<Event> {
@ -56,7 +55,7 @@ pub(crate) fn nth(&self, n: usize) -> SyntaxKind {
assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
self.steps.set(steps + 1);
self.tokens.kind(self.pos + n)
self.inp.kind(self.pos + n)
}
/// Checks if the current token is `kind`.
@ -92,7 +91,7 @@ pub(crate) fn nth_at(&self, n: usize, kind: SyntaxKind) -> bool {
T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
_ => self.tokens.kind(self.pos + n) == kind,
_ => self.inp.kind(self.pos + n) == kind,
}
}
@ -131,17 +130,17 @@ pub(crate) fn eat(&mut self, kind: SyntaxKind) -> bool {
}
fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
self.tokens.kind(self.pos + n) == k1
&& self.tokens.kind(self.pos + n + 1) == k2
&& self.tokens.is_joint(self.pos + n)
self.inp.kind(self.pos + n) == k1
&& self.inp.kind(self.pos + n + 1) == k2
&& self.inp.is_joint(self.pos + n)
}
fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
self.tokens.kind(self.pos + n) == k1
&& self.tokens.kind(self.pos + n + 1) == k2
&& self.tokens.kind(self.pos + n + 2) == k3
&& self.tokens.is_joint(self.pos + n)
&& self.tokens.is_joint(self.pos + n + 1)
self.inp.kind(self.pos + n) == k1
&& self.inp.kind(self.pos + n + 1) == k2
&& self.inp.kind(self.pos + n + 2) == k3
&& self.inp.is_joint(self.pos + n)
&& self.inp.is_joint(self.pos + n + 1)
}
/// Checks if the current token is in `kinds`.
@ -151,7 +150,7 @@ pub(crate) fn at_ts(&self, kinds: TokenSet) -> bool {
/// Checks if the current token is contextual keyword with text `t`.
pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
self.tokens.contextual_kind(self.pos) == kw
self.inp.contextual_kind(self.pos) == kw
}
/// Starts a new node in the syntax tree. All nodes and tokens
@ -196,7 +195,7 @@ pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
/// structured errors with spans and notes, like rustc
/// does.
pub(crate) fn error<T: Into<String>>(&mut self, message: T) {
let msg = ParseError(Box::new(message.into()));
let msg = message.into();
self.push_event(Event::Error { msg });
}

38
crates/syntax/src/parsing.rs
View File

@ -4,24 +4,18 @@
mod text_tree_sink;
mod reparsing;
use parser::SyntaxKind;
use text_tree_sink::TextTreeSink;
use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};
use crate::{
parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode,
};
pub(crate) use crate::parsing::reparsing::incremental_reparse;
pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
let lexed = parser::LexedStr::new(text);
let parser_tokens = lexed.to_tokens();
let mut tree_sink = TextTreeSink::new(lexed);
parser::parse_source_file(&parser_tokens, &mut tree_sink);
let (tree, parser_errors) = tree_sink.finish();
(tree, parser_errors)
let parser_input = lexed.to_input();
let parser_output = parser::parse_source_file(&parser_input);
let (node, errors, _eof) = build_tree(lexed, parser_output, false);
(node, errors)
}
/// Returns `text` parsed as a `T` provided there are no parse errors.
@ -33,21 +27,13 @@ pub(crate) fn parse_text_as<T: AstNode>(
if lexed.errors().next().is_some() {
return Err(());
}
let parser_tokens = lexed.to_tokens();
let parser_input = lexed.to_input();
let parser_output = parser::parse(&parser_input, entry_point);
let (node, errors, eof) = build_tree(lexed, parser_output, true);
let mut tree_sink = TextTreeSink::new(lexed);
// TextTreeSink assumes that there's at least some root node to which it can attach errors and
// tokens. We arbitrarily give it a SourceFile.
use parser::TreeSink;
tree_sink.start_node(SyntaxKind::SOURCE_FILE);
parser::parse(&parser_tokens, &mut tree_sink, entry_point);
tree_sink.finish_node();
let (tree, parser_errors, eof) = tree_sink.finish_eof();
if !parser_errors.is_empty() || !eof {
if !errors.is_empty() || !eof {
return Err(());
}
SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(())
SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(())
}

10
crates/syntax/src/parsing/reparsing.rs
View File

@ -10,7 +10,7 @@
use text_edit::Indel;
use crate::{
parsing::text_tree_sink::TextTreeSink,
parsing::text_tree_sink::build_tree,
syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
SyntaxError,
SyntaxKind::*,
@ -89,16 +89,14 @@ fn reparse_block(
let text = get_text_after_edit(node.clone().into(), edit);
let lexed = parser::LexedStr::new(text.as_str());
let parser_tokens = lexed.to_tokens();
let parser_input = lexed.to_input();
if !is_balanced(&lexed) {
return None;
}
let mut tree_sink = TextTreeSink::new(lexed);
let tree_traversal = reparser.parse(&parser_input);
reparser.parse(&parser_tokens, &mut tree_sink);
let (green, new_parser_errors) = tree_sink.finish();
let (green, new_parser_errors, _eof) = build_tree(lexed, tree_traversal, false);
Some((node.replace_with(green), new_parser_errors, node.text_range()))
}

135
crates/syntax/src/parsing/text_tree_sink.rs
View File

@ -2,7 +2,7 @@
use std::mem;
use parser::{LexedStr, ParseError, TreeSink};
use parser::LexedStr;
use crate::{
ast,
@ -12,10 +12,37 @@
SyntaxTreeBuilder, TextRange,
};
/// Bridges the parser with our specific syntax tree representation.
///
/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
pub(crate) struct TextTreeSink<'a> {
pub(crate) fn build_tree(
lexed: LexedStr<'_>,
parser_output: parser::Output,
synthetic_root: bool,
) -> (GreenNode, Vec<SyntaxError>, bool) {
let mut builder = Builder::new(lexed);
if synthetic_root {
builder.enter(SyntaxKind::SOURCE_FILE);
}
for event in parser_output.iter() {
match event {
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
builder.token(kind, n_raw_tokens)
}
parser::Step::Enter { kind } => builder.enter(kind),
parser::Step::Exit => builder.exit(),
parser::Step::Error { msg } => {
let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();
builder.inner.error(msg.to_string(), text_pos);
}
}
}
if synthetic_root {
builder.exit()
}
builder.build()
}
struct Builder<'a> {
lexed: LexedStr<'a>,
pos: usize,
state: State,
@ -28,61 +55,12 @@ enum State {
PendingFinish,
}
impl<'a> TreeSink for TextTreeSink<'a> {
fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingStart => unreachable!(),
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
self.eat_trivias();
self.do_token(kind, n_tokens as usize);
}
fn start_node(&mut self, kind: SyntaxKind) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingStart => {
self.inner.start_node(kind);
// No need to attach trivias to previous node: there is no
// previous node.
return;
}
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
let n_trivias =
(self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
let leading_trivias = self.pos..self.pos + n_trivias;
let n_attached_trivias = n_attached_trivias(
kind,
leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
);
self.eat_n_trivias(n_trivias - n_attached_trivias);
self.inner.start_node(kind);
self.eat_n_trivias(n_attached_trivias);
}
fn finish_node(&mut self) {
match mem::replace(&mut self.state, State::PendingFinish) {
State::PendingStart => unreachable!(),
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
}
fn error(&mut self, error: ParseError) {
let text_pos = self.lexed.text_start(self.pos).try_into().unwrap();
self.inner.error(error, text_pos);
}
}
impl<'a> TextTreeSink<'a> {
pub(super) fn new(lexed: parser::LexedStr<'a>) -> Self {
impl<'a> Builder<'a> {
fn new(lexed: parser::LexedStr<'a>) -> Self {
Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() }
}
pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
fn build(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingFinish => {
self.eat_trivias();
@ -106,9 +84,46 @@ pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
(node, errors, is_eof)
}
pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
let (node, errors, _eof) = self.finish_eof();
(node, errors)
fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingStart => unreachable!(),
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
self.eat_trivias();
self.do_token(kind, n_tokens as usize);
}
fn enter(&mut self, kind: SyntaxKind) {
match mem::replace(&mut self.state, State::Normal) {
State::PendingStart => {
self.inner.start_node(kind);
// No need to attach trivias to previous node: there is no
// previous node.
return;
}
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
let n_trivias =
(self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
let leading_trivias = self.pos..self.pos + n_trivias;
let n_attached_trivias = n_attached_trivias(
kind,
leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
);
self.eat_n_trivias(n_trivias - n_attached_trivias);
self.inner.start_node(kind);
self.eat_n_trivias(n_attached_trivias);
}
fn exit(&mut self) {
match mem::replace(&mut self.state, State::PendingFinish) {
State::PendingStart => unreachable!(),
State::PendingFinish => self.inner.finish_node(),
State::Normal => (),
}
}
fn eat_trivias(&mut self) {

4
crates/syntax/src/syntax_node.rs
View File

@ -69,7 +69,7 @@ pub fn finish_node(&mut self) {
self.inner.finish_node();
}
pub fn error(&mut self, error: parser::ParseError, text_pos: TextSize) {
self.errors.push(SyntaxError::new_at_offset(*error.0, text_pos));
pub fn error(&mut self, error: String, text_pos: TextSize) {
self.errors.push(SyntaxError::new_at_offset(error, text_pos));
}
}