Merge #11118
11118: internal: move ws attachment logic to the parser crate r=matklad a=matklad
This has to re-introduce the `sink` pattern, because doing this purely
with iterators is awkward :( Maaaybe the event vector was a false start?
But, anyway, I like the current factoring more -- it sort-of obvious
that we do want to keep ws-attachment business in the parser, and that
we also don't want that to depend on the particular tree structure. I
think `shortcuts` module achieves that.
bors r+
🤖
Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>
This commit is contained in:
bors[bot]
GitHub
commit
f2f57489db
@ -122,31 +122,6 @@ pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
|
||||
self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
|
||||
}
|
||||
|
||||
pub fn to_input(&self) -> crate::Input {
|
||||
let mut res = crate::Input::default();
|
||||
let mut was_joint = false;
|
||||
for i in 0..self.len() {
|
||||
let kind = self.kind(i);
|
||||
if kind.is_trivia() {
|
||||
was_joint = false
|
||||
} else {
|
||||
if kind == SyntaxKind::IDENT {
|
||||
let token_text = self.text(i);
|
||||
let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)
|
||||
.unwrap_or(SyntaxKind::IDENT);
|
||||
res.push_ident(contextual_kw);
|
||||
} else {
|
||||
if was_joint {
|
||||
res.was_joint();
|
||||
}
|
||||
res.push(kind);
|
||||
}
|
||||
was_joint = true;
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
fn push(&mut self, kind: SyntaxKind, offset: usize) {
|
||||
self.kind.push(kind);
|
||||
self.start.push(offset as u32);
|
||||
|
||||
@ -26,6 +26,7 @@
|
||||
mod grammar;
|
||||
mod input;
|
||||
mod output;
|
||||
mod shortcuts;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@ -36,6 +37,7 @@
|
||||
input::Input,
|
||||
lexed_str::LexedStr,
|
||||
output::{Output, Step},
|
||||
shortcuts::StrStep,
|
||||
syntax_kind::SyntaxKind,
|
||||
};
|
||||
|
||||
|
||||
220
crates/parser/src/shortcuts.rs
Normal file
220
crates/parser/src/shortcuts.rs
Normal file
@ -0,0 +1,220 @@
|
||||
//! Shortcuts that span lexer/parser abstraction.
|
||||
//!
|
||||
//! The way Rust works, parser doesn't necessary parse text, and you might
|
||||
//! tokenize text without parsing it further. So, it makes sense to keep
|
||||
//! abstract token parsing, and string tokenization as completely separate
|
||||
//! layers.
|
||||
//!
|
||||
//! However, often you do pares text into syntax trees and the glue code for
|
||||
//! that needs to live somewhere. Rather than putting it to lexer or parser, we
|
||||
//! use a separate shortcuts module for that.
|
||||
|
||||
use std::mem;
|
||||
|
||||
use crate::{
|
||||
LexedStr, Step,
|
||||
SyntaxKind::{self, *},
|
||||
};
|
||||
|
||||
pub enum StrStep<'a> {
|
||||
Token { kind: SyntaxKind, text: &'a str },
|
||||
Enter { kind: SyntaxKind },
|
||||
Exit,
|
||||
Error { msg: &'a str, pos: usize },
|
||||
}
|
||||
|
||||
impl<'a> LexedStr<'a> {
|
||||
pub fn to_input(&self) -> crate::Input {
|
||||
let mut res = crate::Input::default();
|
||||
let mut was_joint = false;
|
||||
for i in 0..self.len() {
|
||||
let kind = self.kind(i);
|
||||
if kind.is_trivia() {
|
||||
was_joint = false
|
||||
} else {
|
||||
if kind == SyntaxKind::IDENT {
|
||||
let token_text = self.text(i);
|
||||
let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)
|
||||
.unwrap_or(SyntaxKind::IDENT);
|
||||
res.push_ident(contextual_kw);
|
||||
} else {
|
||||
if was_joint {
|
||||
res.was_joint();
|
||||
}
|
||||
res.push(kind);
|
||||
}
|
||||
was_joint = true;
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
pub fn intersperse_trivia(
|
||||
&self,
|
||||
output: &crate::Output,
|
||||
synthetic_root: bool,
|
||||
sink: &mut dyn FnMut(StrStep),
|
||||
) -> bool {
|
||||
let mut builder = Builder { lexed: self, pos: 0, state: State::PendingEnter, sink };
|
||||
|
||||
if synthetic_root {
|
||||
builder.enter(SyntaxKind::SOURCE_FILE);
|
||||
}
|
||||
for event in output.iter() {
|
||||
match event {
|
||||
Step::Token { kind, n_input_tokens: n_raw_tokens } => {
|
||||
builder.token(kind, n_raw_tokens)
|
||||
}
|
||||
Step::Enter { kind } => builder.enter(kind),
|
||||
Step::Exit => builder.exit(),
|
||||
Step::Error { msg } => {
|
||||
let text_pos = builder.lexed.text_start(builder.pos);
|
||||
(builder.sink)(StrStep::Error { msg, pos: text_pos });
|
||||
}
|
||||
}
|
||||
}
|
||||
if synthetic_root {
|
||||
builder.exit();
|
||||
}
|
||||
|
||||
match mem::replace(&mut builder.state, State::Normal) {
|
||||
State::PendingExit => {
|
||||
builder.eat_trivias();
|
||||
(builder.sink)(StrStep::Exit);
|
||||
}
|
||||
State::PendingEnter | State::Normal => unreachable!(),
|
||||
}
|
||||
|
||||
let is_eof = builder.pos == builder.lexed.len();
|
||||
is_eof
|
||||
}
|
||||
}
|
||||
|
||||
struct Builder<'a, 'b> {
|
||||
lexed: &'a LexedStr<'a>,
|
||||
pos: usize,
|
||||
state: State,
|
||||
sink: &'b mut dyn FnMut(StrStep<'_>),
|
||||
}
|
||||
|
||||
enum State {
|
||||
PendingEnter,
|
||||
Normal,
|
||||
PendingExit,
|
||||
}
|
||||
|
||||
impl Builder<'_, '_> {
|
||||
fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
|
||||
match mem::replace(&mut self.state, State::Normal) {
|
||||
State::PendingEnter => unreachable!(),
|
||||
State::PendingExit => (self.sink)(StrStep::Exit),
|
||||
State::Normal => (),
|
||||
}
|
||||
self.eat_trivias();
|
||||
self.do_token(kind, n_tokens as usize);
|
||||
}
|
||||
|
||||
fn enter(&mut self, kind: SyntaxKind) {
|
||||
match mem::replace(&mut self.state, State::Normal) {
|
||||
State::PendingEnter => {
|
||||
(self.sink)(StrStep::Enter { kind });
|
||||
// No need to attach trivias to previous node: there is no
|
||||
// previous node.
|
||||
return;
|
||||
}
|
||||
State::PendingExit => (self.sink)(StrStep::Exit),
|
||||
State::Normal => (),
|
||||
}
|
||||
|
||||
let n_trivias =
|
||||
(self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
|
||||
let leading_trivias = self.pos..self.pos + n_trivias;
|
||||
let n_attached_trivias = n_attached_trivias(
|
||||
kind,
|
||||
leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
|
||||
);
|
||||
self.eat_n_trivias(n_trivias - n_attached_trivias);
|
||||
(self.sink)(StrStep::Enter { kind });
|
||||
self.eat_n_trivias(n_attached_trivias);
|
||||
}
|
||||
|
||||
fn exit(&mut self) {
|
||||
match mem::replace(&mut self.state, State::PendingExit) {
|
||||
State::PendingEnter => unreachable!(),
|
||||
State::PendingExit => (self.sink)(StrStep::Exit),
|
||||
State::Normal => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_trivias(&mut self) {
|
||||
while self.pos < self.lexed.len() {
|
||||
let kind = self.lexed.kind(self.pos);
|
||||
if !kind.is_trivia() {
|
||||
break;
|
||||
}
|
||||
self.do_token(kind, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_n_trivias(&mut self, n: usize) {
|
||||
for _ in 0..n {
|
||||
let kind = self.lexed.kind(self.pos);
|
||||
assert!(kind.is_trivia());
|
||||
self.do_token(kind, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) {
|
||||
let text = &self.lexed.range_text(self.pos..self.pos + n_tokens);
|
||||
self.pos += n_tokens;
|
||||
(self.sink)(StrStep::Token { kind, text });
|
||||
}
|
||||
}
|
||||
|
||||
fn n_attached_trivias<'a>(
|
||||
kind: SyntaxKind,
|
||||
trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
|
||||
) -> usize {
|
||||
match kind {
|
||||
CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD
|
||||
| STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => {
|
||||
let mut res = 0;
|
||||
let mut trivias = trivias.enumerate().peekable();
|
||||
|
||||
while let Some((i, (kind, text))) = trivias.next() {
|
||||
match kind {
|
||||
WHITESPACE if text.contains("\n\n") => {
|
||||
// we check whether the next token is a doc-comment
|
||||
// and skip the whitespace in this case
|
||||
if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
|
||||
if is_outer(peek_text) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
COMMENT => {
|
||||
if is_inner(text) {
|
||||
break;
|
||||
}
|
||||
res = i + 1;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_outer(text: &str) -> bool {
|
||||
if text.starts_with("////") || text.starts_with("/***") {
|
||||
return false;
|
||||
}
|
||||
text.starts_with("///") || text.starts_with("/**")
|
||||
}
|
||||
|
||||
fn is_inner(text: &str) -> bool {
|
||||
text.starts_with("//!") || text.starts_with("/*!")
|
||||
}
|
||||
@ -1,12 +1,11 @@
|
||||
//! Lexing, bridging to parser (which does the actual parsing) and
|
||||
//! incremental reparsing.
|
||||
|
||||
mod text_tree_sink;
|
||||
mod reparsing;
|
||||
|
||||
use crate::{
|
||||
parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode,
|
||||
};
|
||||
use rowan::TextRange;
|
||||
|
||||
use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode, SyntaxTreeBuilder};
|
||||
|
||||
pub(crate) use crate::parsing::reparsing::incremental_reparse;
|
||||
|
||||
@ -37,3 +36,32 @@ pub(crate) fn parse_text_as<T: AstNode>(
|
||||
|
||||
SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(())
|
||||
}
|
||||
|
||||
pub(crate) fn build_tree(
|
||||
lexed: parser::LexedStr<'_>,
|
||||
parser_output: parser::Output,
|
||||
synthetic_root: bool,
|
||||
) -> (GreenNode, Vec<SyntaxError>, bool) {
|
||||
let mut builder = SyntaxTreeBuilder::default();
|
||||
|
||||
let is_eof = lexed.intersperse_trivia(&parser_output, synthetic_root, &mut |step| match step {
|
||||
parser::StrStep::Token { kind, text } => builder.token(kind, text),
|
||||
parser::StrStep::Enter { kind } => builder.start_node(kind),
|
||||
parser::StrStep::Exit => builder.finish_node(),
|
||||
parser::StrStep::Error { msg, pos } => {
|
||||
builder.error(msg.to_string(), pos.try_into().unwrap())
|
||||
}
|
||||
});
|
||||
|
||||
let (node, mut errors) = builder.finish_raw();
|
||||
for (i, err) in lexed.errors() {
|
||||
let text_range = lexed.text_range(i);
|
||||
let text_range = TextRange::new(
|
||||
text_range.start.try_into().unwrap(),
|
||||
text_range.end.try_into().unwrap(),
|
||||
);
|
||||
errors.push(SyntaxError::new(err, text_range))
|
||||
}
|
||||
|
||||
(node, errors, is_eof)
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
use text_edit::Indel;
|
||||
|
||||
use crate::{
|
||||
parsing::text_tree_sink::build_tree,
|
||||
parsing::build_tree,
|
||||
syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
|
||||
SyntaxError,
|
||||
SyntaxKind::*,
|
||||
|
||||
@ -1,191 +0,0 @@
|
||||
//! See [`TextTreeSink`].
|
||||
|
||||
use std::mem;
|
||||
|
||||
use parser::LexedStr;
|
||||
|
||||
use crate::{
|
||||
ast,
|
||||
syntax_node::GreenNode,
|
||||
SyntaxError,
|
||||
SyntaxKind::{self, *},
|
||||
SyntaxTreeBuilder, TextRange,
|
||||
};
|
||||
|
||||
pub(crate) fn build_tree(
|
||||
lexed: LexedStr<'_>,
|
||||
parser_output: parser::Output,
|
||||
synthetic_root: bool,
|
||||
) -> (GreenNode, Vec<SyntaxError>, bool) {
|
||||
let mut builder = Builder::new(lexed);
|
||||
|
||||
if synthetic_root {
|
||||
builder.enter(SyntaxKind::SOURCE_FILE);
|
||||
}
|
||||
|
||||
for event in parser_output.iter() {
|
||||
match event {
|
||||
parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
|
||||
builder.token(kind, n_raw_tokens)
|
||||
}
|
||||
parser::Step::Enter { kind } => builder.enter(kind),
|
||||
parser::Step::Exit => builder.exit(),
|
||||
parser::Step::Error { msg } => {
|
||||
let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();
|
||||
builder.inner.error(msg.to_string(), text_pos);
|
||||
}
|
||||
}
|
||||
}
|
||||
if synthetic_root {
|
||||
builder.exit()
|
||||
}
|
||||
builder.build()
|
||||
}
|
||||
|
||||
struct Builder<'a> {
|
||||
lexed: LexedStr<'a>,
|
||||
pos: usize,
|
||||
state: State,
|
||||
inner: SyntaxTreeBuilder,
|
||||
}
|
||||
|
||||
enum State {
|
||||
PendingStart,
|
||||
Normal,
|
||||
PendingFinish,
|
||||
}
|
||||
|
||||
impl<'a> Builder<'a> {
|
||||
fn new(lexed: parser::LexedStr<'a>) -> Self {
|
||||
Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() }
|
||||
}
|
||||
|
||||
fn build(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
|
||||
match mem::replace(&mut self.state, State::Normal) {
|
||||
State::PendingFinish => {
|
||||
self.eat_trivias();
|
||||
self.inner.finish_node();
|
||||
}
|
||||
State::PendingStart | State::Normal => unreachable!(),
|
||||
}
|
||||
|
||||
let (node, mut errors) = self.inner.finish_raw();
|
||||
for (i, err) in self.lexed.errors() {
|
||||
let text_range = self.lexed.text_range(i);
|
||||
let text_range = TextRange::new(
|
||||
text_range.start.try_into().unwrap(),
|
||||
text_range.end.try_into().unwrap(),
|
||||
);
|
||||
errors.push(SyntaxError::new(err, text_range))
|
||||
}
|
||||
|
||||
let is_eof = self.pos == self.lexed.len();
|
||||
|
||||
(node, errors, is_eof)
|
||||
}
|
||||
|
||||
fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
|
||||
match mem::replace(&mut self.state, State::Normal) {
|
||||
State::PendingStart => unreachable!(),
|
||||
State::PendingFinish => self.inner.finish_node(),
|
||||
State::Normal => (),
|
||||
}
|
||||
self.eat_trivias();
|
||||
self.do_token(kind, n_tokens as usize);
|
||||
}
|
||||
|
||||
fn enter(&mut self, kind: SyntaxKind) {
|
||||
match mem::replace(&mut self.state, State::Normal) {
|
||||
State::PendingStart => {
|
||||
self.inner.start_node(kind);
|
||||
// No need to attach trivias to previous node: there is no
|
||||
// previous node.
|
||||
return;
|
||||
}
|
||||
State::PendingFinish => self.inner.finish_node(),
|
||||
State::Normal => (),
|
||||
}
|
||||
|
||||
let n_trivias =
|
||||
(self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
|
||||
let leading_trivias = self.pos..self.pos + n_trivias;
|
||||
let n_attached_trivias = n_attached_trivias(
|
||||
kind,
|
||||
leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
|
||||
);
|
||||
self.eat_n_trivias(n_trivias - n_attached_trivias);
|
||||
self.inner.start_node(kind);
|
||||
self.eat_n_trivias(n_attached_trivias);
|
||||
}
|
||||
|
||||
fn exit(&mut self) {
|
||||
match mem::replace(&mut self.state, State::PendingFinish) {
|
||||
State::PendingStart => unreachable!(),
|
||||
State::PendingFinish => self.inner.finish_node(),
|
||||
State::Normal => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_trivias(&mut self) {
|
||||
while self.pos < self.lexed.len() {
|
||||
let kind = self.lexed.kind(self.pos);
|
||||
if !kind.is_trivia() {
|
||||
break;
|
||||
}
|
||||
self.do_token(kind, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn eat_n_trivias(&mut self, n: usize) {
|
||||
for _ in 0..n {
|
||||
let kind = self.lexed.kind(self.pos);
|
||||
assert!(kind.is_trivia());
|
||||
self.do_token(kind, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) {
|
||||
let text = &self.lexed.range_text(self.pos..self.pos + n_tokens);
|
||||
self.pos += n_tokens;
|
||||
self.inner.token(kind, text);
|
||||
}
|
||||
}
|
||||
|
||||
fn n_attached_trivias<'a>(
|
||||
kind: SyntaxKind,
|
||||
trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
|
||||
) -> usize {
|
||||
match kind {
|
||||
CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD
|
||||
| STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => {
|
||||
let mut res = 0;
|
||||
let mut trivias = trivias.enumerate().peekable();
|
||||
|
||||
while let Some((i, (kind, text))) = trivias.next() {
|
||||
match kind {
|
||||
WHITESPACE if text.contains("\n\n") => {
|
||||
// we check whether the next token is a doc-comment
|
||||
// and skip the whitespace in this case
|
||||
if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
|
||||
let comment_kind = ast::CommentKind::from_text(peek_text);
|
||||
if comment_kind.doc == Some(ast::CommentPlacement::Outer) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
COMMENT => {
|
||||
let comment_kind = ast::CommentKind::from_text(text);
|
||||
if comment_kind.doc == Some(ast::CommentPlacement::Inner) {
|
||||
break;
|
||||
}
|
||||
res = i + 1;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
res
|
||||
}
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user