rust/crates/ra_syntax/src/parsing/reparsing.rs

392 lines
9.3 KiB
Rust
Raw Normal View History

2019-02-21 06:24:42 -06:00
//! Implementation of incremental re-parsing.
//!
//! We use two simple strategies for this:
//! - if the edit modifies only a single token (like changing an identifier's
//! letter), we replace only this token.
//! - otherwise, we search for the nearest `{}` block which contains the edit
//! and try to parse only this block.
2019-02-21 04:37:32 -06:00
use ra_text_edit::AtomTextEdit;
use ra_parser::Reparser;
use crate::{
2019-02-21 06:24:42 -06:00
SyntaxKind::*, TextRange, TextUnit, SyntaxError,
algo,
2019-03-30 05:25:53 -05:00
syntax_node::{GreenNode, SyntaxNode, GreenToken, SyntaxElement},
parsing::{
2019-02-23 07:07:29 -06:00
text_token_source::TextTokenSource,
text_tree_sink::TextTreeSink,
lexer::{tokenize, Token},
2019-05-15 07:35:47 -05:00
},
T,
};
pub(crate) fn incremental_reparse(
2019-01-07 07:15:47 -06:00
node: &SyntaxNode,
edit: &AtomTextEdit,
errors: Vec<SyntaxError>,
2019-03-30 05:25:53 -05:00
) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> {
if let Some((green, old_range)) = reparse_token(node, &edit) {
return Some((green, merge_errors(errors, Vec::new(), old_range, edit), old_range));
}
if let Some((green, new_errors, old_range)) = reparse_block(node, &edit) {
return Some((green, merge_errors(errors, new_errors, old_range, edit), old_range));
}
None
}
2019-03-30 05:25:53 -05:00
fn reparse_token<'node>(
root: &'node SyntaxNode,
edit: &AtomTextEdit,
2019-03-30 05:25:53 -05:00
) -> Option<(GreenNode, TextRange)> {
let token = algo::find_covering_element(root, edit.delete).as_token()?;
match token.kind() {
2018-10-31 16:38:18 -05:00
WHITESPACE | COMMENT | IDENT | STRING | RAW_STRING => {
2019-03-30 05:25:53 -05:00
if token.kind() == WHITESPACE || token.kind() == COMMENT {
// removing a new line may extends previous token
2019-03-30 05:25:53 -05:00
if token.text().to_string()[edit.delete - token.range().start()].contains('\n') {
return None;
}
}
2019-03-30 05:25:53 -05:00
let text = get_text_after_edit(token.into(), &edit);
let lex_tokens = tokenize(&text);
let lex_token = match lex_tokens[..] {
[lex_token] if lex_token.kind == token.kind() => lex_token,
_ => return None,
};
2019-03-30 05:25:53 -05:00
if lex_token.kind == IDENT && is_contextual_kw(&text) {
return None;
}
2019-03-30 05:25:53 -05:00
if let Some(next_char) = root.text().char_at(token.range().end()) {
let tokens_with_next_char = tokenize(&format!("{}{}", text, next_char));
if tokens_with_next_char.len() == 1 {
return None;
}
}
2019-04-08 17:06:30 -05:00
let new_token = GreenToken::new(rowan::SyntaxKind(token.kind().into()), text.into());
2019-03-30 05:25:53 -05:00
Some((token.replace_with(new_token), token.range()))
}
_ => None,
}
}
fn reparse_block<'node>(
2019-03-30 05:25:53 -05:00
root: &'node SyntaxNode,
edit: &AtomTextEdit,
2019-03-30 05:25:53 -05:00
) -> Option<(GreenNode, Vec<SyntaxError>, TextRange)> {
let (node, reparser) = find_reparsable_node(root, edit.delete)?;
let text = get_text_after_edit(node.into(), &edit);
let tokens = tokenize(&text);
if !is_balanced(&tokens) {
return None;
}
2019-05-25 07:31:53 -05:00
let mut token_source = TextTokenSource::new(&text, &tokens);
2019-02-23 07:07:29 -06:00
let mut tree_sink = TextTreeSink::new(&text, &tokens);
2019-05-25 07:31:53 -05:00
reparser.parse(&mut token_source, &mut tree_sink);
2019-02-21 04:37:32 -06:00
let (green, new_errors) = tree_sink.finish();
2019-03-30 05:25:53 -05:00
Some((node.replace_with(green), new_errors, node.range()))
}
2019-03-30 05:25:53 -05:00
fn get_text_after_edit(element: SyntaxElement, edit: &AtomTextEdit) -> String {
let edit = AtomTextEdit::replace(edit.delete - element.range().start(), edit.insert.clone());
let text = match element {
SyntaxElement::Token(token) => token.text().to_string(),
SyntaxElement::Node(node) => node.text().to_string(),
};
edit.apply(text)
}
fn is_contextual_kw(text: &str) -> bool {
match text {
"auto" | "default" | "union" => true,
_ => false,
}
}
2019-02-21 04:37:32 -06:00
fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> {
2019-03-30 05:25:53 -05:00
let node = algo::find_covering_element(node, range);
let mut ancestors = match node {
SyntaxElement::Token(it) => it.parent().ancestors(),
SyntaxElement::Node(it) => it.ancestors(),
};
ancestors.find_map(|node| {
let first_child = node.first_child_or_token().map(|it| it.kind());
let parent = node.parent().map(|it| it.kind());
2019-02-21 04:37:32 -06:00
Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r))
})
}
fn is_balanced(tokens: &[Token]) -> bool {
2018-10-17 18:25:37 -05:00
if tokens.is_empty()
2019-05-15 07:35:47 -05:00
|| tokens.first().unwrap().kind != T!['{']
|| tokens.last().unwrap().kind != T!['}']
{
return false;
}
let mut balance = 0usize;
2019-03-21 14:33:05 -05:00
for t in &tokens[1..tokens.len() - 1] {
match t.kind {
2019-05-15 07:35:47 -05:00
T!['{'] => balance += 1,
T!['}'] => {
balance = match balance.checked_sub(1) {
Some(b) => b,
None => return false,
}
}
_ => (),
}
}
balance == 0
}
fn merge_errors(
old_errors: Vec<SyntaxError>,
new_errors: Vec<SyntaxError>,
2019-03-30 05:25:53 -05:00
old_range: TextRange,
edit: &AtomTextEdit,
) -> Vec<SyntaxError> {
let mut res = Vec::new();
for e in old_errors {
2019-03-30 05:25:53 -05:00
if e.offset() <= old_range.start() {
res.push(e)
2019-03-30 05:25:53 -05:00
} else if e.offset() >= old_range.end() {
2019-03-21 12:22:06 -05:00
res.push(e.add_offset(TextUnit::of_str(&edit.insert), edit.delete.len()));
}
}
for e in new_errors {
2019-03-30 05:25:53 -05:00
res.push(e.add_offset(old_range.start(), 0.into()));
}
res
}
2018-09-15 07:35:30 -05:00
#[cfg(test)]
mod tests {
2018-12-06 12:16:37 -06:00
use test_utils::{extract_range, assert_eq_text};
2019-02-21 06:51:22 -06:00
use crate::{SourceFile, AstNode};
2018-12-06 12:16:37 -06:00
use super::*;
2018-09-15 07:35:30 -05:00
2019-03-30 05:25:53 -05:00
fn do_check(before: &str, replace_with: &str, reparsed_len: u32) {
let (range, before) = extract_range(before);
2019-01-08 12:59:55 -06:00
let edit = AtomTextEdit::replace(range, replace_with.to_owned());
let after = edit.apply(before.clone());
2018-09-15 07:35:30 -05:00
2019-01-07 07:15:47 -06:00
let fully_reparsed = SourceFile::parse(&after);
let incrementally_reparsed = {
2019-01-07 07:15:47 -06:00
let f = SourceFile::parse(&before);
2019-02-08 05:49:43 -06:00
let edit = AtomTextEdit { delete: range, insert: replace_with.to_string() };
2019-03-30 05:25:53 -05:00
let (green, new_errors, range) =
incremental_reparse(f.syntax(), &edit, f.errors()).unwrap();
assert_eq!(range.len(), reparsed_len.into(), "reparsed fragment has wrong length");
SourceFile::new(green, new_errors)
};
assert_eq_text!(
2019-02-21 06:51:22 -06:00
&fully_reparsed.syntax().debug_dump(),
&incrementally_reparsed.syntax().debug_dump(),
2019-03-30 05:25:53 -05:00
);
}
2019-03-30 05:25:53 -05:00
#[test] // FIXME: some test here actually test token reparsing
fn reparse_block_tests() {
do_check(
r"
2018-09-15 07:35:30 -05:00
fn foo() {
let x = foo + <|>bar<|>
}
",
"baz",
2019-03-30 05:25:53 -05:00
3,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
fn foo() {
let x = foo<|> + bar<|>
}
",
"baz",
2019-03-30 05:25:53 -05:00
25,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
struct Foo {
f: foo<|><|>
}
",
",\n g: (),",
2019-03-30 05:25:53 -05:00
14,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
fn foo {
let;
1 + 1;
<|>92<|>;
}
",
"62",
2019-03-30 05:25:53 -05:00
31, // FIXME: reparse only int literal here
);
do_check(
r"
2018-09-15 07:35:30 -05:00
mod foo {
fn <|><|>
}
",
"bar",
2019-03-30 05:25:53 -05:00
11,
);
2019-03-30 05:25:53 -05:00
do_check(
r"
2018-09-15 07:35:30 -05:00
trait Foo {
type <|>Foo<|>;
}
",
"Output",
2019-03-30 05:25:53 -05:00
3,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
impl IntoIterator<Item=i32> for Foo {
f<|><|>
}
",
"n next(",
2019-03-30 05:25:53 -05:00
9,
);
2019-03-30 05:25:53 -05:00
do_check(r"use a::b::{foo,<|>,bar<|>};", "baz", 10);
do_check(
r"
2018-09-15 07:35:30 -05:00
pub enum A {
Foo<|><|>
}
",
"\nBar;\n",
2019-03-30 05:25:53 -05:00
11,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
foo!{a, b<|><|> d}
",
", c[3]",
2019-03-30 05:25:53 -05:00
8,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
fn foo() {
vec![<|><|>]
}
",
"123",
2019-03-30 05:25:53 -05:00
14,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
extern {
fn<|>;<|>
}
",
" exit(code: c_int)",
2019-03-30 05:25:53 -05:00
11,
);
}
#[test]
2019-03-30 05:25:53 -05:00
fn reparse_token_tests() {
do_check(
r"<|><|>
fn foo() -> i32 { 1 }
",
"\n\n\n \n",
2019-03-30 05:25:53 -05:00
1,
);
do_check(
r"
2018-09-15 07:35:30 -05:00
fn foo() -> <|><|> {}
",
" \n",
2019-03-30 05:25:53 -05:00
2,
);
do_check(
r"
fn <|>foo<|>() -> i32 { 1 }
",
"bar",
2019-03-30 05:25:53 -05:00
3,
);
do_check(
r"
fn foo<|><|>foo() { }
",
"bar",
2019-03-30 05:25:53 -05:00
6,
);
do_check(
r"
fn foo /* <|><|> */ () {}
",
"some comment",
2019-03-30 05:25:53 -05:00
6,
);
do_check(
r"
fn baz <|><|> () {}
",
" \t\t\n\n",
2019-03-30 05:25:53 -05:00
2,
);
do_check(
r"
fn baz <|><|> () {}
",
" \t\t\n\n",
2019-03-30 05:25:53 -05:00
2,
);
do_check(
r"
/// foo <|><|>omment
mod { }
",
"c",
2019-03-30 05:25:53 -05:00
14,
);
do_check(
r#"
2018-09-15 07:35:30 -05:00
fn -> &str { "Hello<|><|>" }
"#,
", world",
2019-03-30 05:25:53 -05:00
7,
);
do_check(
r#"
2018-09-15 07:35:30 -05:00
fn -> &str { // "Hello<|><|>"
"#,
", world",
2019-03-30 05:25:53 -05:00
10,
);
do_check(
r##"
fn -> &str { r#"Hello<|><|>"#
"##,
", world",
2019-03-30 05:25:53 -05:00
10,
);
do_check(
r"
#[derive(<|>Copy<|>)]
enum Foo {
}
",
"Clone",
2019-03-30 05:25:53 -05:00
4,
);
2018-09-15 07:35:30 -05:00
}
2018-10-02 09:07:12 -05:00
}