From d334b5a1db9ec6a57f54077d422a3f4b3c8c1178 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Thu, 21 Feb 2019 13:27:45 +0300 Subject: [PATCH] move parser to a separate crate --- Cargo.lock | 7 ++ crates/ra_parser/Cargo.toml | 9 +++ .../src/parsing => ra_parser/src}/event.rs | 4 +- .../src/parsing => ra_parser/src}/grammar.rs | 10 ++- .../src}/grammar/attributes.rs | 0 .../src}/grammar/expressions.rs | 0 .../src}/grammar/expressions/atom.rs | 0 .../src}/grammar/items.rs | 0 .../src}/grammar/items/consts.rs | 0 .../src}/grammar/items/nominal.rs | 0 .../src}/grammar/items/traits.rs | 0 .../src}/grammar/items/use_item.rs | 0 .../src}/grammar/params.rs | 0 .../src}/grammar/paths.rs | 0 .../src}/grammar/patterns.rs | 0 .../src}/grammar/type_args.rs | 0 .../src}/grammar/type_params.rs | 0 .../src}/grammar/types.rs | 0 crates/ra_parser/src/lib.rs | 64 +++++++++++++++++++ .../src/parsing => ra_parser/src}/parser.rs | 7 +- .../src/syntax_kind.rs} | 4 +- .../src/syntax_kind}/generated.rs | 4 +- .../src/syntax_kind}/generated.rs.tera | 0 .../parsing => ra_parser/src}/token_set.rs | 0 24 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 crates/ra_parser/Cargo.toml rename crates/{ra_syntax/src/parsing => ra_parser/src}/event.rs (97%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar.rs (97%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/attributes.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/expressions.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/expressions/atom.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/items.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/items/consts.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/items/nominal.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/items/traits.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/items/use_item.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/params.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/paths.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/patterns.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/type_args.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/type_params.rs (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/grammar/types.rs (100%) create mode 100644 crates/ra_parser/src/lib.rs rename crates/{ra_syntax/src/parsing => ra_parser/src}/parser.rs (98%) rename crates/{ra_syntax/src/syntax_kinds.rs => ra_parser/src/syntax_kind.rs} (85%) rename crates/{ra_syntax/src/syntax_kinds => ra_parser/src/syntax_kind}/generated.rs (99%) rename crates/{ra_syntax/src/syntax_kinds => ra_parser/src/syntax_kind}/generated.rs.tera (100%) rename crates/{ra_syntax/src/parsing => ra_parser/src}/token_set.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 7b1412556c5..677a9db1c8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1071,6 +1071,13 @@ dependencies = [ "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ra_parser" +version = "0.1.0" +dependencies = [ + "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ra_project_model" version = "0.1.0" diff --git a/crates/ra_parser/Cargo.toml b/crates/ra_parser/Cargo.toml new file mode 100644 index 00000000000..b110e2bc6d3 --- /dev/null +++ b/crates/ra_parser/Cargo.toml @@ -0,0 +1,9 @@ +[package] +edition = "2018" +name = "ra_parser" +version = "0.1.0" +authors = ["rust-analyzer developers"] +publish = false + +[dependencies] +drop_bomb = "0.1.4" diff --git a/crates/ra_syntax/src/parsing/event.rs b/crates/ra_parser/src/event.rs similarity index 97% rename from crates/ra_syntax/src/parsing/event.rs rename to crates/ra_parser/src/event.rs index d6cbdffe083..d6e8454d43d 100644 --- a/crates/ra_syntax/src/parsing/event.rs +++ b/crates/ra_parser/src/event.rs @@ -10,8 +10,8 @@ use std::mem; use crate::{ + ParseError, TreeSink, SyntaxKind::{self, *}, - parsing::{ParseError, TreeSink}, }; /// `Parser` produces a flat list of `Event`s. @@ -84,7 +84,7 @@ impl Event { } /// Generate the syntax tree with the control of events. -pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec) { +pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { let mut forward_parents = Vec::new(); for i in 0..events.len() { diff --git a/crates/ra_syntax/src/parsing/grammar.rs b/crates/ra_parser/src/grammar.rs similarity index 97% rename from crates/ra_syntax/src/parsing/grammar.rs rename to crates/ra_parser/src/grammar.rs index 800d5a4a29c..15aab6c6fcd 100644 --- a/crates/ra_syntax/src/parsing/grammar.rs +++ b/crates/ra_parser/src/grammar.rs @@ -38,20 +38,18 @@ mod types; use crate::{ SyntaxKind::{self, *}, - parsing::{ - token_set::TokenSet, - parser::{CompletedMarker, Marker, Parser} - }, + TokenSet, + parser::{CompletedMarker, Marker, Parser}, }; -pub(super) fn root(p: &mut Parser) { +pub(crate) fn root(p: &mut Parser) { let m = p.start(); p.eat(SHEBANG); items::mod_contents(p, false); m.complete(p, SOURCE_FILE); } -pub(super) fn reparser( +pub(crate) fn reparser( node: SyntaxKind, first_child: Option, parent: Option, diff --git a/crates/ra_syntax/src/parsing/grammar/attributes.rs b/crates/ra_parser/src/grammar/attributes.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/attributes.rs rename to crates/ra_parser/src/grammar/attributes.rs diff --git a/crates/ra_syntax/src/parsing/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/expressions.rs rename to crates/ra_parser/src/grammar/expressions.rs diff --git a/crates/ra_syntax/src/parsing/grammar/expressions/atom.rs b/crates/ra_parser/src/grammar/expressions/atom.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/expressions/atom.rs rename to crates/ra_parser/src/grammar/expressions/atom.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items.rs b/crates/ra_parser/src/grammar/items.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items.rs rename to crates/ra_parser/src/grammar/items.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/consts.rs b/crates/ra_parser/src/grammar/items/consts.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/consts.rs rename to crates/ra_parser/src/grammar/items/consts.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/nominal.rs b/crates/ra_parser/src/grammar/items/nominal.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/nominal.rs rename to crates/ra_parser/src/grammar/items/nominal.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/traits.rs b/crates/ra_parser/src/grammar/items/traits.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/traits.rs rename to crates/ra_parser/src/grammar/items/traits.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/use_item.rs b/crates/ra_parser/src/grammar/items/use_item.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/use_item.rs rename to crates/ra_parser/src/grammar/items/use_item.rs diff --git a/crates/ra_syntax/src/parsing/grammar/params.rs b/crates/ra_parser/src/grammar/params.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/params.rs rename to crates/ra_parser/src/grammar/params.rs diff --git a/crates/ra_syntax/src/parsing/grammar/paths.rs b/crates/ra_parser/src/grammar/paths.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/paths.rs rename to crates/ra_parser/src/grammar/paths.rs diff --git a/crates/ra_syntax/src/parsing/grammar/patterns.rs b/crates/ra_parser/src/grammar/patterns.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/patterns.rs rename to crates/ra_parser/src/grammar/patterns.rs diff --git a/crates/ra_syntax/src/parsing/grammar/type_args.rs b/crates/ra_parser/src/grammar/type_args.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/type_args.rs rename to crates/ra_parser/src/grammar/type_args.rs diff --git a/crates/ra_syntax/src/parsing/grammar/type_params.rs b/crates/ra_parser/src/grammar/type_params.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/type_params.rs rename to crates/ra_parser/src/grammar/type_params.rs diff --git a/crates/ra_syntax/src/parsing/grammar/types.rs b/crates/ra_parser/src/grammar/types.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/types.rs rename to crates/ra_parser/src/grammar/types.rs diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs new file mode 100644 index 00000000000..fbbac4c69b4 --- /dev/null +++ b/crates/ra_parser/src/lib.rs @@ -0,0 +1,64 @@ +#[macro_use] +mod token_set; +mod syntax_kind; +mod event; +mod parser; +mod grammar; + +pub(crate) use token_set::TokenSet; + +pub use syntax_kind::SyntaxKind; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ParseError(pub String); + +/// `TreeSink` abstracts details of a particular syntax tree implementation. +pub trait TreeSink { + /// Adds new leaf to the current branch. + fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8); + + /// Start new branch and make it current. + fn start_branch(&mut self, kind: SyntaxKind, root: bool); + + /// Finish current branch and restore previous + /// branch as current. + fn finish_branch(&mut self, root: bool); + + fn error(&mut self, error: ParseError); +} + +/// `TokenSource` abstracts the source of the tokens parser operates one. +/// +/// Hopefully this will allow us to treat text and token trees in the same way! +pub trait TokenSource { + fn token_kind(&self, pos: usize) -> SyntaxKind; + fn is_token_joint_to_next(&self, pos: usize) -> bool; + fn is_keyword(&self, pos: usize, kw: &str) -> bool; +} + +pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + let mut p = parser::Parser::new(token_source); + grammar::root(&mut p); + let events = p.finish(); + event::process(tree_sink, events); +} + +pub struct Reparser(fn(&mut parser::Parser)); + +impl Reparser { + pub fn for_node( + node: SyntaxKind, + first_child: Option, + parent: Option, + ) -> Option { + grammar::reparser(node, first_child, parent).map(Reparser) + } +} + +pub fn reparse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, reparser: Reparser) { + let Reparser(r) = reparser; + let mut p = parser::Parser::new(token_source); + r(&mut p); + let events = p.finish(); + event::process(tree_sink, events); +} diff --git a/crates/ra_syntax/src/parsing/parser.rs b/crates/ra_parser/src/parser.rs similarity index 98% rename from crates/ra_syntax/src/parsing/parser.rs rename to crates/ra_parser/src/parser.rs index 923b0f2b20d..a18458148a9 100644 --- a/crates/ra_syntax/src/parsing/parser.rs +++ b/crates/ra_parser/src/parser.rs @@ -4,11 +4,8 @@ use drop_bomb::DropBomb; use crate::{ SyntaxKind::{self, ERROR, EOF, TOMBSTONE}, - parsing::{ - TokenSource, ParseError, - token_set::TokenSet, - event::Event, - }, + TokenSource, ParseError, TokenSet, + event::Event, }; /// `Parser` struct provides the low-level API for diff --git a/crates/ra_syntax/src/syntax_kinds.rs b/crates/ra_parser/src/syntax_kind.rs similarity index 85% rename from crates/ra_syntax/src/syntax_kinds.rs rename to crates/ra_parser/src/syntax_kind.rs index c1118c5ab41..a2353317f87 100644 --- a/crates/ra_syntax/src/syntax_kinds.rs +++ b/crates/ra_parser/src/syntax_kind.rs @@ -2,8 +2,6 @@ mod generated; use std::fmt; -use crate::SyntaxKind::*; - pub use self::generated::SyntaxKind; impl fmt::Debug for SyntaxKind { @@ -20,7 +18,7 @@ pub(crate) struct SyntaxInfo { impl SyntaxKind { pub fn is_trivia(self) -> bool { match self { - WHITESPACE | COMMENT => true, + SyntaxKind::WHITESPACE | SyntaxKind::COMMENT => true, _ => false, } } diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs similarity index 99% rename from crates/ra_syntax/src/syntax_kinds/generated.rs rename to crates/ra_parser/src/syntax_kind/generated.rs index 266b95bbb19..1d8f988aed0 100644 --- a/crates/ra_syntax/src/syntax_kinds/generated.rs +++ b/crates/ra_parser/src/syntax_kind/generated.rs @@ -568,7 +568,7 @@ impl SyntaxKind { EOF => &SyntaxInfo { name: "EOF" }, } } - pub(crate) fn from_keyword(ident: &str) -> Option { + pub fn from_keyword(ident: &str) -> Option { let kw = match ident { "use" => USE_KW, "fn" => FN_KW, @@ -610,7 +610,7 @@ impl SyntaxKind { Some(kw) } - pub(crate) fn from_char(c: char) -> Option { + pub fn from_char(c: char) -> Option { let tok = match c { ';' => SEMI, ',' => COMMA, diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera b/crates/ra_parser/src/syntax_kind/generated.rs.tera similarity index 100% rename from crates/ra_syntax/src/syntax_kinds/generated.rs.tera rename to crates/ra_parser/src/syntax_kind/generated.rs.tera diff --git a/crates/ra_syntax/src/parsing/token_set.rs b/crates/ra_parser/src/token_set.rs similarity index 100% rename from crates/ra_syntax/src/parsing/token_set.rs rename to crates/ra_parser/src/token_set.rs