From 46422f722bdcadbf4462dd5a9c65756434b2d97a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Mon, 1 Jan 2018 22:13:04 +0300 Subject: [PATCH] Parser: first scraches --- grammar.ron | 1 + src/parser/event_parser/grammar.rs | 6 ++++ src/parser/event_parser/mod.rs | 1 + src/parser/event_parser/parser.rs | 22 ++++++++++---- src/parser/mod.rs | 41 +++++++++++++++++++++++--- src/syntax_kinds.rs | 4 ++- src/tree/file_builder.rs | 24 +++++++++++---- tests/data/parser/0001_struct_item.rs | 3 ++ tests/data/parser/0001_struct_item.txt | 12 ++++++++ 9 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 tests/data/parser/0001_struct_item.rs create mode 100644 tests/data/parser/0001_struct_item.txt diff --git a/grammar.ron b/grammar.ron index fb2c6d90ed3..c4f6283e61e 100644 --- a/grammar.ron +++ b/grammar.ron @@ -62,5 +62,6 @@ Grammar( ], nodes: [ "FILE", + "STRUCT_ITEM" ] ) \ No newline at end of file diff --git a/src/parser/event_parser/grammar.rs b/src/parser/event_parser/grammar.rs index 5219ed535de..0896506fb1b 100644 --- a/src/parser/event_parser/grammar.rs +++ b/src/parser/event_parser/grammar.rs @@ -40,6 +40,12 @@ fn mod_items(p: &mut Parser) { fn item(p: &mut Parser) -> Result { outer_attributes(p)?; visibility(p)?; + if p.current_is(STRUCT_KW) { + p.start(STRUCT_ITEM); + p.bump(); + p.finish(); + return OK; + } ERR } diff --git a/src/parser/event_parser/mod.rs b/src/parser/event_parser/mod.rs index bdfd23974ab..1228236a963 100644 --- a/src/parser/event_parser/mod.rs +++ b/src/parser/event_parser/mod.rs @@ -4,6 +4,7 @@ mod grammar; mod parser; +#[derive(Debug)] pub(crate) enum Event { Start { kind: SyntaxKind }, Finish, diff --git a/src/parser/event_parser/parser.rs b/src/parser/event_parser/parser.rs index 0e4d44b7984..2d5418a29a9 100644 --- a/src/parser/event_parser/parser.rs +++ b/src/parser/event_parser/parser.rs @@ -1,5 +1,6 @@ use {Token, SyntaxKind, TextUnit}; -use super::Event; +use super::{Event}; +use super::super::is_insignificant; use syntax_kinds::{WHITESPACE, COMMENT}; pub struct Parser<'t> { @@ -16,9 +17,8 @@ pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> { let mut non_ws_tokens = Vec::new(); let mut len = TextUnit::new(0); for (idx, &token) in raw_tokens.iter().enumerate() { - match token.kind { - WHITESPACE | COMMENT => (), - _ => non_ws_tokens.push((idx, len)), + if !is_insignificant(token.kind) { + non_ws_tokens.push((idx, len)) } len += token.len; } @@ -50,15 +50,25 @@ pub(crate) fn finish(&mut self) { self.event(Event::Finish); } - pub(crate) fn bump(&mut self) -> Option { + pub(crate) fn current(&self) -> Option { if self.is_eof() { return None; } let idx = self.non_ws_tokens[self.pos].0; - self.pos += 1; Some(self.raw_tokens[idx].kind) } + pub(crate) fn current_is(&self, kind: SyntaxKind) -> bool { + self.current() == Some(kind) + } + + pub(crate) fn bump(&mut self) -> Option { + let kind = self.current()?; + self.pos += 1; + self.event(Event::Token { kind, n_raw_tokens: 1 }); + Some(kind) + } + fn event(&mut self, event: Event) { self.events.push(event) } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index ccccd78f9ea..a632fbc019e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8,17 +8,50 @@ pub fn parse(text: String, tokens: &[Token]) -> File { let events = event_parser::parse(&text, tokens); - from_events_to_file(text, events) + from_events_to_file(text, tokens, events) } -fn from_events_to_file(text: String, events: Vec) -> File { +fn from_events_to_file( + text: String, + tokens: &[Token], + events: Vec, +) -> File { let mut builder = FileBuilder::new(text); + let mut idx = 0; for event in events { match event { Event::Start { kind } => builder.start_internal(kind), - Event::Finish => builder.finish_internal(), - Event::Token { .. } => unimplemented!(), + Event::Finish => { + while idx < tokens.len() { + let token = tokens[idx]; + if is_insignificant(token.kind) { + idx += 1; + builder.leaf(token.kind, token.len); + } else { + break; + } + } + builder.finish_internal() + }, + Event::Token { kind, mut n_raw_tokens } => loop { + let token = tokens[idx]; + if !is_insignificant(token.kind) { + n_raw_tokens -= 1; + } + idx += 1; + builder.leaf(token.kind, token.len); + if n_raw_tokens == 0 { + break; + } + }, } } builder.finish() } + +fn is_insignificant(kind: SyntaxKind) -> bool { + match kind { + WHITESPACE | COMMENT => true, + _ => false, + } +} \ No newline at end of file diff --git a/src/syntax_kinds.rs b/src/syntax_kinds.rs index a1bcad06254..d9b1462a7f1 100644 --- a/src/syntax_kinds.rs +++ b/src/syntax_kinds.rs @@ -59,8 +59,9 @@ pub const DOC_COMMENT: SyntaxKind = SyntaxKind(55); pub const SHEBANG: SyntaxKind = SyntaxKind(56); pub const FILE: SyntaxKind = SyntaxKind(57); +pub const STRUCT_ITEM: SyntaxKind = SyntaxKind(58); -static INFOS: [SyntaxInfo; 58] = [ +static INFOS: [SyntaxInfo; 59] = [ SyntaxInfo { name: "USE_KW" }, SyntaxInfo { name: "FN_KW" }, SyntaxInfo { name: "STRUCT_KW" }, @@ -119,6 +120,7 @@ SyntaxInfo { name: "DOC_COMMENT" }, SyntaxInfo { name: "SHEBANG" }, SyntaxInfo { name: "FILE" }, + SyntaxInfo { name: "STRUCT_ITEM" }, ]; pub(crate) fn syntax_info(kind: SyntaxKind) -> &'static SyntaxInfo { diff --git a/src/tree/file_builder.rs b/src/tree/file_builder.rs index eba850fef0f..da8b8f82400 100644 --- a/src/tree/file_builder.rs +++ b/src/tree/file_builder.rs @@ -64,8 +64,14 @@ pub fn new(text: String) -> FileBuilder { } pub fn finish(self) -> File { - assert!(self.in_progress.is_empty()); - assert!(self.pos == (self.text.len() as u32).into()); + assert!( + self.in_progress.is_empty(), + "some nodes in FileBuilder are unfinished" + ); + assert!( + self.pos == (self.text.len() as u32).into(), + "nodes in FileBuilder do not cover the whole file" + ); File { text: self.text, nodes: self.nodes, @@ -81,11 +87,17 @@ fn new_node(&mut self, data: NodeData) -> NodeIdx { fn push_child(&mut self, mut child: NodeData) -> NodeIdx { child.parent = Some(self.current_id()); let id = self.new_node(child); - if let Some(sibling) = self.current_sibling() { - fill(&mut sibling.next_sibling, id); - return id + { + + let (parent, sibling) = *self.in_progress.last().unwrap(); + let slot = if let Some(idx) = sibling { + &mut self.nodes[idx].next_sibling + } else { + &mut self.nodes[parent].first_child + }; + fill(slot, id); } - fill(&mut self.current_parent().first_child, id); + self.in_progress.last_mut().unwrap().1 = Some(id); id } diff --git a/tests/data/parser/0001_struct_item.rs b/tests/data/parser/0001_struct_item.rs new file mode 100644 index 00000000000..d3a8c1d23d4 --- /dev/null +++ b/tests/data/parser/0001_struct_item.rs @@ -0,0 +1,3 @@ +struct S { + +} \ No newline at end of file diff --git a/tests/data/parser/0001_struct_item.txt b/tests/data/parser/0001_struct_item.txt new file mode 100644 index 00000000000..5ef54428234 --- /dev/null +++ b/tests/data/parser/0001_struct_item.txt @@ -0,0 +1,12 @@ +FILE@[0; 13) + STRUCT_ITEM@[0; 7) + STRUCT_KW@[0; 6) + WHITESPACE@[6; 7) + ERROR@[7; 9) + IDENT@[7; 8) + WHITESPACE@[8; 9) + ERROR@[9; 12) + L_CURLY@[9; 10) + WHITESPACE@[10; 12) + ERROR@[12; 13) + R_CURLY@[12; 13)