mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

doc parsing events

Browse Source
This commit is contained in:
csmoe 2019-01-01 16:09:51 +08:00
parent b01e707dba
commit df591a1e48
4 changed files with 71 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
crates/ra_syntax/src/parser_api.rs
View File

@ -142,11 +142,13 @@ fn new(pos: u32) -> Marker {
}
}
/// Finishes the syntax tree node and assigns `kind` to it.
/// Finishes the syntax tree node and assigns `kind` to it,
/// and mark the create a `CompletedMarker` for possible future
/// operation like `.precede()` to deal with forward_parent.
pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker {
self.bomb.defuse();
p.0.complete(self.pos, kind);
CompletedMarker(self.pos, kind)
CompletedMarker::new(self.pos, kind)
}
/// Abandons the syntax tree node. All its children
@ -160,13 +162,22 @@ pub(crate) fn abandon(mut self, p: &mut Parser) {
pub(crate) struct CompletedMarker(u32, SyntaxKind);
impl CompletedMarker {
/// This one is tricky :-)
fn new(pos: u32, kind: SyntaxKind) -> Self {
CompletedMarker(pos, kind)
}
/// This method allows to create a new node which starts
/// *before* the current one. That is, parser could start
/// node `A`, then complete it, and then after parsing the
/// whole `A`, decide that it should have started some node
/// `B` before starting `A`. `precede` allows to do exactly
/// that. See also docs about `forward_parent` in `Event::Start`.
///
/// Given completed events `[START, FINISH]` and its corresponding
/// `CompletedMarker(pos: 0, _)`.
/// Append a new `START` events as `[START, FINISH, NEWSTART]`,
/// then mark `NEWSTART` as `START`'s parent with saving its relative
/// distance to `NEWSTART` into forward_parent(=2 in this case);
pub(crate) fn precede(self, p: &mut Parser) -> Marker {
Marker::new(p.0.precede(self.0))
}

55
crates/ra_syntax/src/parser_impl.rs
View File

@ -63,7 +63,7 @@ pub(crate) fn parse_with<S: Sink>(
/// to a separate struct in order not to pollute
/// the public API of the `Parser`.
pub(crate) struct ParserImpl<'t> {
inp: &'t ParserInput<'t>,
parser_input: &'t ParserInput<'t>,
pos: InputPosition,
events: Vec<Event>,
steps: Cell<u32>,
@ -72,7 +72,7 @@ pub(crate) struct ParserImpl<'t> {
impl<'t> ParserImpl<'t> {
pub(crate) fn new(inp: &'t ParserInput<'t>) -> ParserImpl<'t> {
ParserImpl {
inp,
parser_input: inp,
pos: InputPosition::new(),
events: Vec::new(),
steps: Cell::new(0),
@ -85,10 +85,10 @@ pub(crate) fn into_events(self) -> Vec<Event> {
}
pub(super) fn next2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
let c1 = self.inp.kind(self.pos);
let c2 = self.inp.kind(self.pos + 1);
if self.inp.token_start_at(self.pos + 1)
== self.inp.token_start_at(self.pos) + self.inp.len(self.pos)
let c1 = self.parser_input.kind(self.pos);
let c2 = self.parser_input.kind(self.pos + 1);
if self.parser_input.token_start_at(self.pos + 1)
== self.parser_input.token_start_at(self.pos) + self.parser_input.token_len(self.pos)
{
Some((c1, c2))
} else {
@ -97,13 +97,14 @@ pub(super) fn next2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
}
pub(super) fn next3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
let c1 = self.inp.kind(self.pos);
let c2 = self.inp.kind(self.pos + 1);
let c3 = self.inp.kind(self.pos + 2);
if self.inp.token_start_at(self.pos + 1)
== self.inp.token_start_at(self.pos) + self.inp.len(self.pos)
&& self.inp.token_start_at(self.pos + 2)
== self.inp.token_start_at(self.pos + 1) + self.inp.len(self.pos + 1)
let c1 = self.parser_input.kind(self.pos);
let c2 = self.parser_input.kind(self.pos + 1);
let c3 = self.parser_input.kind(self.pos + 2);
if self.parser_input.token_start_at(self.pos + 1)
== self.parser_input.token_start_at(self.pos) + self.parser_input.token_len(self.pos)
&& self.parser_input.token_start_at(self.pos + 2)
== self.parser_input.token_start_at(self.pos + 1)
+ self.parser_input.token_len(self.pos + 1)
{
Some((c1, c2, c3))
} else {
@ -111,29 +112,27 @@ pub(super) fn next3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
}
}
/// Get the syntax kind of the nth token.
pub(super) fn nth(&self, n: u32) -> SyntaxKind {
let steps = self.steps.get();
if steps > 10_000_000 {
panic!("the parser seems stuck");
}
assert!(steps <= 10_000_000, "the parser seems stuck");
self.steps.set(steps + 1);
self.inp.kind(self.pos + n)
self.parser_input.kind(self.pos + n)
}
pub(super) fn at_kw(&self, t: &str) -> bool {
self.inp.text(self.pos) == t
self.parser_input.token_text(self.pos) == t
}
/// Start parsing right behind the last event.
pub(super) fn start(&mut self) -> u32 {
let pos = self.events.len() as u32;
self.event(Event::Start {
kind: TOMBSTONE,
forward_parent: None,
});
self.push_event(Event::tombstone());
pos
}
/// Advances the parser by one token unconditionally.
pub(super) fn bump(&mut self) {
let kind = self.nth(0);
if kind == EOF {
@ -156,15 +155,17 @@ pub(super) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) {
fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
self.pos += u32::from(n_raw_tokens);
self.event(Event::Token { kind, n_raw_tokens });
self.push_event(Event::Token { kind, n_raw_tokens });
}
/// Append one Error event to the back of events.
pub(super) fn error(&mut self, msg: String) {
self.event(Event::Error {
self.push_event(Event::Error {
msg: ParseError(msg),
})
}
/// Complete an event with appending a `Finish` event.
pub(super) fn complete(&mut self, pos: u32, kind: SyntaxKind) {
match self.events[pos as usize] {
Event::Start {
@ -174,9 +175,10 @@ pub(super) fn complete(&mut self, pos: u32, kind: SyntaxKind) {
}
_ => unreachable!(),
}
self.event(Event::Finish);
self.push_event(Event::Finish);
}
/// Ignore the dummy `Start` event.
pub(super) fn abandon(&mut self, pos: u32) {
let idx = pos as usize;
if idx == self.events.len() - 1 {
@ -190,6 +192,7 @@ pub(super) fn abandon(&mut self, pos: u32) {
}
}
/// Save the relative distance of a completed event to its forward_parent.
pub(super) fn precede(&mut self, pos: u32) -> u32 {
let new_pos = self.start();
match self.events[pos as usize] {
@ -204,7 +207,7 @@ pub(super) fn precede(&mut self, pos: u32) -> u32 {
new_pos
}
fn event(&mut self, event: Event) {
fn push_event(&mut self, event: Event) {
self.events.push(event)
}
}

40
crates/ra_syntax/src/parser_impl/event.rs
View File

@ -36,7 +36,7 @@ pub(crate) enum Event {
///
/// For left-recursive syntactic constructs, the parser produces
/// a child node before it sees a parent. `forward_parent`
/// exists to allow to tweak parent-child relationships.
/// saves the position of current event's parent.
///
/// Consider this path
///
@ -84,6 +84,15 @@ pub(crate) enum Event {
},
}
impl Event {
pub(crate) fn tombstone() -> Self {
Event::Start {
kind: TOMBSTONE,
forward_parent: None,
}
}
}
pub(super) struct EventProcessor<'a, S: Sink> {
sink: S,
text_pos: TextUnit,
@ -110,17 +119,12 @@ pub(super) fn new(
}
}
/// Generate the syntax tree with the control of events.
pub(super) fn process(mut self) -> S {
fn tombstone() -> Event {
Event::Start {
kind: TOMBSTONE,
forward_parent: None,
}
}
let mut forward_parents = Vec::new();
for i in 0..self.events.len() {
match mem::replace(&mut self.events[i], tombstone()) {
match mem::replace(&mut self.events[i], Event::tombstone()) {
Event::Start {
kind: TOMBSTONE, ..
} => (),
@ -129,12 +133,18 @@ fn tombstone() -> Event {
kind,
forward_parent,
} => {
// For events[A, B, C], B is A's forward_parent, C is B's forward_parent,
// in the normal control flow, the parent-child relation: `A -> B -> C`,
// while with the magic forward_parent, it writes: `C <- B <- A`.
// append `A` into parents.
forward_parents.push(kind);
let mut idx = i;
let mut fp = forward_parent;
while let Some(fwd) = fp {
idx += fwd as usize;
fp = match mem::replace(&mut self.events[idx], tombstone()) {
// append `A`'s forward_parent `B`
fp = match mem::replace(&mut self.events[idx], Event::tombstone()) {
Event::Start {
kind,
forward_parent,
@ -144,14 +154,16 @@ fn tombstone() -> Event {
}
_ => unreachable!(),
};
// append `B`'s forward_parent `C` in the next stage.
}
for kind in forward_parents.drain(..).rev() {
self.start(kind);
}
}
Event::Finish => {
let last = i == self.events.len() - 1;
self.finish(last);
let is_last = i == self.events.len() - 1;
self.finish(is_last);
}
Event::Token { kind, n_raw_tokens } => {
self.eat_trivias();
@ -171,6 +183,7 @@ fn tombstone() -> Event {
self.sink
}
/// Add the node into syntax tree but discard the comments/whitespaces.
fn start(&mut self, kind: SyntaxKind) {
if kind == SOURCE_FILE {
self.sink.start_branch(kind);
@ -198,8 +211,8 @@ fn start(&mut self, kind: SyntaxKind) {
self.eat_n_trivias(n_attached_trivias);
}
fn finish(&mut self, last: bool) {
if last {
fn finish(&mut self, is_last: bool) {
if is_last {
self.eat_trivias()
}
self.sink.finish_branch();
@ -235,6 +248,7 @@ fn n_attached_trivias<'a>(
kind: SyntaxKind,
trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
) -> usize {
// FIXME: parse attached trivias of CONST_DEF/TYPE_DEF
match kind {
STRUCT_DEF | ENUM_DEF | FN_DEF | TRAIT_DEF | MODULE => {
let mut res = 0;

2
crates/ra_syntax/src/parser_impl/input.rs
View File

@ -70,7 +70,7 @@ pub fn token_start_at(&self, pos: InputPosition) -> TextUnit {
self.start_offsets[idx]
}
/// Get the raw text of a toen at given input position.
/// Get the raw text of a token at given input position.
pub fn token_text(&self, pos: InputPosition) -> &'t str {
let idx = pos.0 as usize;
if !(idx < self.tokens.len()) {