Merge #11118

11118: internal: move ws attachment logic to the parser crate r=matklad a=matklad

This has to re-introduce the `sink` pattern, because doing this purely
with iterators is awkward :( Maaaybe the event vector was a false start?

But, anyway, I like the current factoring more -- it sort-of obvious
that we do want to keep ws-attachment business in the parser, and that
we also don't want that to depend on the particular tree structure. I
think `shortcuts` module achieves that.

bors r+
🤖

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/parser/src/lexed_str.rs

@@ -122,31 +122,6 @@ pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ {
         self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
     }
 
-    pub fn to_input(&self) -> crate::Input {
-        let mut res = crate::Input::default();
-        let mut was_joint = false;
-        for i in 0..self.len() {
-            let kind = self.kind(i);
-            if kind.is_trivia() {
-                was_joint = false
-            } else {
-                if kind == SyntaxKind::IDENT {
-                    let token_text = self.text(i);
-                    let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)
-                        .unwrap_or(SyntaxKind::IDENT);
-                    res.push_ident(contextual_kw);
-                } else {
-                    if was_joint {
-                        res.was_joint();
-                    }
-                    res.push(kind);
-                }
-                was_joint = true;
-            }
-        }
-        res
-    }
-
     fn push(&mut self, kind: SyntaxKind, offset: usize) {
         self.kind.push(kind);
         self.start.push(offset as u32);

crates/parser/src/lib.rs

@@ -26,6 +26,7 @@
 mod grammar;
 mod input;
 mod output;
+mod shortcuts;
 
 #[cfg(test)]
 mod tests;
@@ -36,6 +37,7 @@
     input::Input,
     lexed_str::LexedStr,
     output::{Output, Step},
+    shortcuts::StrStep,
     syntax_kind::SyntaxKind,
 };
 

crates/parser/src/shortcuts.rs

@@ -0,0 +1,220 @@
+//! Shortcuts that span lexer/parser abstraction.
+//!
+//! The way Rust works, parser doesn't necessary parse text, and you might
+//! tokenize text without parsing it further. So, it makes sense to keep
+//! abstract token parsing, and string tokenization as completely separate
+//! layers.
+//!
+//! However, often you do pares text into syntax trees and the glue code for
+//! that needs to live somewhere. Rather than putting it to lexer or parser, we
+//! use a separate shortcuts module for that.
+
+use std::mem;
+
+use crate::{
+    LexedStr, Step,
+    SyntaxKind::{self, *},
+};
+
+pub enum StrStep<'a> {
+    Token { kind: SyntaxKind, text: &'a str },
+    Enter { kind: SyntaxKind },
+    Exit,
+    Error { msg: &'a str, pos: usize },
+}
+
+impl<'a> LexedStr<'a> {
+    pub fn to_input(&self) -> crate::Input {
+        let mut res = crate::Input::default();
+        let mut was_joint = false;
+        for i in 0..self.len() {
+            let kind = self.kind(i);
+            if kind.is_trivia() {
+                was_joint = false
+            } else {
+                if kind == SyntaxKind::IDENT {
+                    let token_text = self.text(i);
+                    let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)
+                        .unwrap_or(SyntaxKind::IDENT);
+                    res.push_ident(contextual_kw);
+                } else {
+                    if was_joint {
+                        res.was_joint();
+                    }
+                    res.push(kind);
+                }
+                was_joint = true;
+            }
+        }
+        res
+    }
+
+    pub fn intersperse_trivia(
+        &self,
+        output: &crate::Output,
+        synthetic_root: bool,
+        sink: &mut dyn FnMut(StrStep),
+    ) -> bool {
+        let mut builder = Builder { lexed: self, pos: 0, state: State::PendingEnter, sink };
+
+        if synthetic_root {
+            builder.enter(SyntaxKind::SOURCE_FILE);
+        }
+        for event in output.iter() {
+            match event {
+                Step::Token { kind, n_input_tokens: n_raw_tokens } => {
+                    builder.token(kind, n_raw_tokens)
+                }
+                Step::Enter { kind } => builder.enter(kind),
+                Step::Exit => builder.exit(),
+                Step::Error { msg } => {
+                    let text_pos = builder.lexed.text_start(builder.pos);
+                    (builder.sink)(StrStep::Error { msg, pos: text_pos });
+                }
+            }
+        }
+        if synthetic_root {
+            builder.exit();
+        }
+
+        match mem::replace(&mut builder.state, State::Normal) {
+            State::PendingExit => {
+                builder.eat_trivias();
+                (builder.sink)(StrStep::Exit);
+            }
+            State::PendingEnter | State::Normal => unreachable!(),
+        }
+
+        let is_eof = builder.pos == builder.lexed.len();
+        is_eof
+    }
+}
+
+struct Builder<'a, 'b> {
+    lexed: &'a LexedStr<'a>,
+    pos: usize,
+    state: State,
+    sink: &'b mut dyn FnMut(StrStep<'_>),
+}
+
+enum State {
+    PendingEnter,
+    Normal,
+    PendingExit,
+}
+
+impl Builder<'_, '_> {
+    fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
+        match mem::replace(&mut self.state, State::Normal) {
+            State::PendingEnter => unreachable!(),
+            State::PendingExit => (self.sink)(StrStep::Exit),
+            State::Normal => (),
+        }
+        self.eat_trivias();
+        self.do_token(kind, n_tokens as usize);
+    }
+
+    fn enter(&mut self, kind: SyntaxKind) {
+        match mem::replace(&mut self.state, State::Normal) {
+            State::PendingEnter => {
+                (self.sink)(StrStep::Enter { kind });
+                // No need to attach trivias to previous node: there is no
+                // previous node.
+                return;
+            }
+            State::PendingExit => (self.sink)(StrStep::Exit),
+            State::Normal => (),
+        }
+
+        let n_trivias =
+            (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
+        let leading_trivias = self.pos..self.pos + n_trivias;
+        let n_attached_trivias = n_attached_trivias(
+            kind,
+            leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
+        );
+        self.eat_n_trivias(n_trivias - n_attached_trivias);
+        (self.sink)(StrStep::Enter { kind });
+        self.eat_n_trivias(n_attached_trivias);
+    }
+
+    fn exit(&mut self) {
+        match mem::replace(&mut self.state, State::PendingExit) {
+            State::PendingEnter => unreachable!(),
+            State::PendingExit => (self.sink)(StrStep::Exit),
+            State::Normal => (),
+        }
+    }
+
+    fn eat_trivias(&mut self) {
+        while self.pos < self.lexed.len() {
+            let kind = self.lexed.kind(self.pos);
+            if !kind.is_trivia() {
+                break;
+            }
+            self.do_token(kind, 1);
+        }
+    }
+
+    fn eat_n_trivias(&mut self, n: usize) {
+        for _ in 0..n {
+            let kind = self.lexed.kind(self.pos);
+            assert!(kind.is_trivia());
+            self.do_token(kind, 1);
+        }
+    }
+
+    fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) {
+        let text = &self.lexed.range_text(self.pos..self.pos + n_tokens);
+        self.pos += n_tokens;
+        (self.sink)(StrStep::Token { kind, text });
+    }
+}
+
+fn n_attached_trivias<'a>(
+    kind: SyntaxKind,
+    trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
+) -> usize {
+    match kind {
+        CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD
+        | STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => {
+            let mut res = 0;
+            let mut trivias = trivias.enumerate().peekable();
+
+            while let Some((i, (kind, text))) = trivias.next() {
+                match kind {
+                    WHITESPACE if text.contains("\n\n") => {
+                        // we check whether the next token is a doc-comment
+                        // and skip the whitespace in this case
+                        if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
+                            if is_outer(peek_text) {
+                                continue;
+                            }
+                        }
+                        break;
+                    }
+                    COMMENT => {
+                        if is_inner(text) {
+                            break;
+                        }
+                        res = i + 1;
+                    }
+                    _ => (),
+                }
+            }
+            res
+        }
+        _ => 0,
+    }
+}
+
+fn is_outer(text: &str) -> bool {
+    if text.starts_with("////") || text.starts_with("/***") {
+        return false;
+    }
+    text.starts_with("///") || text.starts_with("/**")
+}
+
+fn is_inner(text: &str) -> bool {
+    text.starts_with("//!") || text.starts_with("/*!")
+}

crates/syntax/src/parsing.rs

@@ -1,12 +1,11 @@
 //! Lexing, bridging to parser (which does the actual parsing) and
 //! incremental reparsing.
 
-mod text_tree_sink;
 mod reparsing;
 
-use crate::{
+use rowan::TextRange;
-    parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode,
+
-};
+use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode, SyntaxTreeBuilder};
 
 pub(crate) use crate::parsing::reparsing::incremental_reparse;
 
@@ -37,3 +36,32 @@ pub(crate) fn parse_text_as<T: AstNode>(
 
     SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(())
 }
+
+pub(crate) fn build_tree(
+    lexed: parser::LexedStr<'_>,
+    parser_output: parser::Output,
+    synthetic_root: bool,
+) -> (GreenNode, Vec<SyntaxError>, bool) {
+    let mut builder = SyntaxTreeBuilder::default();
+
+    let is_eof = lexed.intersperse_trivia(&parser_output, synthetic_root, &mut |step| match step {
+        parser::StrStep::Token { kind, text } => builder.token(kind, text),
+        parser::StrStep::Enter { kind } => builder.start_node(kind),
+        parser::StrStep::Exit => builder.finish_node(),
+        parser::StrStep::Error { msg, pos } => {
+            builder.error(msg.to_string(), pos.try_into().unwrap())
+        }
+    });
+
+    let (node, mut errors) = builder.finish_raw();
+    for (i, err) in lexed.errors() {
+        let text_range = lexed.text_range(i);
+        let text_range = TextRange::new(
+            text_range.start.try_into().unwrap(),
+            text_range.end.try_into().unwrap(),
+        );
+        errors.push(SyntaxError::new(err, text_range))
+    }
+
+    (node, errors, is_eof)
+}

crates/syntax/src/parsing/reparsing.rs

@@ -10,7 +10,7 @@
 use text_edit::Indel;
 
 use crate::{
-    parsing::text_tree_sink::build_tree,
+    parsing::build_tree,
     syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
     SyntaxError,
     SyntaxKind::*,

crates/syntax/src/parsing/text_tree_sink.rs

@@ -1,191 +0,0 @@
-//! See [`TextTreeSink`].
-
-use std::mem;
-
-use parser::LexedStr;
-
-use crate::{
-    ast,
-    syntax_node::GreenNode,
-    SyntaxError,
-    SyntaxKind::{self, *},
-    SyntaxTreeBuilder, TextRange,
-};
-
-pub(crate) fn build_tree(
-    lexed: LexedStr<'_>,
-    parser_output: parser::Output,
-    synthetic_root: bool,
-) -> (GreenNode, Vec<SyntaxError>, bool) {
-    let mut builder = Builder::new(lexed);
-
-    if synthetic_root {
-        builder.enter(SyntaxKind::SOURCE_FILE);
-    }
-
-    for event in parser_output.iter() {
-        match event {
-            parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
-                builder.token(kind, n_raw_tokens)
-            }
-            parser::Step::Enter { kind } => builder.enter(kind),
-            parser::Step::Exit => builder.exit(),
-            parser::Step::Error { msg } => {
-                let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();
-                builder.inner.error(msg.to_string(), text_pos);
-            }
-        }
-    }
-    if synthetic_root {
-        builder.exit()
-    }
-    builder.build()
-}
-
-struct Builder<'a> {
-    lexed: LexedStr<'a>,
-    pos: usize,
-    state: State,
-    inner: SyntaxTreeBuilder,
-}
-
-enum State {
-    PendingStart,
-    Normal,
-    PendingFinish,
-}
-
-impl<'a> Builder<'a> {
-    fn new(lexed: parser::LexedStr<'a>) -> Self {
-        Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() }
-    }
-
-    fn build(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {
-        match mem::replace(&mut self.state, State::Normal) {
-            State::PendingFinish => {
-                self.eat_trivias();
-                self.inner.finish_node();
-            }
-            State::PendingStart | State::Normal => unreachable!(),
-        }
-
-        let (node, mut errors) = self.inner.finish_raw();
-        for (i, err) in self.lexed.errors() {
-            let text_range = self.lexed.text_range(i);
-            let text_range = TextRange::new(
-                text_range.start.try_into().unwrap(),
-                text_range.end.try_into().unwrap(),
-            );
-            errors.push(SyntaxError::new(err, text_range))
-        }
-
-        let is_eof = self.pos == self.lexed.len();
-
-        (node, errors, is_eof)
-    }
-
-    fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
-        match mem::replace(&mut self.state, State::Normal) {
-            State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_node(),
-            State::Normal => (),
-        }
-        self.eat_trivias();
-        self.do_token(kind, n_tokens as usize);
-    }
-
-    fn enter(&mut self, kind: SyntaxKind) {
-        match mem::replace(&mut self.state, State::Normal) {
-            State::PendingStart => {
-                self.inner.start_node(kind);
-                // No need to attach trivias to previous node: there is no
-                // previous node.
-                return;
-            }
-            State::PendingFinish => self.inner.finish_node(),
-            State::Normal => (),
-        }
-
-        let n_trivias =
-            (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count();
-        let leading_trivias = self.pos..self.pos + n_trivias;
-        let n_attached_trivias = n_attached_trivias(
-            kind,
-            leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))),
-        );
-        self.eat_n_trivias(n_trivias - n_attached_trivias);
-        self.inner.start_node(kind);
-        self.eat_n_trivias(n_attached_trivias);
-    }
-
-    fn exit(&mut self) {
-        match mem::replace(&mut self.state, State::PendingFinish) {
-            State::PendingStart => unreachable!(),
-            State::PendingFinish => self.inner.finish_node(),
-            State::Normal => (),
-        }
-    }
-
-    fn eat_trivias(&mut self) {
-        while self.pos < self.lexed.len() {
-            let kind = self.lexed.kind(self.pos);
-            if !kind.is_trivia() {
-                break;
-            }
-            self.do_token(kind, 1);
-        }
-    }
-
-    fn eat_n_trivias(&mut self, n: usize) {
-        for _ in 0..n {
-            let kind = self.lexed.kind(self.pos);
-            assert!(kind.is_trivia());
-            self.do_token(kind, 1);
-        }
-    }
-
-    fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) {
-        let text = &self.lexed.range_text(self.pos..self.pos + n_tokens);
-        self.pos += n_tokens;
-        self.inner.token(kind, text);
-    }
-}
-
-fn n_attached_trivias<'a>(
-    kind: SyntaxKind,
-    trivias: impl Iterator<Item = (SyntaxKind, &'a str)>,
-) -> usize {
-    match kind {
-        CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD
-        | STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => {
-            let mut res = 0;
-            let mut trivias = trivias.enumerate().peekable();
-
-            while let Some((i, (kind, text))) = trivias.next() {
-                match kind {
-                    WHITESPACE if text.contains("\n\n") => {
-                        // we check whether the next token is a doc-comment
-                        // and skip the whitespace in this case
-                        if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) {
-                            let comment_kind = ast::CommentKind::from_text(peek_text);
-                            if comment_kind.doc == Some(ast::CommentPlacement::Outer) {
-                                continue;
-                            }
-                        }
-                        break;
-                    }
-                    COMMENT => {
-                        let comment_kind = ast::CommentKind::from_text(text);
-                        if comment_kind.doc == Some(ast::CommentPlacement::Inner) {
-                            break;
-                        }
-                        res = i + 1;
-                    }
-                    _ => (),
-                }
-            }
-            res
-        }
-        _ => 0,
-    }
-}