Merge #1157

1157: Add mbe stmt matcher r=matklad a=edwin0cheng

Add `stmt` matcher in `ra_mbe` , and added corresponding `stmt()` parser in `ra_syntax`. 
This PR also help PR #1148 for `MarcoKind::Items` parsing.

Note: 
* According [the book](https://doc.rust-lang.org/reference/macros-by-example.html), mbe `stmt` matcher will only match statement without the trailing semicolon 
* `item` is a valid statement.





Co-authored-by: Edwin Cheng <edwin0cheng@gmail.com>

crates/ra_mbe/src/lib.rs

@@ -582,4 +582,19 @@ macro_rules! foo {
         );
         assert_expansion(&rules, "foo! { (a, b) }", "fn foo () {let (a , b) ;}");
     }
+
+    #[test]
+    fn test_stmt() {
+        let rules = create_rules(
+            r#"
+        macro_rules! foo {
+            ($ i:stmt) => (
+                fn bar() { $ i; }
+            )
+        }
+"#,
+        );
+        assert_expansion(&rules, "foo! { 2 }", "fn bar () {2 ;}");
+        assert_expansion(&rules, "foo! { let a = 0 }", "fn bar () {let a = 0 ;}");
+    }
 }

crates/ra_mbe/src/mbe_expander.rs

@@ -157,6 +157,10 @@ fn match_lhs(pattern: &crate::Subtree, input: &mut TtCursor) -> Result<Bindings,
                             let pat = input.eat_pat().ok_or(ExpandError::UnexpectedToken)?.clone();
                             res.inner.insert(text.clone(), Binding::Simple(pat.into()));
                         }
+                        "stmt" => {
+                            let pat = input.eat_stmt().ok_or(ExpandError::UnexpectedToken)?.clone();
+                            res.inner.insert(text.clone(), Binding::Simple(pat.into()));
+                        }
                         _ => return Err(ExpandError::UnexpectedToken),
                     }
                 }

crates/ra_mbe/src/subtree_parser.rs

@@ -42,6 +42,10 @@ pub fn parse_pat(self) -> Option<tt::TokenTree> {
         self.parse(ra_parser::parse_pat)
     }
 
+    pub fn parse_stmt(self) -> Option<tt::TokenTree> {
+        self.parse(|src, sink| ra_parser::parse_stmt(src, sink, false))
+    }
+
     fn parse<F>(self, f: F) -> Option<tt::TokenTree>
     where
         F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),

crates/ra_mbe/src/tt_cursor.rs

@@ -99,6 +99,11 @@ pub(crate) fn eat_pat(&mut self) -> Option<tt::TokenTree> {
         parser.parse_pat()
     }
 
+    pub(crate) fn eat_stmt(&mut self) -> Option<tt::TokenTree> {
+        let parser = Parser::new(&mut self.pos, self.subtree);
+        parser.parse_stmt()
+    }
+
     pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ParseError> {
         if self.at_char(char) {
             self.bump();

crates/ra_parser/src/grammar.rs

@@ -65,6 +65,10 @@ pub(crate) fn pattern(p: &mut Parser) {
     patterns::pattern(p)
 }
 
+pub(crate) fn stmt(p: &mut Parser, with_semi: bool) {
+    expressions::stmt(p, with_semi)
+}
+
 pub(crate) fn reparser(
     node: SyntaxKind,
     first_child: Option<SyntaxKind>,

crates/ra_parser/src/grammar/expressions.rs

@@ -48,20 +48,7 @@ fn is_expr_stmt_attr_allowed(kind: SyntaxKind) -> bool {
     }
 }
 
-pub(crate) fn expr_block_contents(p: &mut Parser) {
-    // This is checked by a validator
-    attributes::inner_attributes(p);
-
-    while !p.at(EOF) && !p.at(R_CURLY) {
-        // test nocontentexpr
-        // fn foo(){
-        //     ;;;some_expr();;;;{;;;};;;;Ok(())
-        // }
-        if p.current() == SEMI {
-            p.bump();
-            continue;
-        }
-
+pub(super) fn stmt(p: &mut Parser, with_semi: bool) {
     // test block_items
     // fn a() { fn b() {} }
     let m = p.start();
@@ -74,13 +61,14 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
     // }
     let has_attrs = p.at(POUND);
     attributes::outer_attributes(p);
+
     if p.at(LET_KW) {
-            let_stmt(p, m);
-            continue;
+        let_stmt(p, m, with_semi);
+        return;
     }
 
     let m = match items::maybe_item(p, m, items::ItemFlavor::Mod) {
-            Ok(()) => continue,
+        Ok(()) => return,
         Err(m) => m,
     };
 
@@ -123,13 +111,14 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
         //     }
         //     test!{}
         // }
+        if with_semi {
             if blocklike.is_block() {
                 p.eat(SEMI);
             } else {
                 p.expect(SEMI);
             }
-            m.complete(p, EXPR_STMT);
         }
+        m.complete(p, EXPR_STMT);
     }
 
     // test let_stmt;
@@ -139,7 +128,7 @@ pub(crate) fn expr_block_contents(p: &mut Parser) {
     //     let c = 92;
     //     let d: i32 = 92;
     // }
-    fn let_stmt(p: &mut Parser, m: Marker) {
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: bool) {
         assert!(p.at(LET_KW));
         p.bump();
         patterns::pattern(p);
@@ -149,11 +138,32 @@ fn let_stmt(p: &mut Parser, m: Marker) {
         if p.eat(EQ) {
             expressions::expr(p);
         }
+
+        if with_semi {
             p.expect(SEMI);
+        }
         m.complete(p, LET_STMT);
     }
 }
 
+pub(crate) fn expr_block_contents(p: &mut Parser) {
+    // This is checked by a validator
+    attributes::inner_attributes(p);
+
+    while !p.at(EOF) && !p.at(R_CURLY) {
+        // test nocontentexpr
+        // fn foo(){
+        //     ;;;some_expr();;;;{;;;};;;;Ok(())
+        // }
+        if p.current() == SEMI {
+            p.bump();
+            continue;
+        }
+
+        stmt(p, true)
+    }
+}
+
 #[derive(Clone, Copy)]
 struct Restrictions {
     forbid_structs: bool,

crates/ra_parser/src/lib.rs

@@ -88,6 +88,11 @@ pub fn parse_pat(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
     parse_from_tokens(token_source, tree_sink, grammar::pattern);
 }
 
+/// Parse given tokens into the given sink as a statement
+pub fn parse_stmt(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink, with_semi: bool) {
+    parse_from_tokens(token_source, tree_sink, |p| grammar::stmt(p, with_semi));
+}
+
 /// A parsing function for a specific braced-block.
 pub struct Reparser(fn(&mut parser::Parser));