Merge #11163

11163: internal: start tests for top-level parser entry points r=matklad a=matklad

bors r+
🤖

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/parser/src/tests.rs

@@ -1,5 +1,6 @@
 mod sourcegen_inline_tests;
-mod entries;
+mod prefix_entries;
+mod top_entries;
 
 use std::{
     fmt::Write,
@@ -9,7 +10,7 @@ use std::{
 
 use expect_test::expect_file;
 
-use crate::LexedStr;
+use crate::{LexedStr, TopEntryPoint};
 
 #[test]
 fn lex_ok() {
@@ -45,7 +46,7 @@ fn lex(text: &str) -> String {
 #[test]
 fn parse_ok() {
     for case in TestCase::list("parser/ok") {
-        let (actual, errors) = parse(&case.text);
+        let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
         assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual);
         expect_file![case.txt].assert_eq(&actual);
     }
@@ -54,7 +55,7 @@ fn parse_ok() {
 #[test]
 fn parse_inline_ok() {
     for case in TestCase::list("parser/inline/ok") {
-        let (actual, errors) = parse(&case.text);
+        let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
         assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual);
         expect_file![case.txt].assert_eq(&actual);
     }
@@ -63,7 +64,7 @@ fn parse_inline_ok() {
 #[test]
 fn parse_err() {
     for case in TestCase::list("parser/err") {
-        let (actual, errors) = parse(&case.text);
+        let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
         assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual);
         expect_file![case.txt].assert_eq(&actual)
     }
@@ -72,36 +73,48 @@ fn parse_err() {
 #[test]
 fn parse_inline_err() {
     for case in TestCase::list("parser/inline/err") {
-        let (actual, errors) = parse(&case.text);
+        let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
         assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual);
         expect_file![case.txt].assert_eq(&actual)
     }
 }
 
-fn parse(text: &str) -> (String, bool) {
+fn parse(entry: TopEntryPoint, text: &str) -> (String, bool) {
     let lexed = LexedStr::new(text);
     let input = lexed.to_input();
-    let output = crate::TopEntryPoint::SourceFile.parse(&input);
+    let output = entry.parse(&input);
 
     let mut buf = String::new();
     let mut errors = Vec::new();
     let mut indent = String::new();
+    let mut depth = 0;
+    let mut len = 0;
     lexed.intersperse_trivia(&output, &mut |step| match step {
         crate::StrStep::Token { kind, text } => {
+            assert!(depth > 0);
+            len += text.len();
             write!(buf, "{}", indent).unwrap();
             write!(buf, "{:?} {:?}\n", kind, text).unwrap();
         }
         crate::StrStep::Enter { kind } => {
+            assert!(depth > 0 || len == 0);
+            depth += 1;
             write!(buf, "{}", indent).unwrap();
             write!(buf, "{:?}\n", kind).unwrap();
             indent.push_str("  ");
         }
         crate::StrStep::Exit => {
+            assert!(depth > 0);
+            depth -= 1;
             indent.pop();
             indent.pop();
         }
-        crate::StrStep::Error { msg, pos } => errors.push(format!("error {}: {}\n", pos, msg)),
+        crate::StrStep::Error { msg, pos } => {
+            assert!(depth > 0);
+            errors.push(format!("error {}: {}\n", pos, msg))
+        }
     });
+    assert_eq!(len, text.len());
 
     for (token, msg) in lexed.errors() {
         let pos = lexed.text_start(token);

crates/parser/src/tests/entries.rs

@@ -1,110 +0,0 @@
-use crate::{LexedStr, PrefixEntryPoint, Step};
-
-#[test]
-fn vis() {
-    check_prefix(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)");
-    check_prefix(PrefixEntryPoint::Vis, "fn foo() {}", "");
-    check_prefix(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub");
-    check_prefix(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate");
-    check_prefix(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate");
-}
-
-#[test]
-fn block() {
-    check_prefix(PrefixEntryPoint::Block, "{}, 92", "{}");
-    check_prefix(PrefixEntryPoint::Block, "{, 92)", "{, 92)");
-    check_prefix(PrefixEntryPoint::Block, "()", "");
-}
-
-#[test]
-fn stmt() {
-    check_prefix(PrefixEntryPoint::Stmt, "92; fn", "92");
-    check_prefix(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92");
-    check_prefix(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}");
-    check_prefix(PrefixEntryPoint::Stmt, "struct S;;", "struct S;");
-    check_prefix(PrefixEntryPoint::Stmt, "fn f() {};", "fn f() {}");
-    check_prefix(PrefixEntryPoint::Stmt, ";;;", ";");
-    check_prefix(PrefixEntryPoint::Stmt, "+", "+");
-    check_prefix(PrefixEntryPoint::Stmt, "@", "@");
-    check_prefix(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}");
-}
-
-#[test]
-fn pat() {
-    check_prefix(PrefixEntryPoint::Pat, "x y", "x");
-    check_prefix(PrefixEntryPoint::Pat, "fn f() {}", "fn");
-    // FIXME: This one is wrong, we should consume only one pattern.
-    check_prefix(PrefixEntryPoint::Pat, ".. ..", ".. ..");
-}
-
-#[test]
-fn ty() {
-    check_prefix(PrefixEntryPoint::Ty, "fn() foo", "fn()");
-    check_prefix(PrefixEntryPoint::Ty, "Clone + Copy + fn", "Clone + Copy +");
-    check_prefix(PrefixEntryPoint::Ty, "struct f", "struct");
-}
-
-#[test]
-fn expr() {
-    check_prefix(PrefixEntryPoint::Expr, "92 92", "92");
-    check_prefix(PrefixEntryPoint::Expr, "+1", "+");
-    check_prefix(PrefixEntryPoint::Expr, "-1", "-1");
-    check_prefix(PrefixEntryPoint::Expr, "fn foo() {}", "fn");
-    check_prefix(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()");
-}
-
-#[test]
-fn path() {
-    check_prefix(PrefixEntryPoint::Path, "foo::bar baz", "foo::bar");
-    check_prefix(PrefixEntryPoint::Path, "foo::<> baz", "foo::<>");
-    check_prefix(PrefixEntryPoint::Path, "foo<> baz", "foo<>");
-    check_prefix(PrefixEntryPoint::Path, "Fn() -> i32?", "Fn() -> i32");
-    // FIXME: This shouldn't be accepted as path actually.
-    check_prefix(PrefixEntryPoint::Path, "<_>::foo", "<_>::foo");
-}
-
-#[test]
-fn item() {
-    // FIXME: This shouldn't consume the semicolon.
-    check_prefix(PrefixEntryPoint::Item, "fn foo() {};", "fn foo() {};");
-    check_prefix(PrefixEntryPoint::Item, "#[attr] pub struct S {} 92", "#[attr] pub struct S {}");
-    check_prefix(PrefixEntryPoint::Item, "item!{}?", "item!{}");
-    check_prefix(PrefixEntryPoint::Item, "????", "?");
-}
-
-#[test]
-fn meta_item() {
-    check_prefix(PrefixEntryPoint::MetaItem, "attr, ", "attr");
-    check_prefix(
-        PrefixEntryPoint::MetaItem,
-        "attr(some token {stream});",
-        "attr(some token {stream})",
-    );
-    check_prefix(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2");
-}
-
-fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) {
-    let lexed = LexedStr::new(input);
-    let input = lexed.to_input();
-
-    let mut n_tokens = 0;
-    for step in entry.parse(&input).iter() {
-        match step {
-            Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
-            Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
-        }
-    }
-
-    let mut i = 0;
-    loop {
-        if n_tokens == 0 {
-            break;
-        }
-        if !lexed.kind(i).is_trivia() {
-            n_tokens -= 1;
-        }
-        i += 1;
-    }
-    let buf = &lexed.as_str()[..lexed.text_start(i)];
-    assert_eq!(buf, prefix);
-}

crates/parser/src/tests/prefix_entries.rs

@@ -0,0 +1,107 @@
+use crate::{LexedStr, PrefixEntryPoint, Step};
+
+#[test]
+fn vis() {
+    check(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)");
+    check(PrefixEntryPoint::Vis, "fn foo() {}", "");
+    check(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub");
+    check(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate");
+    check(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate");
+}
+
+#[test]
+fn block() {
+    check(PrefixEntryPoint::Block, "{}, 92", "{}");
+    check(PrefixEntryPoint::Block, "{, 92)", "{, 92)");
+    check(PrefixEntryPoint::Block, "()", "");
+}
+
+#[test]
+fn stmt() {
+    check(PrefixEntryPoint::Stmt, "92; fn", "92");
+    check(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92");
+    check(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}");
+    check(PrefixEntryPoint::Stmt, "struct S;;", "struct S;");
+    check(PrefixEntryPoint::Stmt, "fn f() {};", "fn f() {}");
+    check(PrefixEntryPoint::Stmt, ";;;", ";");
+    check(PrefixEntryPoint::Stmt, "+", "+");
+    check(PrefixEntryPoint::Stmt, "@", "@");
+    check(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}");
+}
+
+#[test]
+fn pat() {
+    check(PrefixEntryPoint::Pat, "x y", "x");
+    check(PrefixEntryPoint::Pat, "fn f() {}", "fn");
+    // FIXME: This one is wrong, we should consume only one pattern.
+    check(PrefixEntryPoint::Pat, ".. ..", ".. ..");
+}
+
+#[test]
+fn ty() {
+    check(PrefixEntryPoint::Ty, "fn() foo", "fn()");
+    check(PrefixEntryPoint::Ty, "Clone + Copy + fn", "Clone + Copy +");
+    check(PrefixEntryPoint::Ty, "struct f", "struct");
+}
+
+#[test]
+fn expr() {
+    check(PrefixEntryPoint::Expr, "92 92", "92");
+    check(PrefixEntryPoint::Expr, "+1", "+");
+    check(PrefixEntryPoint::Expr, "-1", "-1");
+    check(PrefixEntryPoint::Expr, "fn foo() {}", "fn");
+    check(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()");
+}
+
+#[test]
+fn path() {
+    check(PrefixEntryPoint::Path, "foo::bar baz", "foo::bar");
+    check(PrefixEntryPoint::Path, "foo::<> baz", "foo::<>");
+    check(PrefixEntryPoint::Path, "foo<> baz", "foo<>");
+    check(PrefixEntryPoint::Path, "Fn() -> i32?", "Fn() -> i32");
+    // FIXME: This shouldn't be accepted as path actually.
+    check(PrefixEntryPoint::Path, "<_>::foo", "<_>::foo");
+}
+
+#[test]
+fn item() {
+    // FIXME: This shouldn't consume the semicolon.
+    check(PrefixEntryPoint::Item, "fn foo() {};", "fn foo() {};");
+    check(PrefixEntryPoint::Item, "#[attr] pub struct S {} 92", "#[attr] pub struct S {}");
+    check(PrefixEntryPoint::Item, "item!{}?", "item!{}");
+    check(PrefixEntryPoint::Item, "????", "?");
+}
+
+#[test]
+fn meta_item() {
+    check(PrefixEntryPoint::MetaItem, "attr, ", "attr");
+    check(PrefixEntryPoint::MetaItem, "attr(some token {stream});", "attr(some token {stream})");
+    check(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2");
+}
+
+#[track_caller]
+fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) {
+    let lexed = LexedStr::new(input);
+    let input = lexed.to_input();
+
+    let mut n_tokens = 0;
+    for step in entry.parse(&input).iter() {
+        match step {
+            Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
+            Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
+        }
+    }
+
+    let mut i = 0;
+    loop {
+        if n_tokens == 0 {
+            break;
+        }
+        if !lexed.kind(i).is_trivia() {
+            n_tokens -= 1;
+        }
+        i += 1;
+    }
+    let buf = &lexed.as_str()[..lexed.text_start(i)];
+    assert_eq!(buf, prefix);
+}

crates/parser/src/tests/top_entries.rs

@@ -0,0 +1,99 @@
+use expect_test::expect;
+
+use crate::TopEntryPoint;
+
+#[test]
+fn source_file() {
+    check(
+        TopEntryPoint::SourceFile,
+        "",
+        expect![[r#"
+        SOURCE_FILE
+    "#]],
+    );
+
+    check(
+        TopEntryPoint::SourceFile,
+        "struct S;",
+        expect![[r#"
+        SOURCE_FILE
+          STRUCT
+            STRUCT_KW "struct"
+            WHITESPACE " "
+            NAME
+              IDENT "S"
+            SEMICOLON ";"
+    "#]],
+    );
+
+    check(
+        TopEntryPoint::SourceFile,
+        "@error@",
+        expect![[r#"
+        SOURCE_FILE
+          ERROR
+            AT "@"
+          MACRO_CALL
+            PATH
+              PATH_SEGMENT
+                NAME_REF
+                  IDENT "error"
+          ERROR
+            AT "@"
+        error 0: expected an item
+        error 6: expected BANG
+        error 6: expected `{`, `[`, `(`
+        error 6: expected SEMICOLON
+        error 6: expected an item
+    "#]],
+    );
+}
+
+#[test]
+fn macro_stmt() {
+    check(
+        TopEntryPoint::MacroStmts,
+        "#!/usr/bin/rust",
+        expect![[r##"
+            MACRO_STMTS
+              ERROR
+                SHEBANG "#!/usr/bin/rust"
+            error 0: expected expression
+        "##]],
+    );
+    check(
+        TopEntryPoint::MacroStmts,
+        "let x = 1 2 struct S;",
+        expect![[r#"
+            MACRO_STMTS
+              LET_STMT
+                LET_KW "let"
+                WHITESPACE " "
+                IDENT_PAT
+                  NAME
+                    IDENT "x"
+                WHITESPACE " "
+                EQ "="
+                WHITESPACE " "
+                LITERAL
+                  INT_NUMBER "1"
+              WHITESPACE " "
+              EXPR_STMT
+                LITERAL
+                  INT_NUMBER "2"
+              WHITESPACE " "
+              STRUCT
+                STRUCT_KW "struct"
+                WHITESPACE " "
+                NAME
+                  IDENT "S"
+                SEMICOLON ";"
+        "#]],
+    );
+}
+
+#[track_caller]
+fn check(entry: TopEntryPoint, input: &str, expect: expect_test::Expect) {
+    let (parsed, _errors) = super::parse(entry, input);
+    expect.assert_eq(&parsed)
+}