From e366b3c730c343130f7935400e00bba96e77c9da Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 2 Jan 2022 15:01:20 +0300 Subject: [PATCH 1/4] minor: generalize --- crates/parser/src/tests.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs index fb4885e98d5..157c381f4b2 100644 --- a/crates/parser/src/tests.rs +++ b/crates/parser/src/tests.rs @@ -9,7 +9,7 @@ use std::{ use expect_test::expect_file; -use crate::LexedStr; +use crate::{LexedStr, TopEntryPoint}; #[test] fn lex_ok() { @@ -45,7 +45,7 @@ fn lex(text: &str) -> String { #[test] fn parse_ok() { for case in TestCase::list("parser/ok") { - let (actual, errors) = parse(&case.text); + let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual); expect_file![case.txt].assert_eq(&actual); } @@ -54,7 +54,7 @@ fn parse_ok() { #[test] fn parse_inline_ok() { for case in TestCase::list("parser/inline/ok") { - let (actual, errors) = parse(&case.text); + let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); assert!(!errors, "errors in an OK file {}:\n{}", case.rs.display(), actual); expect_file![case.txt].assert_eq(&actual); } @@ -63,7 +63,7 @@ fn parse_inline_ok() { #[test] fn parse_err() { for case in TestCase::list("parser/err") { - let (actual, errors) = parse(&case.text); + let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual); expect_file![case.txt].assert_eq(&actual) } @@ -72,16 +72,16 @@ fn parse_err() { #[test] fn parse_inline_err() { for case in TestCase::list("parser/inline/err") { - let (actual, errors) = parse(&case.text); + let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); assert!(errors, "no errors in an ERR file {}:\n{}", case.rs.display(), actual); expect_file![case.txt].assert_eq(&actual) } } -fn parse(text: &str) -> (String, bool) { +fn parse(entry: TopEntryPoint, text: &str) -> (String, bool) { let lexed = LexedStr::new(text); let input = lexed.to_input(); - let output = crate::TopEntryPoint::SourceFile.parse(&input); + let output = entry.parse(&input); let mut buf = String::new(); let mut errors = Vec::new(); From f2ea7853ee9b2b44464b93a6008ce8e1e39e2127 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 2 Jan 2022 15:15:04 +0300 Subject: [PATCH 2/4] start top-level entry point tests --- crates/parser/src/tests.rs | 14 +++++++- crates/parser/src/tests/entries.rs | 58 +++++++++++++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs index 157c381f4b2..73bf472f386 100644 --- a/crates/parser/src/tests.rs +++ b/crates/parser/src/tests.rs @@ -86,22 +86,34 @@ fn parse(entry: TopEntryPoint, text: &str) -> (String, bool) { let mut buf = String::new(); let mut errors = Vec::new(); let mut indent = String::new(); + let mut depth = 0; + let mut len = 0; lexed.intersperse_trivia(&output, &mut |step| match step { crate::StrStep::Token { kind, text } => { + assert!(depth > 0); + len += text.len(); write!(buf, "{}", indent).unwrap(); write!(buf, "{:?} {:?}\n", kind, text).unwrap(); } crate::StrStep::Enter { kind } => { + assert!(depth > 0 || len == 0); + depth += 1; write!(buf, "{}", indent).unwrap(); write!(buf, "{:?}\n", kind).unwrap(); indent.push_str(" "); } crate::StrStep::Exit => { + assert!(depth > 0); + depth -= 1; indent.pop(); indent.pop(); } - crate::StrStep::Error { msg, pos } => errors.push(format!("error {}: {}\n", pos, msg)), + crate::StrStep::Error { msg, pos } => { + assert!(depth > 0); + errors.push(format!("error {}: {}\n", pos, msg)) + } }); + assert_eq!(len, text.len()); for (token, msg) in lexed.errors() { let pos = lexed.text_start(token); diff --git a/crates/parser/src/tests/entries.rs b/crates/parser/src/tests/entries.rs index d52c6fbb16f..ea8ff415029 100644 --- a/crates/parser/src/tests/entries.rs +++ b/crates/parser/src/tests/entries.rs @@ -1,4 +1,6 @@ -use crate::{LexedStr, PrefixEntryPoint, Step}; +use expect_test::expect; + +use crate::{LexedStr, PrefixEntryPoint, Step, TopEntryPoint}; #[test] fn vis() { @@ -83,6 +85,7 @@ fn meta_item() { check_prefix(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2"); } +#[track_caller] fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) { let lexed = LexedStr::new(input); let input = lexed.to_input(); @@ -108,3 +111,56 @@ fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) { let buf = &lexed.as_str()[..lexed.text_start(i)]; assert_eq!(buf, prefix); } + +#[test] +fn source_file() { + check_top( + TopEntryPoint::SourceFile, + "", + expect![[r#" + SOURCE_FILE + "#]], + ); + + check_top( + TopEntryPoint::SourceFile, + "struct S;", + expect![[r#" + SOURCE_FILE + STRUCT + STRUCT_KW "struct" + WHITESPACE " " + NAME + IDENT "S" + SEMICOLON ";" + "#]], + ); + + check_top( + TopEntryPoint::SourceFile, + "@error@", + expect![[r#" + SOURCE_FILE + ERROR + AT "@" + MACRO_CALL + PATH + PATH_SEGMENT + NAME_REF + IDENT "error" + ERROR + AT "@" + error 0: expected an item + error 6: expected BANG + error 6: expected `{`, `[`, `(` + error 6: expected SEMICOLON + error 6: expected an item + "#]], + ); +} + +#[track_caller] +fn check_top(entry: TopEntryPoint, input: &str, expect: expect_test::Expect) { + let (parsed, _errors) = super::parse(entry, input); + expect.assert_eq(&parsed) +} From 2bd7c747066fc5d1e8537b81a1156ff4e5e40ae6 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 2 Jan 2022 15:17:32 +0300 Subject: [PATCH 3/4] add tests for macro statements --- crates/parser/src/tests/entries.rs | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/crates/parser/src/tests/entries.rs b/crates/parser/src/tests/entries.rs index ea8ff415029..c52f15418f3 100644 --- a/crates/parser/src/tests/entries.rs +++ b/crates/parser/src/tests/entries.rs @@ -159,6 +159,49 @@ fn source_file() { ); } +#[test] +fn macro_stmt() { + check_top( + TopEntryPoint::MacroStmts, + "#!/usr/bin/rust", + expect![[r##" + MACRO_STMTS + ERROR + SHEBANG "#!/usr/bin/rust" + error 0: expected expression + "##]], + ); + check_top( + TopEntryPoint::MacroStmts, + "let x = 1 2 struct S;", + expect![[r#" + MACRO_STMTS + LET_STMT + LET_KW "let" + WHITESPACE " " + IDENT_PAT + NAME + IDENT "x" + WHITESPACE " " + EQ "=" + WHITESPACE " " + LITERAL + INT_NUMBER "1" + WHITESPACE " " + EXPR_STMT + LITERAL + INT_NUMBER "2" + WHITESPACE " " + STRUCT + STRUCT_KW "struct" + WHITESPACE " " + NAME + IDENT "S" + SEMICOLON ";" + "#]], + ); +} + #[track_caller] fn check_top(entry: TopEntryPoint, input: &str, expect: expect_test::Expect) { let (parsed, _errors) = super::parse(entry, input); From bebfb83fd0504ec7c1d246375e39bcf41e0dfcc0 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sun, 2 Jan 2022 15:18:34 +0300 Subject: [PATCH 4/4] split prefix/top tests --- crates/parser/src/tests.rs | 3 +- crates/parser/src/tests/entries.rs | 209 ---------------------- crates/parser/src/tests/prefix_entries.rs | 107 +++++++++++ crates/parser/src/tests/top_entries.rs | 99 ++++++++++ 4 files changed, 208 insertions(+), 210 deletions(-) delete mode 100644 crates/parser/src/tests/entries.rs create mode 100644 crates/parser/src/tests/prefix_entries.rs create mode 100644 crates/parser/src/tests/top_entries.rs diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs index 73bf472f386..52388dacc61 100644 --- a/crates/parser/src/tests.rs +++ b/crates/parser/src/tests.rs @@ -1,5 +1,6 @@ mod sourcegen_inline_tests; -mod entries; +mod prefix_entries; +mod top_entries; use std::{ fmt::Write, diff --git a/crates/parser/src/tests/entries.rs b/crates/parser/src/tests/entries.rs deleted file mode 100644 index c52f15418f3..00000000000 --- a/crates/parser/src/tests/entries.rs +++ /dev/null @@ -1,209 +0,0 @@ -use expect_test::expect; - -use crate::{LexedStr, PrefixEntryPoint, Step, TopEntryPoint}; - -#[test] -fn vis() { - check_prefix(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)"); - check_prefix(PrefixEntryPoint::Vis, "fn foo() {}", ""); - check_prefix(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub"); - check_prefix(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate"); - check_prefix(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate"); -} - -#[test] -fn block() { - check_prefix(PrefixEntryPoint::Block, "{}, 92", "{}"); - check_prefix(PrefixEntryPoint::Block, "{, 92)", "{, 92)"); - check_prefix(PrefixEntryPoint::Block, "()", ""); -} - -#[test] -fn stmt() { - check_prefix(PrefixEntryPoint::Stmt, "92; fn", "92"); - check_prefix(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92"); - check_prefix(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}"); - check_prefix(PrefixEntryPoint::Stmt, "struct S;;", "struct S;"); - check_prefix(PrefixEntryPoint::Stmt, "fn f() {};", "fn f() {}"); - check_prefix(PrefixEntryPoint::Stmt, ";;;", ";"); - check_prefix(PrefixEntryPoint::Stmt, "+", "+"); - check_prefix(PrefixEntryPoint::Stmt, "@", "@"); - check_prefix(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}"); -} - -#[test] -fn pat() { - check_prefix(PrefixEntryPoint::Pat, "x y", "x"); - check_prefix(PrefixEntryPoint::Pat, "fn f() {}", "fn"); - // FIXME: This one is wrong, we should consume only one pattern. - check_prefix(PrefixEntryPoint::Pat, ".. ..", ".. .."); -} - -#[test] -fn ty() { - check_prefix(PrefixEntryPoint::Ty, "fn() foo", "fn()"); - check_prefix(PrefixEntryPoint::Ty, "Clone + Copy + fn", "Clone + Copy +"); - check_prefix(PrefixEntryPoint::Ty, "struct f", "struct"); -} - -#[test] -fn expr() { - check_prefix(PrefixEntryPoint::Expr, "92 92", "92"); - check_prefix(PrefixEntryPoint::Expr, "+1", "+"); - check_prefix(PrefixEntryPoint::Expr, "-1", "-1"); - check_prefix(PrefixEntryPoint::Expr, "fn foo() {}", "fn"); - check_prefix(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()"); -} - -#[test] -fn path() { - check_prefix(PrefixEntryPoint::Path, "foo::bar baz", "foo::bar"); - check_prefix(PrefixEntryPoint::Path, "foo::<> baz", "foo::<>"); - check_prefix(PrefixEntryPoint::Path, "foo<> baz", "foo<>"); - check_prefix(PrefixEntryPoint::Path, "Fn() -> i32?", "Fn() -> i32"); - // FIXME: This shouldn't be accepted as path actually. - check_prefix(PrefixEntryPoint::Path, "<_>::foo", "<_>::foo"); -} - -#[test] -fn item() { - // FIXME: This shouldn't consume the semicolon. - check_prefix(PrefixEntryPoint::Item, "fn foo() {};", "fn foo() {};"); - check_prefix(PrefixEntryPoint::Item, "#[attr] pub struct S {} 92", "#[attr] pub struct S {}"); - check_prefix(PrefixEntryPoint::Item, "item!{}?", "item!{}"); - check_prefix(PrefixEntryPoint::Item, "????", "?"); -} - -#[test] -fn meta_item() { - check_prefix(PrefixEntryPoint::MetaItem, "attr, ", "attr"); - check_prefix( - PrefixEntryPoint::MetaItem, - "attr(some token {stream});", - "attr(some token {stream})", - ); - check_prefix(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2"); -} - -#[track_caller] -fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) { - let lexed = LexedStr::new(input); - let input = lexed.to_input(); - - let mut n_tokens = 0; - for step in entry.parse(&input).iter() { - match step { - Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize, - Step::Enter { .. } | Step::Exit | Step::Error { .. } => (), - } - } - - let mut i = 0; - loop { - if n_tokens == 0 { - break; - } - if !lexed.kind(i).is_trivia() { - n_tokens -= 1; - } - i += 1; - } - let buf = &lexed.as_str()[..lexed.text_start(i)]; - assert_eq!(buf, prefix); -} - -#[test] -fn source_file() { - check_top( - TopEntryPoint::SourceFile, - "", - expect![[r#" - SOURCE_FILE - "#]], - ); - - check_top( - TopEntryPoint::SourceFile, - "struct S;", - expect![[r#" - SOURCE_FILE - STRUCT - STRUCT_KW "struct" - WHITESPACE " " - NAME - IDENT "S" - SEMICOLON ";" - "#]], - ); - - check_top( - TopEntryPoint::SourceFile, - "@error@", - expect![[r#" - SOURCE_FILE - ERROR - AT "@" - MACRO_CALL - PATH - PATH_SEGMENT - NAME_REF - IDENT "error" - ERROR - AT "@" - error 0: expected an item - error 6: expected BANG - error 6: expected `{`, `[`, `(` - error 6: expected SEMICOLON - error 6: expected an item - "#]], - ); -} - -#[test] -fn macro_stmt() { - check_top( - TopEntryPoint::MacroStmts, - "#!/usr/bin/rust", - expect![[r##" - MACRO_STMTS - ERROR - SHEBANG "#!/usr/bin/rust" - error 0: expected expression - "##]], - ); - check_top( - TopEntryPoint::MacroStmts, - "let x = 1 2 struct S;", - expect![[r#" - MACRO_STMTS - LET_STMT - LET_KW "let" - WHITESPACE " " - IDENT_PAT - NAME - IDENT "x" - WHITESPACE " " - EQ "=" - WHITESPACE " " - LITERAL - INT_NUMBER "1" - WHITESPACE " " - EXPR_STMT - LITERAL - INT_NUMBER "2" - WHITESPACE " " - STRUCT - STRUCT_KW "struct" - WHITESPACE " " - NAME - IDENT "S" - SEMICOLON ";" - "#]], - ); -} - -#[track_caller] -fn check_top(entry: TopEntryPoint, input: &str, expect: expect_test::Expect) { - let (parsed, _errors) = super::parse(entry, input); - expect.assert_eq(&parsed) -} diff --git a/crates/parser/src/tests/prefix_entries.rs b/crates/parser/src/tests/prefix_entries.rs new file mode 100644 index 00000000000..e626b4f27e0 --- /dev/null +++ b/crates/parser/src/tests/prefix_entries.rs @@ -0,0 +1,107 @@ +use crate::{LexedStr, PrefixEntryPoint, Step}; + +#[test] +fn vis() { + check(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)"); + check(PrefixEntryPoint::Vis, "fn foo() {}", ""); + check(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub"); + check(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate"); + check(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate"); +} + +#[test] +fn block() { + check(PrefixEntryPoint::Block, "{}, 92", "{}"); + check(PrefixEntryPoint::Block, "{, 92)", "{, 92)"); + check(PrefixEntryPoint::Block, "()", ""); +} + +#[test] +fn stmt() { + check(PrefixEntryPoint::Stmt, "92; fn", "92"); + check(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92"); + check(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}"); + check(PrefixEntryPoint::Stmt, "struct S;;", "struct S;"); + check(PrefixEntryPoint::Stmt, "fn f() {};", "fn f() {}"); + check(PrefixEntryPoint::Stmt, ";;;", ";"); + check(PrefixEntryPoint::Stmt, "+", "+"); + check(PrefixEntryPoint::Stmt, "@", "@"); + check(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}"); +} + +#[test] +fn pat() { + check(PrefixEntryPoint::Pat, "x y", "x"); + check(PrefixEntryPoint::Pat, "fn f() {}", "fn"); + // FIXME: This one is wrong, we should consume only one pattern. + check(PrefixEntryPoint::Pat, ".. ..", ".. .."); +} + +#[test] +fn ty() { + check(PrefixEntryPoint::Ty, "fn() foo", "fn()"); + check(PrefixEntryPoint::Ty, "Clone + Copy + fn", "Clone + Copy +"); + check(PrefixEntryPoint::Ty, "struct f", "struct"); +} + +#[test] +fn expr() { + check(PrefixEntryPoint::Expr, "92 92", "92"); + check(PrefixEntryPoint::Expr, "+1", "+"); + check(PrefixEntryPoint::Expr, "-1", "-1"); + check(PrefixEntryPoint::Expr, "fn foo() {}", "fn"); + check(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()"); +} + +#[test] +fn path() { + check(PrefixEntryPoint::Path, "foo::bar baz", "foo::bar"); + check(PrefixEntryPoint::Path, "foo::<> baz", "foo::<>"); + check(PrefixEntryPoint::Path, "foo<> baz", "foo<>"); + check(PrefixEntryPoint::Path, "Fn() -> i32?", "Fn() -> i32"); + // FIXME: This shouldn't be accepted as path actually. + check(PrefixEntryPoint::Path, "<_>::foo", "<_>::foo"); +} + +#[test] +fn item() { + // FIXME: This shouldn't consume the semicolon. + check(PrefixEntryPoint::Item, "fn foo() {};", "fn foo() {};"); + check(PrefixEntryPoint::Item, "#[attr] pub struct S {} 92", "#[attr] pub struct S {}"); + check(PrefixEntryPoint::Item, "item!{}?", "item!{}"); + check(PrefixEntryPoint::Item, "????", "?"); +} + +#[test] +fn meta_item() { + check(PrefixEntryPoint::MetaItem, "attr, ", "attr"); + check(PrefixEntryPoint::MetaItem, "attr(some token {stream});", "attr(some token {stream})"); + check(PrefixEntryPoint::MetaItem, "path::attr = 2 * 2!", "path::attr = 2 * 2"); +} + +#[track_caller] +fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) { + let lexed = LexedStr::new(input); + let input = lexed.to_input(); + + let mut n_tokens = 0; + for step in entry.parse(&input).iter() { + match step { + Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize, + Step::Enter { .. } | Step::Exit | Step::Error { .. } => (), + } + } + + let mut i = 0; + loop { + if n_tokens == 0 { + break; + } + if !lexed.kind(i).is_trivia() { + n_tokens -= 1; + } + i += 1; + } + let buf = &lexed.as_str()[..lexed.text_start(i)]; + assert_eq!(buf, prefix); +} diff --git a/crates/parser/src/tests/top_entries.rs b/crates/parser/src/tests/top_entries.rs new file mode 100644 index 00000000000..0b77c0a5b96 --- /dev/null +++ b/crates/parser/src/tests/top_entries.rs @@ -0,0 +1,99 @@ +use expect_test::expect; + +use crate::TopEntryPoint; + +#[test] +fn source_file() { + check( + TopEntryPoint::SourceFile, + "", + expect![[r#" + SOURCE_FILE + "#]], + ); + + check( + TopEntryPoint::SourceFile, + "struct S;", + expect![[r#" + SOURCE_FILE + STRUCT + STRUCT_KW "struct" + WHITESPACE " " + NAME + IDENT "S" + SEMICOLON ";" + "#]], + ); + + check( + TopEntryPoint::SourceFile, + "@error@", + expect![[r#" + SOURCE_FILE + ERROR + AT "@" + MACRO_CALL + PATH + PATH_SEGMENT + NAME_REF + IDENT "error" + ERROR + AT "@" + error 0: expected an item + error 6: expected BANG + error 6: expected `{`, `[`, `(` + error 6: expected SEMICOLON + error 6: expected an item + "#]], + ); +} + +#[test] +fn macro_stmt() { + check( + TopEntryPoint::MacroStmts, + "#!/usr/bin/rust", + expect![[r##" + MACRO_STMTS + ERROR + SHEBANG "#!/usr/bin/rust" + error 0: expected expression + "##]], + ); + check( + TopEntryPoint::MacroStmts, + "let x = 1 2 struct S;", + expect![[r#" + MACRO_STMTS + LET_STMT + LET_KW "let" + WHITESPACE " " + IDENT_PAT + NAME + IDENT "x" + WHITESPACE " " + EQ "=" + WHITESPACE " " + LITERAL + INT_NUMBER "1" + WHITESPACE " " + EXPR_STMT + LITERAL + INT_NUMBER "2" + WHITESPACE " " + STRUCT + STRUCT_KW "struct" + WHITESPACE " " + NAME + IDENT "S" + SEMICOLON ";" + "#]], + ); +} + +#[track_caller] +fn check(entry: TopEntryPoint, input: &str, expect: expect_test::Expect) { + let (parsed, _errors) = super::parse(entry, input); + expect.assert_eq(&parsed) +}