diff --git a/Cargo.lock b/Cargo.lock index da19714fe69..1bbae2cbd80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1008,6 +1008,16 @@ dependencies = [ "yaml-rust 0.4.4", ] +[[package]] +name = "expect-test" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e383741ea1982866572109d1a8c807bd36aad91fca701489fdca56ef92b3b8" +dependencies = [ + "difference", + "once_cell", +] + [[package]] name = "failure" version = "0.1.8" @@ -3653,6 +3663,7 @@ dependencies = [ name = "rustc_lexer" version = "0.1.0" dependencies = [ + "expect-test", "unicode-xid", ] diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index ac833a55d4c..afa72b5d58c 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1754,6 +1754,11 @@ fn run(self, builder: &Builder<'_>) { cargo.arg("--quiet"); } + if builder.config.cmd.bless() { + // Bless `expect!` tests. + cargo.env("UPDATE_EXPECT", "1"); + } + if target.contains("emscripten") { cargo.env( format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), diff --git a/src/librustc_lexer/Cargo.toml b/src/librustc_lexer/Cargo.toml index 950771f0a69..28b56f6fef4 100644 --- a/src/librustc_lexer/Cargo.toml +++ b/src/librustc_lexer/Cargo.toml @@ -19,3 +19,6 @@ name = "rustc_lexer" # Note that this crate purposefully does not depend on other rustc crates [dependencies] unicode-xid = "0.2.0" + +[dev-dependencies] +expect-test = "0.1" diff --git a/src/librustc_lexer/src/lib.rs b/src/librustc_lexer/src/lib.rs index 2d80ca5a4de..b7d6194cd77 100644 --- a/src/librustc_lexer/src/lib.rs +++ b/src/librustc_lexer/src/lib.rs @@ -35,6 +35,7 @@ /// Parsed token. /// It doesn't contain information about data that has been parsed, /// only the type of the token and its size. +#[derive(Debug)] pub struct Token { pub kind: TokenKind, pub len: usize, diff --git a/src/librustc_lexer/src/tests.rs b/src/librustc_lexer/src/tests.rs index b0209ac2899..a1ea5ceb1f6 100644 --- a/src/librustc_lexer/src/tests.rs +++ b/src/librustc_lexer/src/tests.rs @@ -1,5 +1,7 @@ use super::*; +use expect_test::{expect, Expect}; + fn check_raw_str(s: &str, expected_hashes: u16, expected_err: Option) { let s = &format!("r{}", s); let mut cursor = Cursor::new(s); @@ -120,3 +122,46 @@ fn test_shebang_followed_by_attrib() { let input = "#!/bin/rust-scripts\n#![allow_unused(true)]"; assert_eq!(strip_shebang(input), Some(19)); } + +fn check_lexing(src: &str, expect: Expect) { + let actual: String = tokenize(src).map(|token| format!("{:?}\n", token)).collect(); + expect.assert_eq(&actual) +} + +#[test] +fn comment_flavors() { + check_lexing( + r" +// line +//// line as well +/// outer doc line +//! inner doc line +/* block */ +/**/ +/*** also block */ +/** outer doc block */ +/*! inner doc block */ +", + expect![[r#" + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 7 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: None }, len: 17 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Outer) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: LineComment { doc_style: Some(Inner) }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 11 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 4 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: None, terminated: true }, len: 18 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Outer), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + Token { kind: BlockComment { doc_style: Some(Inner), terminated: true }, len: 22 } + Token { kind: Whitespace, len: 1 } + "#]], + ) +} diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4f98944e4c8..af3fb403703 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -85,11 +85,13 @@ "crossbeam-queue", "crossbeam-utils", "datafrog", + "difference", "digest", "dlmalloc", "either", "ena", "env_logger", + "expect-test", "fake-simd", "filetime", "flate2",