Infra for "unit" benchmarking

Cargo.lock

@@ -1640,6 +1640,7 @@ name = "test_utils"
 version = "0.0.0"
 dependencies = [
  "dissimilar",
+ "profile",
  "rustc-hash",
  "serde_json",
  "stdx",

crates/hir_ty/src/tests/macros.rs

@@ -1,7 +1,5 @@
-use std::fs;
-
 use expect_test::expect;
-use test_utils::project_dir;
+use test_utils::{bench, bench_fixture, skip_slow_tests};
 
 use super::{check_infer, check_types};
 
@@ -617,12 +615,11 @@ fn main() {
 }
 
 #[test]
-#[ignore]
+fn benchmark_include_macro() {
-fn include_accidentally_quadratic() {
+    if skip_slow_tests() {
-    let file = project_dir().join("crates/syntax/test_data/accidentally_quadratic");
+        return;
-    let big_file = fs::read_to_string(file).unwrap();
+    }
-    let big_file = vec![big_file; 10].join("\n");
+    let data = bench_fixture::big_struct();
-
     let fixture = r#"
 //- /main.rs
 #[rustc_builtin_macro]
@@ -635,8 +632,12 @@ fn main() {
                   //^ RegisterBlock
 }
     "#;
-    let fixture = format!("{}\n//- /foo.rs\n{}", fixture, big_file);
+    let fixture = format!("{}\n//- /foo.rs\n{}", fixture, data);
-    check_types(&fixture);
+
+    {
+        let _b = bench("include macro");
+        check_types(&fixture);
+    }
 }
 
 #[test]

crates/ide/src/syntax_highlighting/tests.rs

@@ -1,7 +1,5 @@
-use std::fs;
-
 use expect_test::{expect_file, ExpectFile};
-use test_utils::project_dir;
+use test_utils::{bench, bench_fixture, skip_slow_tests};
 
 use crate::{fixture, FileRange, TextRange};
 
@@ -228,15 +226,19 @@ fn bar() {
 }
 
 #[test]
-fn accidentally_quadratic() {
+fn benchmark_syntax_highlighting() {
-    let file = project_dir().join("crates/syntax/test_data/accidentally_quadratic");
+    if skip_slow_tests() {
-    let src = fs::read_to_string(file).unwrap();
+        return;
+    }
 
-    let (analysis, file_id) = fixture::file(&src);
+    let fixture = bench_fixture::big_struct();
+    let (analysis, file_id) = fixture::file(&fixture);
 
-    // let t = std::time::Instant::now();
+    let hash = {
-    let _ = analysis.highlight(file_id).unwrap();
+        let _pt = bench("syntax highlighting");
-    // eprintln!("elapsed: {:?}", t.elapsed());
+        analysis.highlight(file_id).unwrap().len()
+    };
+    assert_eq!(hash, 32009);
 }
 
 #[test]

crates/test_utils/Cargo.toml

@@ -17,3 +17,4 @@ serde_json = "1.0.48"
 rustc-hash = "1.1.0"
 
 stdx = { path = "../stdx", version = "0.0.0" }
+profile = { path = "../profile", version = "0.0.0" }

crates/test_utils/src/bench_fixture.rs

@@ -0,0 +1,28 @@
+//! Generates large snippets of Rust code for usage in the benchmarks.
+
+use stdx::format_to;
+
+pub fn big_struct() -> String {
+    let n = 1_000;
+
+    let mut buf = "pub struct RegisterBlock {".to_string();
+    for i in 0..n {
+        format_to!(buf, "  /// Doc comment for {}.\n", i);
+        format_to!(buf, "  pub s{}: S{},\n", i, i);
+    }
+    buf.push_str("}\n\n");
+    for i in 0..n {
+        format_to!(
+            buf,
+            "
+
+#[repr(transparent)]
+struct S{} {{
+    field: u32,
+}}",
+            i
+        );
+    }
+
+    buf
+}

crates/test_utils/src/lib.rs

@@ -8,6 +8,7 @@
 
 #[macro_use]
 pub mod mark;
+pub mod bench_fixture;
 mod fixture;
 
 use std::{
@@ -16,6 +17,7 @@
     path::PathBuf,
 };
 
+use profile::StopWatch;
 use serde_json::Value;
 use stdx::lines_with_ends;
 use text_size::{TextRange, TextSize};
@@ -406,3 +408,44 @@ pub fn format_diff(chunks: Vec<dissimilar::Chunk>) -> String {
     }
     buf
 }
+
+/// Utility for writing benchmark tests.
+///
+/// A benchmark test looks like this:
+///
+/// ```
+/// #[test]
+/// fn benchmark_foo() {
+///     if skip_slow_tests() { return; }
+///
+///     let data = bench_fixture::some_fixture();
+///     let analysis = some_setup();
+///
+///     let hash = {
+///         let _b = bench("foo");
+///         actual_work(analysis)
+///     };
+///     assert_eq!(hash, 92);
+/// }
+/// ```
+///
+/// * We skip benchmarks by default, to save time.
+///   Ideal benchmark time is 800 -- 1500 ms in debug.
+/// * We don't count preparation as part of the benchmark
+/// * The benchmark itself returns some kind of numeric hash.
+///   The hash is used as a sanity check that some code is actually run.
+///   Otherwise, it's too easy to win the benchmark by just doing nothing.
+pub fn bench(label: &'static str) -> impl Drop {
+    struct Bencher {
+        sw: StopWatch,
+        label: &'static str,
+    }
+
+    impl Drop for Bencher {
+        fn drop(&mut self) {
+            eprintln!("{}: {}", self.label, self.sw.elapsed())
+        }
+    }
+
+    Bencher { sw: StopWatch::start(), label }
+}

docs/dev/architecture.md

@@ -397,6 +397,11 @@ There's no additional checks in CI, formatting and tidy tests are run with `carg
 
 **Architecture Invariant:** tests do not depend on any kind of external resources, they are perfectly reproducible.
 
+
+### Performance Testing
+
+TBA, take a look at the `metrics` xtask and `#[test] fn benchmark_xxx()` functions.
+
 ### Error Handling
 
 **Architecture Invariant:** core parts of rust-analyzer (`ide`/`hir`) don't interact with the outside world and thus can't fail.