From 4db40579baa1d69116e3ce7863951e36d1ddf425 Mon Sep 17 00:00:00 2001 From: Martin Nordholts Date: Sat, 2 Dec 2023 19:15:55 +0100 Subject: [PATCH 1/2] libtest: Add regression tests for padding As you can see the padding is wrong when running benches as tests. This will be fixed in the next commit. (Benches should only be padded when run as benches to make it easy to compare the benchmark numbers.) --- tests/run-make/libtest-padding/Makefile | 14 ++++++++++++++ tests/run-make/libtest-padding/bench.stdout | 9 +++++++++ tests/run-make/libtest-padding/test.stdout | 9 +++++++++ tests/run-make/libtest-padding/tests.rs | 18 ++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 tests/run-make/libtest-padding/Makefile create mode 100644 tests/run-make/libtest-padding/bench.stdout create mode 100644 tests/run-make/libtest-padding/test.stdout create mode 100644 tests/run-make/libtest-padding/tests.rs diff --git a/tests/run-make/libtest-padding/Makefile b/tests/run-make/libtest-padding/Makefile new file mode 100644 index 00000000000..063d0dc0a6c --- /dev/null +++ b/tests/run-make/libtest-padding/Makefile @@ -0,0 +1,14 @@ +# ignore-cross-compile because we run the compiled code +# needs-unwind because #[bench] and -Cpanic=abort requires -Zpanic-abort-tests +include ../tools.mk + +NORMALIZE=sed 's%[0-9,]\+ ns/iter (+/- [0-9,]\+)%?? ns/iter (+/- ??)%' | sed 's%finished in [0-9\.]\+%finished in ??%' + +all: + $(RUSTC) --test tests.rs + + $(call RUN,tests) --test-threads=1 | $(NORMALIZE) > "$(TMPDIR)"/test.stdout + $(RUSTC_TEST_OP) "$(TMPDIR)"/test.stdout test.stdout + + $(call RUN,tests) --test-threads=1 --bench | $(NORMALIZE) > "$(TMPDIR)"/bench.stdout + $(RUSTC_TEST_OP) "$(TMPDIR)"/bench.stdout bench.stdout diff --git a/tests/run-make/libtest-padding/bench.stdout b/tests/run-make/libtest-padding/bench.stdout new file mode 100644 index 00000000000..3d55401c93a --- /dev/null +++ b/tests/run-make/libtest-padding/bench.stdout @@ -0,0 +1,9 @@ + +running 4 tests +test short_test_name ... ignored +test this_is_a_really_long_test_name ... ignored +test short_bench_name ... bench: ?? ns/iter (+/- ??) +test this_is_a_really_long_bench_name ... bench: ?? ns/iter (+/- ??) + +test result: ok. 0 passed; 0 failed; 2 ignored; 2 measured; 0 filtered out; finished in ??s + diff --git a/tests/run-make/libtest-padding/test.stdout b/tests/run-make/libtest-padding/test.stdout new file mode 100644 index 00000000000..c655f034756 --- /dev/null +++ b/tests/run-make/libtest-padding/test.stdout @@ -0,0 +1,9 @@ + +running 4 tests +test short_bench_name ... ok +test short_test_name ... ok +test this_is_a_really_long_bench_name ... ok +test this_is_a_really_long_test_name ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in ??s + diff --git a/tests/run-make/libtest-padding/tests.rs b/tests/run-make/libtest-padding/tests.rs new file mode 100644 index 00000000000..cadf07237e9 --- /dev/null +++ b/tests/run-make/libtest-padding/tests.rs @@ -0,0 +1,18 @@ +#![feature(test)] +extern crate test; + +#[test] +fn short_test_name() {} + +#[test] +fn this_is_a_really_long_test_name() {} + +#[bench] +fn short_bench_name(b: &mut test::Bencher) { + b.iter(|| 1); +} + +#[bench] +fn this_is_a_really_long_bench_name(b: &mut test::Bencher) { + b.iter(|| 1); +} From 12e6bcfcabee7bfa9130bb5fcd4b8ab23172b1ee Mon Sep 17 00:00:00 2001 From: Martin Nordholts Date: Sat, 2 Dec 2023 18:24:37 +0100 Subject: [PATCH 2/2] libtest: Fix padding of benchmarks run as tests Before this fix we applied padding before manually doing what `convert_benchmarks_to_tests()` does. Instead use `convert_benchmarks_to_tests()` if applicable and then apply padding afterwards so it becomes correct. (Benches should only be padded when run as benches to make it easy to compare the benchmark numbers.) --- library/test/src/lib.rs | 22 ++++++++-------------- tests/run-make/libtest-padding/Makefile | 2 +- tests/run-make/libtest-padding/test.stdout | 2 +- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/library/test/src/lib.rs b/library/test/src/lib.rs index 2fa5a8e5e38..2e93796d981 100644 --- a/library/test/src/lib.rs +++ b/library/test/src/lib.rs @@ -298,24 +298,18 @@ struct TimeoutEntry { let mut filtered = FilteredTests { tests: Vec::new(), benches: Vec::new(), next_id: 0 }; - for test in filter_tests(opts, tests) { + let mut filtered_tests = filter_tests(opts, tests); + if !opts.bench_benchmarks { + filtered_tests = convert_benchmarks_to_tests(filtered_tests); + } + + for test in filtered_tests { let mut desc = test.desc; desc.name = desc.name.with_padding(test.testfn.padding()); match test.testfn { - DynBenchFn(benchfn) => { - if opts.bench_benchmarks { - filtered.add_bench(desc, DynBenchFn(benchfn)); - } else { - filtered.add_test(desc, DynBenchAsTestFn(benchfn)); - } - } - StaticBenchFn(benchfn) => { - if opts.bench_benchmarks { - filtered.add_bench(desc, StaticBenchFn(benchfn)); - } else { - filtered.add_test(desc, StaticBenchAsTestFn(benchfn)); - } + DynBenchFn(_) | StaticBenchFn(_) => { + filtered.add_bench(desc, test.testfn); } testfn => { filtered.add_test(desc, testfn); diff --git a/tests/run-make/libtest-padding/Makefile b/tests/run-make/libtest-padding/Makefile index 063d0dc0a6c..42bc1192925 100644 --- a/tests/run-make/libtest-padding/Makefile +++ b/tests/run-make/libtest-padding/Makefile @@ -2,7 +2,7 @@ # needs-unwind because #[bench] and -Cpanic=abort requires -Zpanic-abort-tests include ../tools.mk -NORMALIZE=sed 's%[0-9,]\+ ns/iter (+/- [0-9,]\+)%?? ns/iter (+/- ??)%' | sed 's%finished in [0-9\.]\+%finished in ??%' +NORMALIZE=sed 's%[0-9,]\{1,\} ns/iter (+/- [0-9,]\{1,\})%?? ns/iter (+/- ??)%' | sed 's%finished in [0-9\.]\{1,\}%finished in ??%' all: $(RUSTC) --test tests.rs diff --git a/tests/run-make/libtest-padding/test.stdout b/tests/run-make/libtest-padding/test.stdout index c655f034756..8ffc810f6cd 100644 --- a/tests/run-make/libtest-padding/test.stdout +++ b/tests/run-make/libtest-padding/test.stdout @@ -1,6 +1,6 @@ running 4 tests -test short_bench_name ... ok +test short_bench_name ... ok test short_test_name ... ok test this_is_a_really_long_bench_name ... ok test this_is_a_really_long_test_name ... ok