mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rollup merge of #66369 - tmiasko:compiletest-stamp, r=Mark-Simulacrum

Browse Source 
compiletest: Obtain timestamps for common inputs only once

Obtain timestamps for common inputs (e.g., libraries in run-lib path, or
sources in `src/tool/compiletest/`) only once and reuse the result,
instead of repeating the work for each test case.
This commit is contained in:
Yuki Okushi 2019-11-14 14:16:26 +09:00 committed by GitHub
commit 8bd84653af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 91 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
src/tools/compiletest/src/main.rs
View File

@ -574,22 +574,59 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> { pub fn make_tests(config: &Config) -> Vec<test::TestDescAndFn> {
debug!("making tests from {:?}", config.src_base.display()); debug!("making tests from {:?}", config.src_base.display());
let inputs = common_inputs_stamp(config);
let mut tests = Vec::new(); let mut tests = Vec::new();
collect_tests_from_dir( collect_tests_from_dir(
config, config,
&config.src_base, &config.src_base,
&config.src_base, &config.src_base,
&PathBuf::new(), &PathBuf::new(),
&inputs,
&mut tests, &mut tests,
).expect(&format!("Could not read tests from {}", config.src_base.display())); ).expect(&format!("Could not read tests from {}", config.src_base.display()));
tests tests
} }
/// Returns a stamp constructed from input files common to all test cases.
fn common_inputs_stamp(config: &Config) -> Stamp {
let rust_src_dir = config
.find_rust_src_root()
.expect("Could not find Rust source root");
let mut stamp = Stamp::from_path(&config.rustc_path);
// Relevant pretty printer files
let pretty_printer_files = [
"src/etc/debugger_pretty_printers_common.py",
"src/etc/gdb_load_rust_pretty_printers.py",
"src/etc/gdb_rust_pretty_printing.py",
"src/etc/lldb_batchmode.py",
"src/etc/lldb_rust_formatters.py",
];
for file in &pretty_printer_files {
let path = rust_src_dir.join(file);
stamp.add_path(&path);
}
stamp.add_dir(&config.run_lib_path);
if let Some(ref rustdoc_path) = config.rustdoc_path {
stamp.add_path(&rustdoc_path);
stamp.add_path(&rust_src_dir.join("src/etc/htmldocck.py"));
}
// Compiletest itself.
stamp.add_dir(&rust_src_dir.join("src/tools/compiletest/"));
stamp
}
fn collect_tests_from_dir( fn collect_tests_from_dir(
config: &Config, config: &Config,
base: &Path, base: &Path,
dir: &Path, dir: &Path,
relative_dir_path: &Path, relative_dir_path: &Path,
inputs: &Stamp,
tests: &mut Vec<test::TestDescAndFn>, tests: &mut Vec<test::TestDescAndFn>,
) -> io::Result<()> { ) -> io::Result<()> {
// Ignore directories that contain a file named `compiletest-ignore-dir`. // Ignore directories that contain a file named `compiletest-ignore-dir`.
@ -602,7 +639,7 @@ fn collect_tests_from_dir(
file: dir.to_path_buf(), file: dir.to_path_buf(),
relative_dir: relative_dir_path.parent().unwrap().to_path_buf(), relative_dir: relative_dir_path.parent().unwrap().to_path_buf(),
}; };
tests.extend(make_test(config, &paths)); tests.extend(make_test(config, &paths, inputs));
return Ok(()); return Ok(());
} }
@ -627,12 +664,14 @@ fn collect_tests_from_dir(
file: file_path, file: file_path,
relative_dir: relative_dir_path.to_path_buf(), relative_dir: relative_dir_path.to_path_buf(),
}; };
tests.extend(make_test(config, &paths)) tests.extend(make_test(config, &paths, inputs))
} else if file_path.is_dir() { } else if file_path.is_dir() {
let relative_file_path = relative_dir_path.join(file.file_name()); let relative_file_path = relative_dir_path.join(file.file_name());
if &file_name != "auxiliary" { if &file_name != "auxiliary" {
debug!("found directory: {:?}", file_path.display()); debug!("found directory: {:?}", file_path.display());
collect_tests_from_dir(config, base, &file_path, &relative_file_path, tests)?; collect_tests_from_dir(
config, base, &file_path, &relative_file_path,
inputs, tests)?;
} }
} else { } else {
debug!("found other file/directory: {:?}", file_path.display()); debug!("found other file/directory: {:?}", file_path.display());
@ -655,7 +694,7 @@ pub fn is_test(file_name: &OsString) -> bool {
!invalid_prefixes.iter().any(|p| file_name.starts_with(p)) !invalid_prefixes.iter().any(|p| file_name.starts_with(p))
} }
pub fn make_test(config: &Config, testpaths: &TestPaths) -> Vec<test::TestDescAndFn> { fn make_test(config: &Config, testpaths: &TestPaths, inputs: &Stamp) -> Vec<test::TestDescAndFn> {
let early_props = if config.mode == Mode::RunMake { let early_props = if config.mode == Mode::RunMake {
// Allow `ignore` directives to be in the Makefile. // Allow `ignore` directives to be in the Makefile.
EarlyProps::from_file(config, &testpaths.file.join("Makefile")) EarlyProps::from_file(config, &testpaths.file.join("Makefile"))
@ -685,19 +724,21 @@ pub fn make_test(config: &Config, testpaths: &TestPaths) -> Vec<test::TestDescAn
revisions revisions
.into_iter() .into_iter()
.map(|revision| { .map(|revision| {
// Debugging emscripten code doesn't make sense today
let ignore = early_props.ignore == Ignore::Ignore let ignore = early_props.ignore == Ignore::Ignore
|| !up_to_date( // Debugging emscripten code doesn't make sense today
config,
testpaths,
&early_props,
revision.map(|s| s.as_str()),
)
|| ((config.mode == DebugInfoGdbLldb || config.mode == DebugInfoCdb || || ((config.mode == DebugInfoGdbLldb || config.mode == DebugInfoCdb ||
config.mode == DebugInfoGdb || config.mode == DebugInfoLldb) config.mode == DebugInfoGdb || config.mode == DebugInfoLldb)
&& config.target.contains("emscripten")) && config.target.contains("emscripten"))
|| (config.mode == DebugInfoGdb && !early_props.ignore.can_run_gdb()) || (config.mode == DebugInfoGdb && !early_props.ignore.can_run_gdb())
|| (config.mode == DebugInfoLldb && !early_props.ignore.can_run_lldb()); || (config.mode == DebugInfoLldb && !early_props.ignore.can_run_lldb())
// Ignore tests that already run and are up to date with respect to inputs.
|| is_up_to_date(
config,
testpaths,
&early_props,
revision.map(|s| s.as_str()),
inputs,
);
test::TestDescAndFn { test::TestDescAndFn {
desc: test::TestDesc { desc: test::TestDesc {
name: make_test_name(config, testpaths, revision), name: make_test_name(config, testpaths, revision),
@ -716,98 +757,75 @@ fn stamp(config: &Config, testpaths: &TestPaths, revision: Option<&str>) -> Path
output_base_dir(config, testpaths, revision).join("stamp") output_base_dir(config, testpaths, revision).join("stamp")
} }
fn up_to_date( fn is_up_to_date(
config: &Config, config: &Config,
testpaths: &TestPaths, testpaths: &TestPaths,
props: &EarlyProps, props: &EarlyProps,
revision: Option<&str>, revision: Option<&str>,
inputs: &Stamp,
) -> bool { ) -> bool {
let stamp_name = stamp(config, testpaths, revision); let stamp_name = stamp(config, testpaths, revision);
// Check hash. // Check hash.
let contents = match fs::read_to_string(&stamp_name) { let contents = match fs::read_to_string(&stamp_name) {
Ok(f) => f, Ok(f) => f,
Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"), Err(ref e) if e.kind() == ErrorKind::InvalidData => panic!("Can't read stamp contents"),
Err(_) => return true, Err(_) => return false,
}; };
let expected_hash = runtest::compute_stamp_hash(config); let expected_hash = runtest::compute_stamp_hash(config);
if contents != expected_hash { if contents != expected_hash {
return true; return false;
} }
// Check timestamps. // Check timestamps.
let rust_src_dir = config let mut inputs = inputs.clone();
.find_rust_src_root() inputs.add_path(&testpaths.file);
.expect("Could not find Rust source root");
let stamp = Stamp::from_path(&stamp_name); for aux in &props.aux {
let mut inputs = vec![Stamp::from_path(&testpaths.file), Stamp::from_path(&config.rustc_path)]; let path = testpaths.file.parent()
inputs.extend( .unwrap()
props .join("auxiliary")
.aux .join(aux);
.iter() inputs.add_path(&path);
.map(|aux| {
Stamp::from_path(&testpaths.file.parent().unwrap().join("auxiliary").join(aux))
}),
);
// Relevant pretty printer files
let pretty_printer_files = [
"src/etc/debugger_pretty_printers_common.py",
"src/etc/gdb_load_rust_pretty_printers.py",
"src/etc/gdb_rust_pretty_printing.py",
"src/etc/lldb_batchmode.py",
"src/etc/lldb_rust_formatters.py",
];
inputs.extend(pretty_printer_files.iter().map(|pretty_printer_file| {
Stamp::from_path(&rust_src_dir.join(pretty_printer_file))
}));
inputs.extend(Stamp::from_dir(&config.run_lib_path));
if let Some(ref rustdoc_path) = config.rustdoc_path {
inputs.push(Stamp::from_path(&rustdoc_path));
inputs.push(Stamp::from_path(&rust_src_dir.join("src/etc/htmldocck.py")));
} }
// UI test files. // UI test files.
inputs.extend(UI_EXTENSIONS.iter().map(|extension| { for extension in UI_EXTENSIONS {
let path = &expected_output_path(testpaths, revision, &config.compare_mode, extension); let path = &expected_output_path(testpaths, revision, &config.compare_mode, extension);
Stamp::from_path(path) inputs.add_path(path);
}));
// Compiletest itself.
inputs.extend(Stamp::from_dir(&rust_src_dir.join("src/tools/compiletest/")));
inputs.iter().any(|input| input > &stamp)
} }
#[derive(Debug, PartialEq, PartialOrd, Ord, Eq)] inputs < Stamp::from_path(&stamp_name)
}
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
struct Stamp { struct Stamp {
time: SystemTime, time: SystemTime,
file: PathBuf,
} }
impl Stamp { impl Stamp {
fn from_path(p: &Path) -> Self { fn from_path(path: &Path) -> Self {
let time = fs::metadata(p) let mut stamp = Stamp { time: SystemTime::UNIX_EPOCH };
stamp.add_path(path);
stamp
}
fn add_path(&mut self, path: &Path) {
let modified = fs::metadata(path)
.and_then(|metadata| metadata.modified()) .and_then(|metadata| metadata.modified())
.unwrap_or(SystemTime::UNIX_EPOCH); .unwrap_or(SystemTime::UNIX_EPOCH);
self.time = self.time.max(modified);
Stamp {
time,
file: p.into(),
}
} }
fn from_dir(path: &Path) -> impl Iterator<Item = Stamp> { fn add_dir(&mut self, path: &Path) {
WalkDir::new(path) for entry in WalkDir::new(path) {
.into_iter() let entry = entry.unwrap();
.map(|entry| entry.unwrap()) if entry.file_type().is_file() {
.filter(|entry| entry.file_type().is_file()) let modified = entry.metadata().ok()
.map(|entry| { .and_then(|metadata| metadata.modified().ok())
let time = (|| -> io::Result<_> { entry.metadata()?.modified() })(); .unwrap_or(SystemTime::UNIX_EPOCH);
self.time = self.time.max(modified);
Stamp { }
time: time.unwrap_or(SystemTime::UNIX_EPOCH),
file: entry.path().into(),
} }
})
} }
} }