From 267aaef81dade6f1c1ae09fe820a31ff6de95e38 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 21:26:14 +0100 Subject: [PATCH 1/4] Move some top-level folders into `build` --- build_system/src/clean.rs | 19 ++++------ build_system/src/config.rs | 12 ++++-- build_system/src/main.rs | 2 + build_system/src/prepare.rs | 7 ++-- build_system/src/test.rs | 75 +++++++++++++++++++++---------------- 5 files changed, 62 insertions(+), 53 deletions(-) diff --git a/build_system/src/clean.rs b/build_system/src/clean.rs index 929a878113d..cd8e691a0ed 100644 --- a/build_system/src/clean.rs +++ b/build_system/src/clean.rs @@ -1,6 +1,7 @@ use crate::utils::{remove_file, run_command}; use std::fs::remove_dir_all; +use std::path::Path; #[derive(Default)] enum CleanArg { @@ -46,12 +47,14 @@ fn clean_all() -> Result<(), String> { "build_sysroot/sysroot", "build_sysroot/sysroot_src", "build_sysroot/target", - "regex", - "simple-raytracer", ]; for dir in dirs_to_remove { let _ = remove_dir_all(dir); } + let dirs_to_remove = ["regex", "rand", "simple-raytracer"]; + for dir in dirs_to_remove { + let _ = remove_dir_all(Path::new(crate::BUILD_DIR).join(dir)); + } let files_to_remove = ["build_sysroot/Cargo.lock", "perf.data", "perf.data.old"]; @@ -64,16 +67,8 @@ fn clean_all() -> Result<(), String> { } fn clean_ui_tests() -> Result<(), String> { - run_command( - &[ - &"find", - &"rust/build/x86_64-unknown-linux-gnu/test/ui/", - &"-name", - &"stamp", - &"-delete", - ], - None, - )?; + let path = Path::new(crate::BUILD_DIR).join("rust/build/x86_64-unknown-linux-gnu/test/ui/"); + run_command(&[&"find", &path, &"-name", &"stamp", &"-delete"], None)?; Ok(()) } diff --git a/build_system/src/config.rs b/build_system/src/config.rs index c9bfcb9e6ba..d1047436ceb 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -523,7 +523,11 @@ fn download_gccjit( &"--retry", &"3", &"-SRfL", - if with_progress_bar { &"--progress-bar" } else { &"-s" }, + if with_progress_bar { + &"--progress-bar" + } else { + &"-s" + }, &url.as_str(), ], Some(&output_dir), @@ -538,9 +542,9 @@ fn download_gccjit( &"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", &format!( "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, - tempfile_name, - ).as_str(), + url, tempfile_name, + ) + .as_str(), ], Some(&output_dir), ); diff --git a/build_system/src/main.rs b/build_system/src/main.rs index c6958f0c512..18dc4b21a96 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -11,6 +11,8 @@ mod test; mod utils; +const BUILD_DIR: &str = "build"; + macro_rules! arg_error { ($($err:tt)*) => {{ eprintln!($($err)*); diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index 7f1401e594c..979438d0415 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -152,11 +152,11 @@ fn clone_and_setup(repo_url: &str, checkout_commit: &str, extra: Option) - where F: Fn(&Path) -> Result<(), String>, { - let clone_result = git_clone(repo_url, None, false)?; + let clone_result = git_clone(repo_url, Some(&Path::new(crate::BUILD_DIR)), false)?; if !clone_result.ran_clone { println!("`{}` has already been cloned", clone_result.repo_name); } - let repo_path = Path::new(&clone_result.repo_name); + let repo_path = Path::new(crate::BUILD_DIR).join(&clone_result.repo_name); run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?; run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?; let filter = format!("-{}-", clone_result.repo_name); @@ -219,8 +219,7 @@ fn usage() { --only-libcore : Only setup libcore and don't clone other repositories --cross : Apply the patches needed to do cross-compilation --libgccjit12-patches : Apply patches needed for libgccjit12 - --help : Show this help -"# + --help : Show this help"# ) } } diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 806e18431c4..d7f7a0eb47e 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -485,19 +485,25 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { Ok(()) } -fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { +fn setup_rustc(env: &mut Env, args: &TestArg) -> Result { let toolchain = format!( "+{channel}-{host}", channel = get_toolchain()?, // May also include date host = args.config_info.host_triple ); - let rust_dir = Some(Path::new("rust")); + let rust_dir_path = Path::new(crate::BUILD_DIR).join("rust"); // If the repository was already cloned, command will fail, so doesn't matter. let _ = run_command_with_output_and_env( - &[&"git", &"clone", &"https://github.com/rust-lang/rust.git"], + &[ + &"git", + &"clone", + &"https://github.com/rust-lang/rust.git", + &rust_dir_path, + ], None, Some(env), ); + let rust_dir: Option<&Path> = Some(&rust_dir_path); run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?; run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { @@ -561,8 +567,9 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { String::new() } }; + let file_path = rust_dir_path.join("config.toml"); std::fs::write( - "rust/config.toml", + &file_path, &format!( r#"change-id = 115898 @@ -587,13 +594,19 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { llvm_filecheck = llvm_filecheck.trim(), ), ) - .map_err(|error| format!("Failed to write into `rust/config.toml`: {:?}", error))?; - Ok(()) + .map_err(|error| { + format!( + "Failed to write into `{}`: {:?}", + file_path.display(), + error + ) + })?; + Ok(rust_dir_path) } fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { let mut env = env.clone(); - setup_rustc(&mut env, args)?; + let rust_dir = setup_rustc(&mut env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rustc asm test suite"); @@ -621,7 +634,7 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { ) .as_str(), ], - Some(Path::new("rust")), + Some(&rust_dir), Some(&env), )?; Ok(()) @@ -761,11 +774,11 @@ fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { println!("Not using GCC master branch. Skipping `extended_rand_tests`."); return Ok(()); } - let path = Path::new("rand"); - run_cargo_command(&[&"clean"], Some(path), env, args)?; + let path = Path::new(crate::BUILD_DIR).join("rand"); + run_cargo_command(&[&"clean"], Some(&path), env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rust-random/rand"); - run_cargo_command(&[&"test", &"--workspace"], Some(path), env, args)?; + run_cargo_command(&[&"test", &"--workspace"], Some(&path), env, args)?; Ok(()) } @@ -774,8 +787,8 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> println!("Not using GCC master branch. Skipping `extended_regex_example_tests`."); return Ok(()); } - let path = Path::new("regex"); - run_cargo_command(&[&"clean"], Some(path), env, args)?; + let path = Path::new(crate::BUILD_DIR).join("regex"); + run_cargo_command(&[&"clean"], Some(&path), env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rust-lang/regex example shootout-regex-dna"); let mut env = env.clone(); @@ -788,14 +801,14 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> // Make sure `[codegen mono items] start` doesn't poison the diff run_cargo_command( &[&"build", &"--example", &"shootout-regex-dna"], - Some(path), + Some(&path), &env, args, )?; run_cargo_command_with_callback( &[&"run", &"--example", &"shootout-regex-dna"], - Some(path), + Some(&path), &env, args, |cargo_command, cwd, env| { @@ -838,6 +851,7 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { env.get("RUSTFLAGS").cloned().unwrap_or_default() ); env.insert("RUSTFLAGS".to_string(), rustflags); + let path = Path::new(crate::BUILD_DIR).join("regex"); run_cargo_command( &[ &"test", @@ -850,7 +864,7 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"-Zunstable-options", &"-q", ], - Some(Path::new("regex")), + Some(&path), &env, args, )?; @@ -928,17 +942,15 @@ fn should_remove_test(file_path: &Path) -> Result { fn test_rustc_inner(env: &Env, args: &TestArg, prepare_files_callback: F) -> Result<(), String> where - F: Fn() -> Result, + F: Fn(&Path) -> Result, { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rust-lang/rust"); let mut env = env.clone(); - setup_rustc(&mut env, args)?; - - let rust_path = Path::new("rust"); + let rust_path = setup_rustc(&mut env, args)?; walk_dir( - "rust/tests/ui", + rust_path.join("tests/ui"), |dir| { let dir_name = dir.file_name().and_then(|name| name.to_str()).unwrap_or(""); if [ @@ -1001,7 +1013,7 @@ fn file_handling(file_path: &Path) -> Result<(), String> { walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; - if !prepare_files_callback()? { + if !prepare_files_callback(&rust_path)? { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("Keeping all UI tests"); } @@ -1027,7 +1039,7 @@ fn file_handling(file_path: &Path) -> Result<(), String> { &"-path", &"*/auxiliary/*", ], - Some(rust_path), + Some(&rust_path), )? .stdout, ) @@ -1072,18 +1084,18 @@ fn file_handling(file_path: &Path) -> Result<(), String> { &"--rustc-args", &rustc_args, ], - Some(rust_path), + Some(&rust_path), Some(&env), )?; Ok(()) } fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - test_rustc_inner(env, args, || Ok(false)) + test_rustc_inner(env, args, |_| Ok(false)) } fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - test_rustc_inner(env, args, || { + test_rustc_inner(env, args, |rust_path| { // Removing all tests. run_command( &[ @@ -1098,7 +1110,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { &"*/auxiliary/*", &"-delete", ], - Some(Path::new("rust")), + Some(rust_path), )?; // Putting back only the failing ones. let path = "tests/failing-ui-tests.txt"; @@ -1108,10 +1120,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { .map(|line| line.trim()) .filter(|line| !line.is_empty()) { - run_command( - &[&"git", &"checkout", &"--", &file], - Some(Path::new("rust")), - )?; + run_command(&[&"git", &"checkout", &"--", &file], Some(&rust_path))?; } } else { println!( @@ -1124,7 +1133,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { } fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { - test_rustc_inner(env, args, || { + test_rustc_inner(env, args, |rust_path| { // Removing the failing tests. let path = "tests/failing-ui-tests.txt"; if let Ok(files) = std::fs::read_to_string(path) { @@ -1133,7 +1142,7 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { .map(|line| line.trim()) .filter(|line| !line.is_empty()) { - let path = Path::new("rust").join(file); + let path = rust_path.join(file); remove_file(&path)?; } } else { From 896b1a9049631477e2549144a2a3a773c470213e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 21:50:00 +0100 Subject: [PATCH 2/4] Generate libgccjit.so into the `build` folder --- build_system/src/config.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index d1047436ceb..c89a6d5eb9b 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -191,12 +191,7 @@ pub fn rustc_command_vec(&self) -> Vec<&dyn AsRef> { } fn download_gccjit_if_needed(&mut self) -> Result<(), String> { - let output_dir = Path::new( - std::env::var("CARGO_TARGET_DIR") - .as_deref() - .unwrap_or("target"), - ) - .join("libgccjit"); + let output_dir = Path::new(crate::BUILD_DIR).join("libgccjit"); let commit_hash_file = self.compute_path("libgccjit.version"); let content = fs::read_to_string(&commit_hash_file).map_err(|_| { From 436fea8efbb332362a9c3b7f6854fea6bd35cd11 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 21:50:34 +0100 Subject: [PATCH 3/4] Add `build` folder into the ignored git entries --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 687c3a6797a..ac695da16f8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ tools/llvmint-2 llvm build_system/target config.toml +build \ No newline at end of file From 46d6e772c087c3bffb3228f36530010f16a57431 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 13 Feb 2024 21:56:17 +0100 Subject: [PATCH 4/4] Update `tests/lang_tests_common.rs` test --- tests/lang_tests_common.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lang_tests_common.rs b/tests/lang_tests_common.rs index 33dc6ef62ae..4cc429cfa45 100644 --- a/tests/lang_tests_common.rs +++ b/tests/lang_tests_common.rs @@ -28,7 +28,7 @@ pub fn main_inner(profile: Profile) { } else { // then we try to retrieve it from the `target` folder. let commit = include_str!("../libgccjit.version").trim(); - Path::new("target/libgccjit").join(commit) + Path::new("build/libgccjit").join(commit) }; let gcc_path = Path::new(&gcc_path)