mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2 from GuillaumeGomez/rm-build_sysroot-folder

Browse Source 
Remove `build_sysroot` folder
This commit is contained in:
antoyo 2024-03-22 16:37:06 -04:00 committed by GitHub
commit eea2f89c80
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 193 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/ci.yml vendored
View File

@ -103,7 +103,10 @@ jobs:
./y.sh test --release --clean --build-sysroot ${{ matrix.commands }}
- name: Check formatting
run: cargo fmt -- --check
run: |
cargo fmt -- --check
cd build_system
cargo fmt -- --check
- name: clippy
run: |

4
.github/workflows/release.yml vendored
View File

@ -62,12 +62,12 @@ jobs:
git config --global user.email "user@example.com"
git config --global user.name "User"
./y.sh prepare
# FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
echo -n 'lto = "fat"' >> build_sysroot/Cargo.toml
- name: Add more failing tests because of undefined symbol errors (FIXME)
run: cat tests/failing-lto-tests.txt >> tests/failing-ui-tests.txt
- name: Run tests
run: |
# FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
echo -n 'lto = "fat"' >> build_system/build_sysroot/Cargo.toml
EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}

6
.github/workflows/stdarch.yml vendored
View File

@ -89,12 +89,10 @@ jobs:
- name: Run stdarch tests
if: ${{ !matrix.cargo_runner }}
run: |
cd build_sysroot/sysroot_src/library/stdarch/
CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test
CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml
- name: Run stdarch tests
if: ${{ matrix.cargo_runner }}
run: |
cd build_sysroot/sysroot_src/library/stdarch/
# FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro.
STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../y.sh cargo test -- --skip rtm --skip tbm --skip sse4a
STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a

4
.gitignore vendored
View File

@ -6,10 +6,6 @@ perf.data
perf.data.old
*.events
*.string*
/build_sysroot/sysroot
/build_sysroot/sysroot_src
/build_sysroot/Cargo.lock
/build_sysroot/test_target/Cargo.lock
gimple*
*asm
res

2
build_sysroot/Cargo.toml → build_system/build_sysroot/Cargo.toml
View File

@ -1,5 +1,5 @@
[package]
authors = ["bjorn3 <bjorn3@users.noreply.github.com>"]
authors = ["rustc_codegen_gcc devs"]
name = "sysroot"
version = "0.0.0"
resolver = "2"

0
build_sysroot/src/lib.rs → build_system/build_sysroot/lib.rs
View File

79
build_system/src/build.rs
View File

@ -1,5 +1,7 @@
use crate::config::{Channel, ConfigInfo};
use crate::utils::{run_command, run_command_with_output_and_env, walk_dir};
use crate::utils::{
copy_file, create_dir, get_sysroot_dir, run_command, run_command_with_output_and_env, walk_dir,
};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs;
@ -55,8 +57,7 @@ fn usage() {
}
}
pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> {
let start_dir = Path::new("build_sysroot");
fn cleanup_sysroot_previous_build(start_dir: &Path) {
// Cleanup for previous run
// Clean target dir except for build scripts and incremental cache
let _ = walk_dir(
@ -100,6 +101,26 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
let _ = fs::remove_file(start_dir.join("Cargo.lock"));
let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock"));
let _ = fs::remove_dir_all(start_dir.join("sysroot"));
}
pub fn create_build_sysroot_content(start_dir: &Path) -> Result<(), String> {
if !start_dir.is_dir() {
create_dir(start_dir)?;
}
copy_file("build_system/build_sysroot/Cargo.toml", &start_dir.join("Cargo.toml"))?;
let src_dir = start_dir.join("src");
if !src_dir.is_dir() {
create_dir(&src_dir)?;
}
copy_file("build_system/build_sysroot/lib.rs", &start_dir.join("src/lib.rs"))
}
pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Result<(), String> {
let start_dir = get_sysroot_dir();
cleanup_sysroot_previous_build(&start_dir);
create_build_sysroot_content(&start_dir)?;
// Builds libs
let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default();
@ -110,7 +131,6 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
if config.no_default_features {
rustflags.push_str(" -Csymbol-mangling-version=v0");
}
let mut env = env.clone();
let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"build", &"--target", &config.target];
@ -127,18 +147,13 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
"debug"
};
let mut env = env.clone();
env.insert("RUSTFLAGS".to_string(), rustflags);
run_command_with_output_and_env(&args, Some(start_dir), Some(&env))?;
run_command_with_output_and_env(&args, Some(&start_dir), Some(&env))?;
// Copy files to sysroot
let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple));
fs::create_dir_all(&sysroot_path).map_err(|error| {
format!(
"Failed to create directory `{}`: {:?}",
sysroot_path.display(),
error
)
})?;
create_dir(&sysroot_path)?;
let copier = |dir_to_copy: &Path| {
// FIXME: should not use shell command!
run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
@ -151,22 +166,8 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
// Copy the source files to the sysroot (Rust for Linux needs this).
let sysroot_src_path = start_dir.join("sysroot/lib/rustlib/src/rust");
fs::create_dir_all(&sysroot_src_path).map_err(|error| {
format!(
"Failed to create directory `{}`: {:?}",
sysroot_src_path.display(),
error
)
})?;
run_command(
&[
&"cp",
&"-r",
&start_dir.join("sysroot_src/library/"),
&sysroot_src_path,
],
None,
)?;
create_dir(&sysroot_src_path)?;
run_command(&[&"cp", &"-r", &start_dir.join("sysroot_src/library/"), &sysroot_src_path], None)?;
Ok(())
}
@ -174,20 +175,11 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
let mut env = HashMap::new();
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert("LD_LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
env.insert("LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
if args.config_info.no_default_features {
env.insert(
"RUSTFLAGS".to_string(),
"-Csymbol-mangling-version=v0".to_string(),
);
env.insert("RUSTFLAGS".to_string(), "-Csymbol-mangling-version=v0".to_string());
}
let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
@ -212,12 +204,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> {
// We voluntarily ignore the error.
let _ = fs::remove_dir_all("target/out");
let gccjit_target = "target/out/gccjit";
fs::create_dir_all(gccjit_target).map_err(|error| {
format!(
"Failed to create directory `{}`: {:?}",
gccjit_target, error
)
})?;
create_dir(gccjit_target)?;
println!("[BUILD] sysroot");
build_sysroot(&env, &args.config_info)?;

27
build_system/src/cargo.rs
View File

@ -16,9 +16,7 @@ fn args() -> Result<Option<Vec<String>>, String> {
}
let args = std::env::args().skip(2).collect::<Vec<_>>();
if args.is_empty() {
return Err(
"Expected at least one argument for `cargo` subcommand, found none".to_string(),
);
return Err("Expected at least one argument for `cargo` subcommand, found none".to_string());
}
Ok(Some(args))
}
@ -48,17 +46,10 @@ pub fn run() -> Result<(), String> {
let current_exe = std::env::current_exe()
.and_then(|path| path.canonicalize())
.map_err(|error| format!("Failed to get current exe path: {:?}", error))?;
let mut parent_dir = current_exe
.components()
.map(|comp| comp.as_os_str())
.collect::<Vec<_>>();
let mut parent_dir = current_exe.components().map(|comp| comp.as_os_str()).collect::<Vec<_>>();
// We run this script from "build_system/target/release/y", so we need to remove these elements.
for to_remove in &["y", "release", "target", "build_system"] {
if parent_dir
.last()
.map(|part| part == to_remove)
.unwrap_or(false)
{
if parent_dir.last().map(|part| part == to_remove).unwrap_or(false) {
parent_dir.pop();
} else {
return Err(format!(
@ -69,11 +60,7 @@ pub fn run() -> Result<(), String> {
}
let parent_dir = PathBuf::from(parent_dir.join(&OsStr::new("/")));
std::env::set_current_dir(&parent_dir).map_err(|error| {
format!(
"Failed to go to `{}` folder: {:?}",
parent_dir.display(),
error
)
format!("Failed to go to `{}` folder: {:?}", parent_dir.display(), error)
})?;
let mut env: HashMap<String, String> = std::env::vars().collect();
@ -92,11 +79,7 @@ pub fn run() -> Result<(), String> {
// We go back to the original folder since we now have set up everything we needed.
std::env::set_current_dir(&current_dir).map_err(|error| {
format!(
"Failed to go back to `{}` folder: {:?}",
current_dir.display(),
error
)
format!("Failed to go back to `{}` folder: {:?}", current_dir.display(), error)
})?;
let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default();

16
build_system/src/clean.rs
View File

@ -1,4 +1,4 @@
use crate::utils::{remove_file, run_command};
use crate::utils::{get_sysroot_dir, remove_file, run_command};
use std::fs::remove_dir_all;
use std::path::Path;
@ -42,11 +42,12 @@ fn usage() {
}
fn clean_all() -> Result<(), String> {
let build_sysroot = get_sysroot_dir();
let dirs_to_remove = [
"target",
"build_sysroot/sysroot",
"build_sysroot/sysroot_src",
"build_sysroot/target",
"target".into(),
build_sysroot.join("sysroot"),
build_sysroot.join("sysroot_src"),
build_sysroot.join("target"),
];
for dir in dirs_to_remove {
let _ = remove_dir_all(dir);
@ -56,10 +57,11 @@ fn clean_all() -> Result<(), String> {
let _ = remove_dir_all(Path::new(crate::BUILD_DIR).join(dir));
}
let files_to_remove = ["build_sysroot/Cargo.lock", "perf.data", "perf.data.old"];
let files_to_remove =
[build_sysroot.join("Cargo.lock"), "perf.data".into(), "perf.data.old".into()];
for file in files_to_remove {
let _ = remove_file(file);
let _ = remove_file(&file);
}
println!("Successfully ran `clean all`");

74
build_system/src/config.rs
View File

@ -1,5 +1,6 @@
use crate::utils::{
create_symlink, get_os_name, run_command_with_output, rustc_version_info, split_args,
create_dir, create_symlink, get_os_name, get_sysroot_dir, run_command_with_output,
rustc_version_info, split_args,
};
use std::collections::HashMap;
use std::env as std_env;
@ -26,11 +27,7 @@ pub fn as_str(self) -> &'static str {
}
fn failed_config_parsing(config_file: &Path, err: &str) -> Result<ConfigFile, String> {
Err(format!(
"Failed to parse `{}`: {}",
config_file.display(),
err
))
Err(format!("Failed to parse `{}`: {}", config_file.display(), err))
}
#[derive(Default)]
@ -48,11 +45,7 @@ pub fn new(config_file: &Path) -> Result<Self, String> {
)
})?;
let toml = Toml::parse(&content).map_err(|err| {
format!(
"Error occurred around `{}`: {:?}",
&content[err.start..=err.end],
err.kind
)
format!("Error occurred around `{}`: {:?}", &content[err.start..=err.end], err.kind)
})?;
let mut config = Self::default();
for (key, value) in toml.iter() {
@ -181,11 +174,7 @@ pub fn parse_argument(
},
"--use-backend" => match args.next() {
Some(backend) if !backend.is_empty() => self.backend = Some(backend),
_ => {
return Err(
"Expected an argument after `--use-backend`, found nothing".into()
)
}
_ => return Err("Expected an argument after `--use-backend`, found nothing".into()),
},
"--no-default-features" => self.no_default_features = true,
_ => return Ok(false),
@ -228,20 +217,10 @@ fn download_gccjit_if_needed(&mut self) -> Result<(), String> {
let output_dir = output_dir.join(&commit);
if !output_dir.is_dir() {
std::fs::create_dir_all(&output_dir).map_err(|err| {
format!(
"failed to create folder `{}`: {:?}",
output_dir.display(),
err,
)
})?;
create_dir(&output_dir)?;
}
let output_dir = output_dir.canonicalize().map_err(|err| {
format!(
"Failed to get absolute path of `{}`: {:?}",
output_dir.display(),
err
)
format!("Failed to get absolute path of `{}`: {:?}", output_dir.display(), err)
})?;
let libgccjit_so_name = "libgccjit.so";
@ -275,10 +254,7 @@ fn download_gccjit_if_needed(&mut self) -> Result<(), String> {
println!("Downloaded libgccjit.so version {} successfully!", commit);
// We need to create a link named `libgccjit.so.0` because that's what the linker is
// looking for.
create_symlink(
&libgccjit_so,
output_dir.join(&format!("{}.0", libgccjit_so_name)),
)?;
create_symlink(&libgccjit_so, output_dir.join(&format!("{}.0", libgccjit_so_name)))?;
}
self.gcc_path = output_dir.display().to_string();
@ -298,10 +274,7 @@ pub fn setup_gcc_path(&mut self) -> Result<(), String> {
Some(config_file) => config_file.into(),
None => self.compute_path("config.toml"),
};
let ConfigFile {
gcc_path,
download_gccjit,
} = ConfigFile::new(&config_file)?;
let ConfigFile { gcc_path, download_gccjit } = ConfigFile::new(&config_file)?;
if let Some(true) = download_gccjit {
self.download_gccjit_if_needed()?;
@ -310,10 +283,7 @@ pub fn setup_gcc_path(&mut self) -> Result<(), String> {
self.gcc_path = match gcc_path {
Some(path) => path,
None => {
return Err(format!(
"missing `gcc-path` value from `{}`",
config_file.display(),
))
return Err(format!("missing `gcc-path` value from `{}`", config_file.display(),))
}
};
Ok(())
@ -393,17 +363,16 @@ pub fn setup(
.join(&format!("librustc_codegen_gcc.{}", self.dylib_ext))
.display()
.to_string();
self.sysroot_path = current_dir
.join("build_sysroot/sysroot")
.display()
.to_string();
self.sysroot_path =
current_dir.join(&get_sysroot_dir()).join("sysroot").display().to_string();
if let Some(backend) = &self.backend {
// This option is only used in the rust compiler testsuite. The sysroot is handled
// by its build system directly so no need to set it ourselves.
rustflags.push(format!("-Zcodegen-backend={}", backend));
} else {
rustflags.extend_from_slice(&[
"--sysroot".to_string(), self.sysroot_path.clone(),
"--sysroot".to_string(),
self.sysroot_path.clone(),
format!("-Zcodegen-backend={}", self.cg_backend_path),
]);
}
@ -424,8 +393,6 @@ pub fn setup(
rustflags.push("-Csymbol-mangling-version=v0".to_string());
}
// Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
// TODO(antoyo): remove when we can handle ThinLTO.
if !env.contains_key(&"FAT_LTO".to_string()) {
@ -442,10 +409,9 @@ pub fn setup(
// display metadata load errors
env.insert("RUSTC_LOG".to_string(), "warn".to_string());
let sysroot = current_dir.join(&format!(
"build_sysroot/sysroot/lib/rustlib/{}/lib",
self.target_triple,
));
let sysroot = current_dir
.join(&get_sysroot_dir())
.join(&format!("sysroot/lib/rustlib/{}/lib", self.target_triple));
let ld_library_path = format!(
"{target}:{sysroot}:{gcc_path}",
target = self.cargo_target_dir,
@ -523,11 +489,7 @@ fn download_gccjit(
&"--retry",
&"3",
&"-SRfL",
if with_progress_bar {
&"--progress-bar"
} else {
&"-s"
},
if with_progress_bar { &"--progress-bar" } else { &"-s" },
&url.as_str(),
],
Some(&output_dir),

80
build_system/src/prepare.rs
View File

@ -1,6 +1,7 @@
use crate::rustc_info::get_rustc_path;
use crate::utils::{
cargo_install, git_clone_root_dir, remove_file, run_command, run_command_with_output, walk_dir,
cargo_install, create_dir, get_sysroot_dir, git_clone_root_dir, remove_file, run_command,
run_command_with_output, walk_dir,
};
use std::fs;
@ -32,27 +33,14 @@ fn prepare_libcore(
let sysroot_dir = sysroot_path.join("sysroot_src");
if sysroot_dir.is_dir() {
if let Err(error) = fs::remove_dir_all(&sysroot_dir) {
return Err(format!(
"Failed to remove `{}`: {:?}",
sysroot_dir.display(),
error,
));
return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), error,));
}
}
let sysroot_library_dir = sysroot_dir.join("library");
fs::create_dir_all(&sysroot_library_dir).map_err(|error| {
format!(
"Failed to create folder `{}`: {:?}",
sysroot_library_dir.display(),
error,
)
})?;
create_dir(&sysroot_library_dir)?;
run_command(
&[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir],
None,
)?;
run_command(&[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], None)?;
println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
@ -63,26 +51,11 @@ fn prepare_libcore(
// This is needed on systems where nothing is configured.
// git really needs something here, or it will fail.
// Even using --author is not enough.
run_command(
&[&"git", &"config", &"user.email", &"none@example.com"],
Some(&sysroot_dir),
)?;
run_command(
&[&"git", &"config", &"user.name", &"None"],
Some(&sysroot_dir),
)?;
run_command(
&[&"git", &"config", &"core.autocrlf", &"false"],
Some(&sysroot_dir),
)?;
run_command(
&[&"git", &"config", &"commit.gpgSign", &"false"],
Some(&sysroot_dir),
)?;
run_command(
&[&"git", &"commit", &"-m", &"Initial commit", &"-q"],
Some(&sysroot_dir),
)?;
run_command(&[&"git", &"config", &"user.email", &"none@example.com"], Some(&sysroot_dir))?;
run_command(&[&"git", &"config", &"user.name", &"None"], Some(&sysroot_dir))?;
run_command(&[&"git", &"config", &"core.autocrlf", &"false"], Some(&sysroot_dir))?;
run_command(&[&"git", &"config", &"commit.gpgSign", &"false"], Some(&sysroot_dir))?;
run_command(&[&"git", &"commit", &"-m", &"Initial commit", &"-q"], Some(&sysroot_dir))?;
let mut patches = Vec::new();
walk_dir(
@ -116,17 +89,11 @@ fn prepare_libcore(
patches.sort();
for file_path in patches {
println!("[GIT] apply `{}`", file_path.display());
let path = Path::new("../..").join(file_path);
let path = Path::new("../../..").join(file_path);
run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
run_command_with_output(
&[
&"git",
&"commit",
&"--no-gpg-sign",
&"-m",
&format!("Patch {}", path.display()),
],
&[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())],
Some(&sysroot_dir),
)?;
}
@ -145,13 +112,7 @@ fn prepare_rand() -> Result<(), String> {
run_command_with_output(&[&"git", &"apply", &path], Some(rand_dir))?;
run_command_with_output(&[&"git", &"add", &"-A"], Some(rand_dir))?;
run_command_with_output(
&[
&"git",
&"commit",
&"--no-gpg-sign",
&"-m",
&format!("Patch {}", path.display()),
],
&[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())],
Some(rand_dir),
)?;
@ -165,10 +126,7 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
if mv_target.is_file() {
remove_file(&mv_target)?;
}
run_command(
&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"],
Some(repo_dir),
)?;
run_command(&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], Some(repo_dir))?;
Ok(())
}
@ -213,11 +171,7 @@ fn new() -> Result<Option<Self>, String> {
a => return Err(format!("Unknown argument `{a}`")),
}
}
Ok(Some(Self {
cross_compile,
only_libcore,
libgccjit12_patches,
}))
Ok(Some(Self { cross_compile, only_libcore, libgccjit12_patches }))
}
fn usage() {
@ -238,8 +192,8 @@ pub fn run() -> Result<(), String> {
Some(a) => a,
None => return Ok(()),
};
let sysroot_path = Path::new("build_sysroot");
prepare_libcore(sysroot_path, args.libgccjit12_patches, args.cross_compile)?;
let sysroot_path = get_sysroot_dir();
prepare_libcore(&sysroot_path, args.libgccjit12_patches, args.cross_compile)?;
if !args.only_libcore {
cargo_install("hyperfine")?;

249
build_system/src/test.rs
View File

@ -1,13 +1,14 @@
use crate::build;
use crate::config::{Channel, ConfigInfo};
use crate::utils::{
get_toolchain, git_clone, git_clone_root_dir, remove_file, run_command, run_command_with_env,
run_command_with_output_and_env, rustc_version_info, split_args, walk_dir,
create_dir, get_sysroot_dir, get_toolchain, git_clone, git_clone_root_dir, remove_file,
run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info,
split_args, walk_dir,
};
use std::collections::{BTreeSet, HashMap};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::fs::{create_dir_all, remove_dir_all, File};
use std::fs::{remove_dir_all, File};
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::str::FromStr;
@ -19,46 +20,23 @@
fn get_runners() -> Runners {
let mut runners = HashMap::new();
runners.insert(
"--test-rustc",
("Run all rustc tests", test_rustc as Runner),
);
runners.insert(
"--test-successful-rustc",
("Run successful rustc tests", test_successful_rustc),
);
runners.insert(
"--test-failing-rustc",
("Run failing rustc tests", test_failing_rustc),
);
runners.insert(
"--projects",
("Run the tests of popular crates", test_projects),
);
runners.insert("--test-rustc", ("Run all rustc tests", test_rustc as Runner));
runners
.insert("--test-successful-rustc", ("Run successful rustc tests", test_successful_rustc));
runners.insert("--test-failing-rustc", ("Run failing rustc tests", test_failing_rustc));
runners.insert("--projects", ("Run the tests of popular crates", test_projects));
runners.insert("--test-libcore", ("Run libcore tests", test_libcore));
runners.insert("--clean", ("Empty cargo target directory", clean));
runners.insert("--build-sysroot", ("Build sysroot", build_sysroot));
runners.insert("--std-tests", ("Run std tests", std_tests));
runners.insert("--asm-tests", ("Run asm tests", asm_tests));
runners.insert(
"--extended-tests",
("Run extended sysroot tests", extended_sysroot_tests),
);
runners.insert(
"--extended-rand-tests",
("Run extended rand tests", extended_rand_tests),
);
runners.insert("--extended-tests", ("Run extended sysroot tests", extended_sysroot_tests));
runners.insert("--extended-rand-tests", ("Run extended rand tests", extended_rand_tests));
runners.insert(
"--extended-regex-example-tests",
(
"Run extended regex example tests",
extended_regex_example_tests,
),
);
runners.insert(
"--extended-regex-tests",
("Run extended regex tests", extended_regex_tests),
("Run extended regex example tests", extended_regex_example_tests),
);
runners.insert("--extended-regex-tests", ("Run extended regex tests", extended_regex_tests));
runners.insert("--mini-tests", ("Run mini tests", mini_tests));
runners
@ -71,15 +49,9 @@ fn get_number_after_arg(
match args.next() {
Some(nb) if !nb.is_empty() => match usize::from_str(&nb) {
Ok(nb) => Ok(nb),
Err(_) => Err(format!(
"Expected a number after `{}`, found `{}`",
option, nb
)),
Err(_) => Err(format!("Expected a number after `{}`, found `{}`", option, nb)),
},
_ => Err(format!(
"Expected a number after `{}`, found nothing",
option
)),
_ => Err(format!("Expected a number after `{}`, found nothing", option)),
}
}
@ -110,7 +82,7 @@ fn show_usage() {
struct TestArg {
build_only: bool,
use_system_gcc: bool,
runners: BTreeSet<String>,
runners: Vec<String>,
flags: Vec<String>,
nb_parts: Option<usize>,
current_part: Option<usize>,
@ -130,9 +102,7 @@ fn new() -> Result<Option<Self>, String> {
match arg.as_str() {
"--features" => match args.next() {
Some(feature) if !feature.is_empty() => {
test_arg
.flags
.extend_from_slice(&["--features".into(), feature]);
test_arg.flags.extend_from_slice(&["--features".into(), feature]);
}
_ => {
return Err("Expected an argument after `--features`, found nothing".into())
@ -157,8 +127,10 @@ fn new() -> Result<Option<Self>, String> {
show_usage();
return Ok(None);
}
x if runners.contains_key(x) => {
test_arg.runners.insert(x.into());
x if runners.contains_key(x)
&& !test_arg.runners.iter().any(|runner| runner == x) =>
{
test_arg.runners.push(x.into());
}
arg => {
if !test_arg.config_info.parse_argument(arg, &mut args)? {
@ -211,8 +183,7 @@ fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> {
fn clean(_env: &Env, args: &TestArg) -> Result<(), String> {
let _ = std::fs::remove_dir_all(&args.config_info.cargo_target_dir);
let path = Path::new(&args.config_info.cargo_target_dir).join("gccjit");
std::fs::create_dir_all(&path)
.map_err(|error| format!("failed to create folder `{}`: {:?}", path.display(), error))
create_dir(&path)
}
fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> {
@ -304,13 +275,8 @@ fn maybe_run_command_in_vm(
let sudo_command: &[&dyn AsRef<OsStr>] = &[&"sudo", &"cp", &exe, &vm_exe_path];
run_command_with_env(sudo_command, None, Some(env))?;
let mut vm_command: Vec<&dyn AsRef<OsStr>> = vec![
&"sudo",
&"chroot",
&vm_dir,
&"qemu-m68k-static",
&inside_vm_exe_path,
];
let mut vm_command: Vec<&dyn AsRef<OsStr>> =
vec![&"sudo", &"chroot", &vm_dir, &"qemu-m68k-static", &inside_vm_exe_path];
vm_command.extend_from_slice(command);
run_command_with_output_and_env(&vm_command, Some(&vm_parent_dir), Some(env))?;
Ok(())
@ -399,11 +365,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> {
}
run_command_with_env(&command, None, Some(env))?;
maybe_run_command_in_vm(
&[
&cargo_target_dir.join("std_example"),
&"--target",
&args.config_info.target_triple,
],
&[&cargo_target_dir.join("std_example"), &"--target", &args.config_info.target_triple],
env,
args,
)?;
@ -427,11 +389,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> {
command.push(test_flag);
}
run_command_with_env(&command, None, Some(env))?;
maybe_run_command_in_vm(
&[&cargo_target_dir.join("subslice-patterns-const-eval")],
env,
args,
)?;
maybe_run_command_in_vm(&[&cargo_target_dir.join("subslice-patterns-const-eval")], env, args)?;
// FIXME: create a function "display_if_not_quiet" or something along the line.
println!("[AOT] track-caller-attribute");
@ -447,11 +405,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> {
command.push(test_flag);
}
run_command_with_env(&command, None, Some(env))?;
maybe_run_command_in_vm(
&[&cargo_target_dir.join("track-caller-attribute")],
env,
args,
)?;
maybe_run_command_in_vm(&[&cargo_target_dir.join("track-caller-attribute")], env, args)?;
// FIXME: create a function "display_if_not_quiet" or something along the line.
println!("[AOT] mod_bench");
@ -477,11 +431,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> {
);
let rust_dir_path = Path::new(crate::BUILD_DIR).join("rust");
// If the repository was already cloned, command will fail, so doesn't matter.
let _ = git_clone(
"https://github.com/rust-lang/rust.git",
Some(&rust_dir_path),
false,
);
let _ = git_clone("https://github.com/rust-lang/rust.git", Some(&rust_dir_path), false);
let rust_dir: Option<&Path> = Some(&rust_dir_path);
run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?;
run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?;
@ -511,12 +461,8 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> {
}
})?;
let rustc = String::from_utf8(
run_command_with_env(
&[&"rustup", &toolchain, &"which", &"rustc"],
rust_dir,
Some(env),
)?
.stdout,
run_command_with_env(&[&"rustup", &toolchain, &"which", &"rustc"], rust_dir, Some(env))?
.stdout,
)
.map_err(|error| format!("Failed to retrieve rustc path: {:?}", error))
.and_then(|rustc| {
@ -573,13 +519,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<PathBuf, String> {
llvm_filecheck = llvm_filecheck.trim(),
),
)
.map_err(|error| {
format!(
"Failed to write into `{}`: {:?}",
file_path.display(),
error
)
})?;
.map_err(|error| format!("Failed to write into `{}`: {:?}", file_path.display(), error))?;
Ok(rust_dir_path)
}
@ -591,21 +531,19 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> {
env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string());
let extra = if args.is_using_gcc_master_branch() {
""
} else {
" -Csymbol-mangling-version=v0"
};
let extra =
if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" };
let rustc_args = &format!(
r#"-Zpanic-abort-tests \
-Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \
--sysroot "{pwd}/build_sysroot/sysroot" -Cpanic=abort{extra}"#,
--sysroot "{sysroot_dir}" -Cpanic=abort{extra}"#,
pwd = std::env::current_dir()
.map_err(|error| format!("`current_dir` failed: {:?}", error))?
.display(),
channel = args.config_info.channel.as_str(),
dylib_ext = args.config_info.dylib_ext,
sysroot_dir = args.config_info.sysroot_path,
extra = extra,
);
@ -715,8 +653,7 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
};
let projects_path = Path::new("projects");
create_dir_all(projects_path)
.map_err(|err| format!("Failed to create directory `projects`: {}", err))?;
create_dir(projects_path)?;
let nb_parts = args.nb_parts.unwrap_or(0);
if nb_parts > 0 {
@ -737,9 +674,9 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> {
// FIXME: create a function "display_if_not_quiet" or something along the line.
println!("[TEST] libcore");
let path = Path::new("build_sysroot/sysroot_src/library/core/tests");
let path = get_sysroot_dir().join("sysroot_src/library/core/tests");
let _ = remove_dir_all(path.join("target"));
run_cargo_command(&[&"test"], Some(path), env, args)?;
run_cargo_command(&[&"test"], Some(&path), env, args)?;
Ok(())
}
@ -763,10 +700,8 @@ fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> {
}
let mut env = env.clone();
// newer aho_corasick versions throw a deprecation warning
let rustflags = format!(
"{} --cap-lints warn",
env.get("RUSTFLAGS").cloned().unwrap_or_default()
);
let rustflags =
format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default());
env.insert("RUSTFLAGS".to_string(), rustflags);
let path = Path::new(crate::BUILD_DIR).join("rand");
@ -788,18 +723,11 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String>
println!("[TEST] rust-lang/regex example shootout-regex-dna");
let mut env = env.clone();
// newer aho_corasick versions throw a deprecation warning
let rustflags = format!(
"{} --cap-lints warn",
env.get("RUSTFLAGS").cloned().unwrap_or_default()
);
let rustflags =
format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default());
env.insert("RUSTFLAGS".to_string(), rustflags);
// Make sure `[codegen mono items] start` doesn't poison the diff
run_cargo_command(
&[&"build", &"--example", &"shootout-regex-dna"],
Some(&path),
&env,
args,
)?;
run_cargo_command(&[&"build", &"--example", &"shootout-regex-dna"], Some(&path), &env, args)?;
run_cargo_command_with_callback(
&[&"run", &"--example", &"shootout-regex-dna"],
@ -810,10 +738,8 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String>
// FIXME: rewrite this with `child.stdin.write_all()` because
// `examples/regexdna-input.txt` is very small.
let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"bash", &"-c"];
let cargo_args = cargo_command
.iter()
.map(|s| s.as_ref().to_str().unwrap())
.collect::<Vec<_>>();
let cargo_args =
cargo_command.iter().map(|s| s.as_ref().to_str().unwrap()).collect::<Vec<_>>();
let bash_command = format!(
"cat examples/regexdna-input.txt | {} | grep -v 'Spawned thread' > res.txt",
cargo_args.join(" "),
@ -841,10 +767,8 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> {
println!("[TEST] rust-lang/regex tests");
let mut env = env.clone();
// newer aho_corasick versions throw a deprecation warning
let rustflags = format!(
"{} --cap-lints warn",
env.get("RUSTFLAGS").cloned().unwrap_or_default()
);
let rustflags =
format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default());
env.insert("RUSTFLAGS".to_string(), rustflags);
let path = Path::new(crate::BUILD_DIR).join("regex");
run_cargo_command(
@ -899,6 +823,7 @@ fn should_not_remove_test(file: &str) -> bool {
.any(|to_ignore| file.ends_with(to_ignore))
}
#[rustfmt::skip]
fn should_remove_test(file_path: &Path) -> Result<bool, String> {
// Tests generating errors.
let file = File::open(file_path)
@ -916,8 +841,8 @@ fn should_remove_test(file_path: &Path) -> Result<bool, String> {
"//~",
"thread",
]
.iter()
.any(|check| line.contains(check))
.iter()
.any(|check| line.contains(check))
{
return Ok(true);
}
@ -925,11 +850,7 @@ fn should_remove_test(file_path: &Path) -> Result<bool, String> {
return Ok(true);
}
}
if file_path
.display()
.to_string()
.contains("ambiguous-4-extern.rs")
{
if file_path.display().to_string().contains("ambiguous-4-extern.rs") {
eprintln!("nothing found for {file_path:?}");
}
Ok(false)
@ -972,21 +893,13 @@ fn test_rustc_inner<F>(env: &Env, args: &TestArg, prepare_files_callback: F) ->
// These two functions are used to remove files that are known to not be working currently
// with the GCC backend to reduce noise.
fn dir_handling(dir: &Path) -> Result<(), String> {
if dir
.file_name()
.map(|name| name == "auxiliary")
.unwrap_or(true)
{
if dir.file_name().map(|name| name == "auxiliary").unwrap_or(true) {
return Ok(());
}
walk_dir(dir, dir_handling, file_handling)
}
fn file_handling(file_path: &Path) -> Result<(), String> {
if !file_path
.extension()
.map(|extension| extension == "rs")
.unwrap_or(false)
{
if !file_path.extension().map(|extension| extension == "rs").unwrap_or(false) {
return Ok(());
}
let path_str = file_path.display().to_string().replace("\\", "/");
@ -1019,10 +932,7 @@ fn file_handling(file_path: &Path) -> Result<(), String> {
if nb_parts > 0 {
let current_part = args.current_part.unwrap();
// FIXME: create a function "display_if_not_quiet" or something along the line.
println!(
"Splitting ui_test into {} parts (and running part {})",
nb_parts, current_part
);
println!("Splitting ui_test into {} parts (and running part {})", nb_parts, current_part);
let out = String::from_utf8(
run_command(
&[
@ -1062,18 +972,15 @@ fn file_handling(file_path: &Path) -> Result<(), String> {
println!("[TEST] rustc test suite");
env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string());
let extra = if args.is_using_gcc_master_branch() {
""
} else {
" -Csymbol-mangling-version=v0"
};
let extra =
if args.is_using_gcc_master_branch() { "" } else { " -Csymbol-mangling-version=v0" };
let rustc_args = format!(
"{} -Zcodegen-backend={} --sysroot {}{}",
env.get("TEST_FLAGS").unwrap_or(&String::new()),
args.config_info.cg_backend_path,
args.config_info.sysroot_path,
extra,
"{test_flags} -Zcodegen-backend={backend} --sysroot {sysroot}{extra}",
test_flags = env.get("TEST_FLAGS").unwrap_or(&String::new()),
backend = args.config_info.cg_backend_path,
sysroot = args.config_info.sysroot_path,
extra = extra,
);
env.get_mut("RUSTFLAGS").unwrap().clear();
@ -1120,18 +1027,11 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
// Putting back only the failing ones.
let path = "tests/failing-ui-tests.txt";
if let Ok(files) = std::fs::read_to_string(path) {
for file in files
.split('\n')
.map(|line| line.trim())
.filter(|line| !line.is_empty())
{
for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) {
run_command(&[&"git", &"checkout", &"--", &file], Some(&rust_path))?;
}
} else {
println!(
"Failed to read `{}`, not putting back failing ui tests",
path
);
println!("Failed to read `{}`, not putting back failing ui tests", path);
}
Ok(true)
})
@ -1142,19 +1042,12 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
// Removing the failing tests.
let path = "tests/failing-ui-tests.txt";
if let Ok(files) = std::fs::read_to_string(path) {
for file in files
.split('\n')
.map(|line| line.trim())
.filter(|line| !line.is_empty())
{
for file in files.split('\n').map(|line| line.trim()).filter(|line| !line.is_empty()) {
let path = rust_path.join(file);
remove_file(&path)?;
}
} else {
println!(
"Failed to read `{}`, not putting back failing ui tests",
path
);
println!("Failed to read `{}`, not putting back failing ui tests", path);
}
Ok(true)
})
@ -1181,14 +1074,8 @@ pub fn run() -> Result<(), String> {
if !args.use_system_gcc {
args.config_info.setup_gcc_path()?;
env.insert(
"LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert(
"LD_LIBRARY_PATH".to_string(),
args.config_info.gcc_path.clone(),
);
env.insert("LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
env.insert("LD_LIBRARY_PATH".to_string(), args.config_info.gcc_path.clone());
}
build_if_no_backend(&env, &args)?;

73
build_system/src/utils.rs
View File

@ -37,13 +37,8 @@ fn check_exit_status(
}
let mut error = format!(
"Command `{}`{} exited with status {:?}",
input
.iter()
.map(|s| s.as_ref().to_str().unwrap())
.collect::<Vec<_>>()
.join(" "),
cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display()))
.unwrap_or_default(),
input.iter().map(|s| s.as_ref().to_str().unwrap()).collect::<Vec<_>>().join(" "),
cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())).unwrap_or_default(),
exit_status.code()
);
let input = input.iter().map(|i| i.as_ref()).collect::<Vec<&OsStr>>();
@ -68,11 +63,7 @@ fn check_exit_status(
fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String {
format!(
"Command `{}`{} failed to run: {error:?}",
input
.iter()
.map(|s| s.as_ref().to_str().unwrap())
.collect::<Vec<_>>()
.join(" "),
input.iter().map(|s| s.as_ref().to_str().unwrap()).collect::<Vec<_>>().join(" "),
cwd.as_ref()
.map(|cwd| format!(" (running in folder `{}`)", cwd.display(),))
.unwrap_or_default(),
@ -88,9 +79,8 @@ pub fn run_command_with_env(
cwd: Option<&Path>,
env: Option<&HashMap<String, String>>,
) -> Result<Output, String> {
let output = get_command_inner(input, cwd, env)
.output()
.map_err(|e| command_error(input, &cwd, e))?;
let output =
get_command_inner(input, cwd, env).output().map_err(|e| command_error(input, &cwd, e))?;
check_exit_status(input, cwd, output.status, Some(&output), true)?;
Ok(output)
}
@ -164,10 +154,7 @@ pub fn cargo_install(to_install: &str) -> Result<(), String> {
pub fn get_os_name() -> Result<String, String> {
let output = run_command(&[&"uname"], None)?;
let name = std::str::from_utf8(&output.stdout)
.unwrap_or("")
.trim()
.to_string();
let name = std::str::from_utf8(&output.stdout).unwrap_or("").trim().to_string();
if !name.is_empty() {
Ok(name)
} else {
@ -274,11 +261,7 @@ fn git_clone_inner(
command.push(&"1");
}
run_command_with_output(&command, None)?;
Ok(CloneResult {
ran_clone: true,
repo_name,
repo_dir: dest.display().to_string(),
})
Ok(CloneResult { ran_clone: true, repo_name, repo_dir: dest.display().to_string() })
}
fn get_repo_name(url: &str) -> String {
@ -307,6 +290,25 @@ pub fn git_clone(
git_clone_inner(to_clone, dest, shallow_clone, repo_name)
}
pub fn create_dir<P: AsRef<Path>>(path: P) -> Result<(), String> {
fs::create_dir_all(&path).map_err(|error| {
format!("Failed to create directory `{}`: {:?}", path.as_ref().display(), error)
})
}
pub fn copy_file<F: AsRef<Path>, T: AsRef<Path>>(from: F, to: T) -> Result<(), String> {
fs::copy(&from, &to)
.map_err(|error| {
format!(
"Failed to copy file `{}` into `{}`: {:?}",
from.as_ref().display(),
to.as_ref().display(),
error
)
})
.map(|_| ())
}
/// This function differs from `git_clone` in how it handles *where* the repository will be cloned.
/// In `git_clone`, it is cloned in the provided path. In this function, the path you provide is
/// the parent folder. So if you pass "a" as folder and try to clone "b.git", it will be cloned into
@ -318,12 +320,7 @@ pub fn git_clone_root_dir(
) -> Result<CloneResult, String> {
let repo_name = get_repo_name(to_clone);
git_clone_inner(
to_clone,
&dest_parent_dir.join(&repo_name),
shallow_clone,
repo_name,
)
git_clone_inner(to_clone, &dest_parent_dir.join(&repo_name), shallow_clone, repo_name)
}
pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String>
@ -383,11 +380,7 @@ pub fn split_args(args: &str) -> Result<Vec<String>, String> {
}
}
if !found_end {
return Err(format!(
"Didn't find `{}` at the end of `{}`",
end,
&args[start..]
));
return Err(format!("Didn't find `{}` at the end of `{}`", end, &args[start..]));
}
} else if c == '\\' {
// We skip the escaped character.
@ -403,11 +396,7 @@ pub fn split_args(args: &str) -> Result<Vec<String>, String> {
pub fn remove_file<P: AsRef<Path> + ?Sized>(file_path: &P) -> Result<(), String> {
std::fs::remove_file(file_path).map_err(|error| {
format!(
"Failed to remove `{}`: {:?}",
file_path.as_ref().display(),
error
)
format!("Failed to remove `{}`: {:?}", file_path.as_ref().display(), error)
})
}
@ -427,6 +416,10 @@ pub fn create_symlink<P: AsRef<Path>, Q: AsRef<Path>>(original: P, link: Q) -> R
})
}
pub fn get_sysroot_dir() -> PathBuf {
Path::new(crate::BUILD_DIR).join("build_sysroot")
}
#[cfg(test)]
mod tests {
use super::*;

2
tests/lang_tests_common.rs
View File

@ -79,7 +79,7 @@ fn filter(filename: &Path) -> bool {
compiler.args([
&format!("-Zcodegen-backend={}/target/debug/librustc_codegen_gcc.so", current_dir),
"--sysroot",
&format!("{}/build_sysroot/sysroot/", current_dir),
&format!("{}/build/build_sysroot/sysroot/", current_dir),
"-Zno-parallel-llvm",
"-C",
"link-arg=-lc",