mikros/rust
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1340 from bjorn3/non_rustup_build

Browse Source 
Push up a lot of rustc and cargo references
This commit is contained in:
bjorn3 2023-01-14 16:45:25 +01:00 committed by GitHub
commit b31b74e8fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 135 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build_system/abi_cafe.rs
View File

@ -18,10 +18,10 @@ pub(crate) fn run(
sysroot_kind: SysrootKind,
dirs: &Dirs,
cg_clif_dylib: &Path,
host_compiler: &Compiler,
bootstrap_host_compiler: &Compiler,
) {
if !config::get_bool("testsuite.abi-cafe") {
eprintln!("[SKIP] abi-cafe");
eprintln!("[RUN] abi-cafe (skipped)");
return;
}
@ -31,15 +31,15 @@ pub(crate) fn run(
channel,
sysroot_kind,
cg_clif_dylib,
host_compiler,
&host_compiler.triple,
bootstrap_host_compiler,
bootstrap_host_compiler.triple.clone(),
);
eprintln!("Running abi-cafe");
let pairs = ["rustc_calls_cgclif", "cgclif_calls_rustc", "cgclif_calls_cc", "cc_calls_cgclif"];
let mut cmd = ABI_CAFE.run(host_compiler, dirs);
let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs);
cmd.arg("--");
cmd.arg("--pairs");
cmd.args(pairs);

10
build_system/bench.rs
View File

@ -21,11 +21,11 @@ pub(crate) static SIMPLE_RAYTRACER_LLVM: CargoProject =
pub(crate) static SIMPLE_RAYTRACER: CargoProject =
CargoProject::new(&SIMPLE_RAYTRACER_REPO.source_dir(), "simple_raytracer");
pub(crate) fn benchmark(dirs: &Dirs, host_compiler: &Compiler) {
benchmark_simple_raytracer(dirs, host_compiler);
pub(crate) fn benchmark(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
benchmark_simple_raytracer(dirs, bootstrap_host_compiler);
}
fn benchmark_simple_raytracer(dirs: &Dirs, host_compiler: &Compiler) {
fn benchmark_simple_raytracer(dirs: &Dirs, bootstrap_host_compiler: &Compiler) {
if std::process::Command::new("hyperfine").output().is_err() {
eprintln!("Hyperfine not installed");
eprintln!("Hint: Try `cargo install hyperfine` to install hyperfine");
@ -33,12 +33,12 @@ fn benchmark_simple_raytracer(dirs: &Dirs, host_compiler: &Compiler) {
}
eprintln!("[LLVM BUILD] simple-raytracer");
let build_cmd = SIMPLE_RAYTRACER_LLVM.build(host_compiler, dirs);
let build_cmd = SIMPLE_RAYTRACER_LLVM.build(bootstrap_host_compiler, dirs);
spawn_and_wait(build_cmd);
fs::copy(
SIMPLE_RAYTRACER_LLVM
.target_dir(dirs)
.join(&host_compiler.triple)
.join(&bootstrap_host_compiler.triple)
.join("debug")
.join(get_file_name("main", "bin")),
RelPath::BUILD.to_path(dirs).join(get_file_name("raytracer_cg_llvm", "bin")),

6
build_system/build_backend.rs
View File

@ -10,10 +10,10 @@ pub(crate) static CG_CLIF: CargoProject = CargoProject::new(&RelPath::SOURCE, "c
pub(crate) fn build_backend(
dirs: &Dirs,
channel: &str,
host_compiler: &Compiler,
bootstrap_host_compiler: &Compiler,
use_unstable_features: bool,
) -> PathBuf {
let mut cmd = CG_CLIF.build(&host_compiler, dirs);
let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
cmd.env("CARGO_BUILD_INCREMENTAL", "true"); // Force incr comp even in release mode
@ -48,7 +48,7 @@ pub(crate) fn build_backend(
CG_CLIF
.target_dir(dirs)
.join(&host_compiler.triple)
.join(&bootstrap_host_compiler.triple)
.join(channel)
.join(get_file_name("rustc_codegen_cranelift", "dylib"))
}

56
build_system/build_sysroot.rs
View File

@ -3,7 +3,9 @@ use std::path::Path;
use std::process::{self, Command};
use super::path::{Dirs, RelPath};
use super::rustc_info::{get_file_name, get_rustc_version, get_wrapper_file_name};
use super::rustc_info::{
get_file_name, get_rustc_version, get_toolchain_name, get_wrapper_file_name,
};
use super::utils::{spawn_and_wait, try_hard_link, CargoProject, Compiler};
use super::SysrootKind;
@ -17,15 +19,17 @@ pub(crate) fn build_sysroot(
channel: &str,
sysroot_kind: SysrootKind,
cg_clif_dylib_src: &Path,
host_compiler: &Compiler,
target_triple: &str,
) {
bootstrap_host_compiler: &Compiler,
target_triple: String,
) -> Compiler {
eprintln!("[BUILD] sysroot {:?}", sysroot_kind);
DIST_DIR.ensure_fresh(dirs);
BIN_DIR.ensure_exists(dirs);
LIB_DIR.ensure_exists(dirs);
let is_native = bootstrap_host_compiler.triple == target_triple;
// Copy the backend
let cg_clif_dylib_path = if cfg!(windows) {
// Windows doesn't have rpath support, so the cg_clif dylib needs to be next to the
@ -42,8 +46,9 @@ pub(crate) fn build_sysroot(
for wrapper in ["rustc-clif", "rustdoc-clif", "cargo-clif"] {
let wrapper_name = get_wrapper_file_name(wrapper, "bin");
let mut build_cargo_wrapper_cmd = Command::new("rustc");
let mut build_cargo_wrapper_cmd = Command::new(&bootstrap_host_compiler.rustc);
build_cargo_wrapper_cmd
.env("TOOLCHAIN_NAME", get_toolchain_name())
.arg(RelPath::SCRIPTS.to_path(dirs).join(&format!("{wrapper}.rs")))
.arg("-o")
.arg(DIST_DIR.to_path(dirs).join(wrapper_name))
@ -51,15 +56,16 @@ pub(crate) fn build_sysroot(
spawn_and_wait(build_cargo_wrapper_cmd);
}
let default_sysroot = super::rustc_info::get_default_sysroot();
let default_sysroot = super::rustc_info::get_default_sysroot(&bootstrap_host_compiler.rustc);
let host_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(&host_compiler.triple).join("lib");
let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(target_triple).join("lib");
let host_rustlib_lib =
RUSTLIB_DIR.to_path(dirs).join(&bootstrap_host_compiler.triple).join("lib");
let target_rustlib_lib = RUSTLIB_DIR.to_path(dirs).join(&target_triple).join("lib");
fs::create_dir_all(&host_rustlib_lib).unwrap();
fs::create_dir_all(&target_rustlib_lib).unwrap();
if target_triple == "x86_64-pc-windows-gnu" {
if !default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib").exists() {
if !default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib").exists() {
eprintln!(
"The x86_64-pc-windows-gnu target needs to be installed first before it is possible \
to compile a sysroot for it.",
@ -67,7 +73,7 @@ pub(crate) fn build_sysroot(
process::exit(1);
}
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
)
.unwrap()
{
@ -83,7 +89,11 @@ pub(crate) fn build_sysroot(
SysrootKind::None => {} // Nothing to do
SysrootKind::Llvm => {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(&host_compiler.triple).join("lib"),
default_sysroot
.join("lib")
.join("rustlib")
.join(&bootstrap_host_compiler.triple)
.join("lib"),
)
.unwrap()
{
@ -103,9 +113,9 @@ pub(crate) fn build_sysroot(
try_hard_link(&file, host_rustlib_lib.join(file.file_name().unwrap()));
}
if target_triple != host_compiler.triple {
if !is_native {
for file in fs::read_dir(
default_sysroot.join("lib").join("rustlib").join(target_triple).join("lib"),
default_sysroot.join("lib").join("rustlib").join(&target_triple).join("lib"),
)
.unwrap()
{
@ -118,19 +128,19 @@ pub(crate) fn build_sysroot(
build_clif_sysroot_for_triple(
dirs,
channel,
host_compiler.clone(),
bootstrap_host_compiler.clone(),
&cg_clif_dylib_path,
);
if host_compiler.triple != target_triple {
if !is_native {
build_clif_sysroot_for_triple(
dirs,
channel,
{
let mut target_compiler = host_compiler.clone();
target_compiler.triple = target_triple.to_owned();
target_compiler.set_cross_linker_and_runner();
target_compiler
let mut bootstrap_target_compiler = bootstrap_host_compiler.clone();
bootstrap_target_compiler.triple = target_triple.clone();
bootstrap_target_compiler.set_cross_linker_and_runner();
bootstrap_target_compiler
},
&cg_clif_dylib_path,
);
@ -147,6 +157,12 @@ pub(crate) fn build_sysroot(
}
}
}
let mut target_compiler = Compiler::clif_with_triple(&dirs, target_triple);
if !is_native {
target_compiler.set_cross_linker_and_runner();
}
target_compiler
}
pub(crate) static ORIG_BUILD_SYSROOT: RelPath = RelPath::SOURCE.join("build_sysroot");
@ -169,7 +185,7 @@ fn build_clif_sysroot_for_triple(
process::exit(1);
}
Ok(source_version) => {
let rustc_version = get_rustc_version();
let rustc_version = get_rustc_version(&compiler.rustc);
if source_version != rustc_version {
eprintln!("The patched sysroot source is outdated");
eprintln!("Source version: {}", source_version.trim());

41
build_system/mod.rs
View File

@ -97,7 +97,7 @@ pub fn main() {
}
}
let host_compiler = Compiler::llvm_with_triple(
let bootstrap_host_compiler = Compiler::bootstrap_with_triple(
std::env::var("HOST_TRIPLE")
.ok()
.or_else(|| config::get_value("host"))
@ -106,7 +106,7 @@ pub fn main() {
let target_triple = std::env::var("TARGET_TRIPLE")
.ok()
.or_else(|| config::get_value("target"))
.unwrap_or_else(|| host_compiler.triple.clone());
.unwrap_or_else(|| bootstrap_host_compiler.triple.clone());
// FIXME allow changing the location of these dirs using cli arguments
let current_dir = std::env::current_dir().unwrap();
@ -134,8 +134,15 @@ pub fn main() {
process::exit(0);
}
let cg_clif_dylib =
build_backend::build_backend(&dirs, channel, &host_compiler, use_unstable_features);
env::set_var("RUSTC", "rustc_should_be_set_explicitly");
env::set_var("RUSTDOC", "rustdoc_should_be_set_explicitly");
let cg_clif_dylib = build_backend::build_backend(
&dirs,
channel,
&bootstrap_host_compiler,
use_unstable_features,
);
match command {
Command::Prepare => {
// Handled above
@ -146,14 +153,20 @@ pub fn main() {
channel,
sysroot_kind,
&cg_clif_dylib,
&host_compiler,
&target_triple,
&bootstrap_host_compiler,
target_triple.clone(),
);
if host_compiler.triple == target_triple {
abi_cafe::run(channel, sysroot_kind, &dirs, &cg_clif_dylib, &host_compiler);
if bootstrap_host_compiler.triple == target_triple {
abi_cafe::run(
channel,
sysroot_kind,
&dirs,
&cg_clif_dylib,
&bootstrap_host_compiler,
);
} else {
eprintln!("[SKIP] abi-cafe (cross-compilation not supported)");
eprintln!("[RUN] abi-cafe (skipped, cross-compilation not supported)");
return;
}
}
@ -163,8 +176,8 @@ pub fn main() {
channel,
sysroot_kind,
&cg_clif_dylib,
&host_compiler,
&target_triple,
&bootstrap_host_compiler,
target_triple,
);
}
Command::Bench => {
@ -173,10 +186,10 @@ pub fn main() {
channel,
sysroot_kind,
&cg_clif_dylib,
&host_compiler,
&target_triple,
&bootstrap_host_compiler,
target_triple,
);
bench::benchmark(&dirs, &host_compiler);
bench::benchmark(&dirs, &bootstrap_host_compiler);
}
}
}

4
build_system/prepare.rs
View File

@ -35,7 +35,7 @@ pub(crate) fn prepare(dirs: &Dirs) {
}
fn prepare_sysroot(dirs: &Dirs) {
let sysroot_src_orig = get_default_sysroot().join("lib/rustlib/src/rust");
let sysroot_src_orig = get_default_sysroot(Path::new("rustc")).join("lib/rustlib/src/rust");
assert!(sysroot_src_orig.exists());
eprintln!("[COPY] sysroot src");
@ -50,7 +50,7 @@ fn prepare_sysroot(dirs: &Dirs) {
&SYSROOT_SRC.to_path(dirs).join("library"),
);
let rustc_version = get_rustc_version();
let rustc_version = get_rustc_version(Path::new("rustc"));
fs::write(SYSROOT_RUSTC_VERSION.to_path(dirs), &rustc_version).unwrap();
eprintln!("[GIT] init");

18
build_system/rustc_info.rs
View File

@ -1,9 +1,9 @@
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
pub(crate) fn get_rustc_version() -> String {
pub(crate) fn get_rustc_version(rustc: &Path) -> String {
let version_info =
Command::new("rustc").stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
Command::new(rustc).stderr(Stdio::inherit()).args(&["-V"]).output().unwrap().stdout;
String::from_utf8(version_info).unwrap()
}
@ -23,6 +23,16 @@ pub(crate) fn get_host_triple() -> String {
.to_owned()
}
pub(crate) fn get_toolchain_name() -> String {
let active_toolchain = Command::new("rustup")
.stderr(Stdio::inherit())
.args(&["show", "active-toolchain"])
.output()
.unwrap()
.stdout;
String::from_utf8(active_toolchain).unwrap().trim().split_once(' ').unwrap().0.to_owned()
}
pub(crate) fn get_cargo_path() -> PathBuf {
let cargo_path = Command::new("rustup")
.stderr(Stdio::inherit())
@ -53,8 +63,8 @@ pub(crate) fn get_rustdoc_path() -> PathBuf {
Path::new(String::from_utf8(rustc_path).unwrap().trim()).to_owned()
}
pub(crate) fn get_default_sysroot() -> PathBuf {
let default_sysroot = Command::new("rustc")
pub(crate) fn get_default_sysroot(rustc: &Path) -> PathBuf {
let default_sysroot = Command::new(rustc)
.stderr(Stdio::inherit())
.args(&["--print", "sysroot"])
.output()

77
build_system/tests.rs
View File

@ -3,6 +3,7 @@ use super::build_sysroot::{self, SYSROOT_SRC};
use super::config;
use super::path::{Dirs, RelPath};
use super::prepare::GitRepo;
use super::rustc_info::get_host_triple;
use super::utils::{spawn_and_wait, spawn_and_wait_with_input, CargoProject, Compiler};
use super::SysrootKind;
use std::env;
@ -119,7 +120,7 @@ pub(crate) static LIBCORE_TESTS: CargoProject =
const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
TestCase::custom("test.rust-random/rand", &|runner| {
spawn_and_wait(RAND.clean(&runner.target_compiler.cargo, &runner.dirs));
RAND.clean(&runner.dirs);
if runner.is_native {
eprintln!("[TEST] rust-random/rand");
@ -134,11 +135,11 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.simple-raytracer", &|runner| {
spawn_and_wait(SIMPLE_RAYTRACER.clean(&runner.host_compiler.cargo, &runner.dirs));
SIMPLE_RAYTRACER.clean(&runner.dirs);
spawn_and_wait(SIMPLE_RAYTRACER.build(&runner.target_compiler, &runner.dirs));
}),
TestCase::custom("test.libcore", &|runner| {
spawn_and_wait(LIBCORE_TESTS.clean(&runner.host_compiler.cargo, &runner.dirs));
LIBCORE_TESTS.clean(&runner.dirs);
if runner.is_native {
spawn_and_wait(LIBCORE_TESTS.test(&runner.target_compiler, &runner.dirs));
@ -150,7 +151,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.regex-shootout-regex-dna", &|runner| {
spawn_and_wait(REGEX.clean(&runner.target_compiler.cargo, &runner.dirs));
REGEX.clean(&runner.dirs);
// newer aho_corasick versions throw a deprecation warning
let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
@ -194,7 +195,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.regex", &|runner| {
spawn_and_wait(REGEX.clean(&runner.host_compiler.cargo, &runner.dirs));
REGEX.clean(&runner.dirs);
// newer aho_corasick versions throw a deprecation warning
let lint_rust_flags = format!("{} --cap-lints warn", runner.target_compiler.rustflags);
@ -221,7 +222,7 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
}
}),
TestCase::custom("test.portable-simd", &|runner| {
spawn_and_wait(PORTABLE_SIMD.clean(&runner.host_compiler.cargo, &runner.dirs));
PORTABLE_SIMD.clean(&runner.dirs);
let mut build_cmd = PORTABLE_SIMD.build(&runner.target_compiler, &runner.dirs);
build_cmd.arg("--all-targets");
@ -240,22 +241,22 @@ pub(crate) fn run_tests(
channel: &str,
sysroot_kind: SysrootKind,
cg_clif_dylib: &Path,
host_compiler: &Compiler,
target_triple: &str,
bootstrap_host_compiler: &Compiler,
target_triple: String,
) {
let runner =
TestRunner::new(dirs.clone(), host_compiler.triple.clone(), target_triple.to_string());
if config::get_bool("testsuite.no_sysroot") {
build_sysroot::build_sysroot(
let target_compiler = build_sysroot::build_sysroot(
dirs,
channel,
SysrootKind::None,
cg_clif_dylib,
host_compiler,
&target_triple,
bootstrap_host_compiler,
target_triple.clone(),
);
let runner =
TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
BUILD_EXAMPLE_OUT_DIR.ensure_fresh(dirs);
runner.run_testsuite(NO_SYSROOT_SUITE);
} else {
@ -266,26 +267,29 @@ pub(crate) fn run_tests(
let run_extended_sysroot = config::get_bool("testsuite.extended_sysroot");
if run_base_sysroot || run_extended_sysroot {
build_sysroot::build_sysroot(
let target_compiler = build_sysroot::build_sysroot(
dirs,
channel,
sysroot_kind,
cg_clif_dylib,
host_compiler,
&target_triple,
bootstrap_host_compiler,
target_triple.clone(),
);
}
if run_base_sysroot {
runner.run_testsuite(BASE_SYSROOT_SUITE);
} else {
eprintln!("[SKIP] base_sysroot tests");
}
let runner =
TestRunner::new(dirs.clone(), target_compiler, get_host_triple() == target_triple);
if run_extended_sysroot {
runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
} else {
eprintln!("[SKIP] extended_sysroot tests");
if run_base_sysroot {
runner.run_testsuite(BASE_SYSROOT_SUITE);
} else {
eprintln!("[SKIP] base_sysroot tests");
}
if run_extended_sysroot {
runner.run_testsuite(EXTENDED_SYSROOT_SUITE);
} else {
eprintln!("[SKIP] extended_sysroot tests");
}
}
}
@ -293,22 +297,11 @@ struct TestRunner {
is_native: bool,
jit_supported: bool,
dirs: Dirs,
host_compiler: Compiler,
target_compiler: Compiler,
}
impl TestRunner {
pub fn new(dirs: Dirs, host_triple: String, target_triple: String) -> Self {
let is_native = host_triple == target_triple;
let jit_supported =
is_native && host_triple.contains("x86_64") && !host_triple.contains("windows");
let host_compiler = Compiler::clif_with_triple(&dirs, host_triple);
let mut target_compiler = Compiler::clif_with_triple(&dirs, target_triple);
if !is_native {
target_compiler.set_cross_linker_and_runner();
}
pub fn new(dirs: Dirs, mut target_compiler: Compiler, is_native: bool) -> Self {
if let Ok(rustflags) = env::var("RUSTFLAGS") {
target_compiler.rustflags.push(' ');
target_compiler.rustflags.push_str(&rustflags);
@ -323,7 +316,11 @@ impl TestRunner {
target_compiler.rustflags.push_str(" -Clink-arg=-undefined -Clink-arg=dynamic_lookup");
}
Self { is_native, jit_supported, dirs, host_compiler, target_compiler }
let jit_supported = is_native
&& target_compiler.triple.contains("x86_64")
&& !target_compiler.triple.contains("windows");
Self { is_native, jit_supported, dirs, target_compiler }
}
pub fn run_testsuite(&self, tests: &[TestCase]) {

7
build_system/utils.rs
View File

@ -19,7 +19,7 @@ pub(crate) struct Compiler {
}
impl Compiler {
pub(crate) fn llvm_with_triple(triple: String) -> Compiler {
pub(crate) fn bootstrap_with_triple(triple: String) -> Compiler {
Compiler {
cargo: get_cargo_path(),
rustc: get_rustc_path(),
@ -146,9 +146,8 @@ impl CargoProject {
cmd
}
#[must_use]
pub(crate) fn clean(&self, cargo: &Path, dirs: &Dirs) -> Command {
self.base_cmd("clean", cargo, dirs)
pub(crate) fn clean(&self, dirs: &Dirs) {
let _ = fs::remove_dir_all(self.target_dir(dirs));
}
#[must_use]

2
scripts/cargo-clif.rs
View File

@ -26,7 +26,7 @@ fn main() {
env::set_var("RUSTDOCFLAGS", env::var("RUSTDOCFLAGS").unwrap_or(String::new()) + &rustflags);
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
let args: Vec<_> = match env::args().nth(1).as_deref() {
Some("jit") => {

2
scripts/rustc-clif.rs
View File

@ -24,7 +24,7 @@ fn main() {
}
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
#[cfg(unix)]
Command::new("rustc").args(args).exec();

2
scripts/rustdoc-clif.rs
View File

@ -24,7 +24,7 @@ fn main() {
}
// Ensure that the right toolchain is used
env::set_var("RUSTUP_TOOLCHAIN", env!("RUSTUP_TOOLCHAIN"));
env::set_var("RUSTUP_TOOLCHAIN", env!("TOOLCHAIN_NAME"));
#[cfg(unix)]
Command::new("rustdoc").args(args).exec();