Merge pull request #345 from GuillaumeGomez/build-sysroot

Rustify build.sh script

.github/workflows/ci.yml

@@ -119,7 +119,7 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ${{ matrix.libgccjit_version.env_extra }} ./build.sh ${{ matrix.libgccjit_version.extra }}
+        ${{ matrix.libgccjit_version.env_extra }} ./y.sh build ${{ matrix.libgccjit_version.extra }}
         ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
         ./clean_all.sh
 

.github/workflows/release.yml

@@ -86,7 +86,7 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        EMBED_LTO_BITCODE=1 ./build.sh --release --release-sysroot
+        EMBED_LTO_BITCODE=1 ./y.sh build --release --release-sysroot
         cargo test
         ./clean_all.sh
 

.github/workflows/stdarch.yml

@@ -100,7 +100,7 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ./build.sh --release --release-sysroot
+        ./y.sh build --release --release-sysroot
         cargo test
 
     - name: Clean

Readme.md

@@ -66,7 +66,7 @@ Then you can run commands like this:
 
 ```bash
 $ ./y.sh prepare # download and patch sysroot src and install hyperfine for benchmarking
-$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./build.sh --release
+$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./y.sh build --release
 ```
 
 To run the tests:

build.sh

@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-
-#set -x
-set -e
-
-codegen_channel=debug
-sysroot_channel=debug
-
-flags=
-
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --release)
-            codegen_channel=release
-            shift
-            ;;
-        --release-sysroot)
-            sysroot_channel=release
-            shift
-            ;;
-        --no-default-features)
-            flags="$flags --no-default-features"
-            shift
-            ;;
-        --features)
-            shift
-            flags="$flags --features $1"
-            shift
-            ;;
-        *)
-            echo "Unknown option $1"
-            exit 1
-            ;;
-    esac
-done
-
-if [ -f ./gcc_path ]; then
-    export GCC_PATH=$(cat gcc_path)
-else
-    echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details'
-    exit 1
-fi
-
-export LD_LIBRARY_PATH="$GCC_PATH"
-export LIBRARY_PATH="$GCC_PATH"
-
-if [[ "$codegen_channel" == "release" ]]; then
-    export CHANNEL='release'
-    CARGO_INCREMENTAL=1 cargo rustc --release $flags
-else
-    echo $LD_LIBRARY_PATH
-    export CHANNEL='debug'
-    cargo rustc $flags
-fi
-
-source config.sh
-
-rm -r target/out || true
-mkdir -p target/out/gccjit
-
-echo "[BUILD] sysroot"
-if [[ "$sysroot_channel" == "release" ]]; then
-    time ./build_sysroot/build_sysroot.sh --release
-else
-    time ./build_sysroot/build_sysroot.sh
-fi
-

build_sysroot/build_sysroot.sh

@@ -5,9 +5,9 @@
 set -e
 cd $(dirname "$0")
 
-pushd ../ >/dev/null
+pushd ../
 source ./config.sh
-popd >/dev/null
+popd
 
 # Cleanup for previous run
 #     v Clean target dir except for build scripts and incremental cache

build_system/src/build.rs

@@ -1,3 +1,233 @@
-pub fn run() -> Result<(), String> {
+use crate::config::set_config;
+use crate::utils::{
+    get_gcc_path, run_command, run_command_with_env, run_command_with_output_and_env, walk_dir,
+};
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+
+#[derive(Default)]
+struct BuildArg {
+    codegen_release_channel: bool,
+    sysroot_release_channel: bool,
+    features: Vec<String>,
+    gcc_path: String,
+}
+
+impl BuildArg {
+    fn new() -> Result<Option<Self>, String> {
+        let gcc_path = get_gcc_path()?;
+        let mut build_arg = Self {
+            gcc_path,
+            ..Default::default()
+        };
+        // We skip binary name and the `build` command.
+        let mut args = std::env::args().skip(2);
+
+        while let Some(arg) = args.next() {
+            match arg.as_str() {
+                "--release" => build_arg.codegen_release_channel = true,
+                "--release-sysroot" => build_arg.sysroot_release_channel = true,
+                "--no-default-features" => {
+                    build_arg.features.push("--no-default-features".to_string());
+                }
+                "--features" => {
+                    if let Some(arg) = args.next() {
+                        build_arg.features.push("--features".to_string());
+                        build_arg.features.push(arg.as_str().into());
+                    } else {
+                        return Err(
+                            "Expected a value after `--features`, found nothing".to_string()
+                        );
+                    }
+                }
+                "--help" => {
+                    Self::usage();
+                    return Ok(None);
+                }
+                arg => return Err(format!("Unknown argument `{}`", arg)),
+            }
+        }
+        Ok(Some(build_arg))
+    }
+
+    fn usage() {
+        println!(
+            r#"
+`build` command help:
+
+    --release              : Build codegen in release mode
+    --release-sysroot      : Build sysroot in release mode
+    --no-default-features  : Add `--no-default-features` flag
+    --features [arg]       : Add a new feature [arg]
+    --help                 : Show this help
+"#
+        )
+    }
+}
+
+fn build_sysroot(
+    env: &mut HashMap<String, String>,
+    release_mode: bool,
+    target_triple: &str,
+) -> Result<(), String> {
+    std::env::set_current_dir("build_sysroot")
+        .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?;
+    // Cleanup for previous run
+    // Clean target dir except for build scripts and incremental cache
+    let _ = walk_dir(
+        "target",
+        |dir: &Path| {
+            for top in &["debug", "release"] {
+                let _ = fs::remove_dir_all(dir.join(top).join("build"));
+                let _ = fs::remove_dir_all(dir.join(top).join("deps"));
+                let _ = fs::remove_dir_all(dir.join(top).join("examples"));
+                let _ = fs::remove_dir_all(dir.join(top).join("native"));
+
+                let _ = walk_dir(
+                    dir.join(top),
+                    |sub_dir: &Path| {
+                        if sub_dir
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_dir_all(sub_dir);
+                        }
+                        Ok(())
+                    },
+                    |file: &Path| {
+                        if file
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_file(file);
+                        }
+                        Ok(())
+                    },
+                );
+            }
+            Ok(())
+        },
+        |_| Ok(()),
+    );
+
+    let _ = fs::remove_file("Cargo.lock");
+    let _ = fs::remove_file("test_target/Cargo.lock");
+    let _ = fs::remove_dir_all("sysroot");
+
+    // Builds libs
+    let channel = if release_mode {
+        let rustflags = env
+            .get("RUSTFLAGS")
+            .cloned()
+            .unwrap_or_default();
+        env.insert(
+            "RUSTFLAGS".to_string(),
+            format!("{} -Zmir-opt-level=3", rustflags),
+        );
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--release",
+            ],
+            None,
+            Some(&env),
+        )?;
+        "release"
+    } else {
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--features",
+                &"compiler_builtins/c",
+            ],
+            None,
+            Some(env),
+        )?;
+        "debug"
+    };
+
+    // Copy files to sysroot
+    let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", target_triple);
+    fs::create_dir_all(&sysroot_path)
+        .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?;
+    let copier = |dir_to_copy: &Path| {
+        run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
+    };
+    walk_dir(
+        &format!("target/{}/{}/deps", target_triple, channel),
+        copier,
+        copier,
+    )?;
+
+    Ok(())
+}
+
+fn build_codegen(args: &BuildArg) -> Result<(), String> {
+    let mut env = HashMap::new();
+
+    let current_dir =
+        std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    if let Ok(rt_root) = std::env::var("RUST_COMPILER_RT_ROOT") {
+        env.insert("RUST_COMPILER_RT_ROOT".to_string(), rt_root);
+    } else {
+        env.insert(
+            "RUST_COMPILER_RT_ROOT".to_string(),
+            format!("{}", current_dir.join("llvm/compiler-rt").display()),
+        );
+    }
+    env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
+    env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
+
+    let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
+    if args.codegen_release_channel {
+        command.push(&"--release");
+        env.insert("CHANNEL".to_string(), "release".to_string());
+        env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string());
+    } else {
+        env.insert("CHANNEL".to_string(), "debug".to_string());
+    }
+    let ref_features = args.features.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    for feature in &ref_features {
+        command.push(feature);
+    }
+    run_command_with_env(&command, None, Some(&env))?;
+
+    let config = set_config(&mut env, &[], Some(&args.gcc_path))?;
+
+    // We voluntarily ignore the error.
+    let _ = fs::remove_dir_all("target/out");
+    let gccjit_target = "target/out/gccjit";
+    fs::create_dir_all(gccjit_target).map_err(|error| {
+        format!(
+            "Failed to create directory `{}`: {:?}",
+            gccjit_target, error
+        )
+    })?;
+
+    println!("[BUILD] sysroot");
+    build_sysroot(
+        &mut env,
+        args.sysroot_release_channel,
+        &config.target_triple,
+    )?;
+    Ok(())
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match BuildArg::new()? {
+        Some(args) => args,
+        None => return Ok(()),
+    };
+    build_codegen(&args)?;
     Ok(())
 }

build_system/src/config.rs

@@ -0,0 +1,125 @@
+use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple};
+use std::collections::HashMap;
+use std::env as std_env;
+
+pub struct ConfigInfo {
+    pub target_triple: String,
+    pub rustc_command: Vec<String>,
+    pub run_wrapper: Option<&'static str>,
+}
+
+// Returns the beginning for the command line of rustc.
+pub fn set_config(
+    env: &mut HashMap<String, String>,
+    test_flags: &[String],
+    gcc_path: Option<&str>,
+) -> Result<ConfigInfo, String> {
+    env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
+
+    let gcc_path = match gcc_path {
+        Some(path) => path.to_string(),
+        None => get_gcc_path()?,
+    };
+    env.insert("GCC_PATH".to_string(), gcc_path.clone());
+
+    let os_name = get_os_name()?;
+    let dylib_ext = match os_name.as_str() {
+        "Linux" => "so",
+        "Darwin" => "dylib",
+        os => return Err(format!("unsupported OS `{}`", os)),
+    };
+    let host_triple = get_rustc_host_triple()?;
+    let mut linker = None;
+    let mut target_triple = host_triple.as_str();
+    let mut run_wrapper = None;
+    // FIXME: handle this with a command line flag?
+    // let mut target_triple = "m68k-unknown-linux-gnu";
+
+    if host_triple != target_triple {
+        if target_triple == "m68k-unknown-linux-gnu" {
+            target_triple = "mips-unknown-linux-gnu";
+            linker = Some("-Clinker=m68k-linux-gcc");
+        } else if target_triple == "aarch64-unknown-linux-gnu" {
+            // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+            linker = Some("-Clinker=aarch64-linux-gnu-gcc");
+            run_wrapper = Some("qemu-aarch64 -L /usr/aarch64-linux-gnu");
+        } else {
+            return Err(format!("unknown non-native platform `{}`", target_triple));
+        }
+    }
+    let current_dir =
+        std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    let channel = if let Some(channel) = env.get("CHANNEL") {
+        channel.as_str()
+    } else {
+        "debug"
+    };
+    let cg_backend_path = current_dir
+        .join("target")
+        .join(channel)
+        .join(&format!("librustc_codegen_gcc.{}", dylib_ext));
+    let sysroot_path = current_dir.join("build_sysroot/sysroot");
+    let mut rustflags = Vec::new();
+    if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") {
+        rustflags.push(cg_rustflags.clone());
+    }
+    if let Some(linker) = linker {
+        rustflags.push(linker.to_string());
+    }
+    rustflags.extend_from_slice(&[
+        "-Csymbol-mangling-version=v0".to_string(),
+        "-Cdebuginfo=2".to_string(),
+        format!("-Zcodegen-backend={}", cg_backend_path.display()),
+        "--sysroot".to_string(),
+        sysroot_path.display().to_string(),
+    ]);
+
+    // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
+    // TODO(antoyo): remove when we can handle ThinLTO.
+    if !env.contains_key(&"FAT_LTO".to_string()) {
+        rustflags.push("-Clto=off".to_string());
+    }
+    rustflags.extend_from_slice(test_flags);
+    // FIXME(antoyo): remove once the atomic shim is gone
+    if os_name == "Darwin" {
+        rustflags.extend_from_slice(&[
+            "-Clink-arg=-undefined".to_string(),
+            "-Clink-arg=dynamic_lookup".to_string(),
+        ]);
+    }
+    env.insert("RUSTFLAGS".to_string(), rustflags.join(" "));
+    // display metadata load errors
+    env.insert("RUSTC_LOG".to_string(), "warn".to_string());
+
+    let sysroot = current_dir.join(&format!(
+        "build_sysroot/sysroot/lib/rustlib/{}/lib",
+        target_triple
+    ));
+    let ld_library_path = format!(
+        "{target}:{sysroot}:{gcc_path}",
+        target = current_dir.join("target/out").display(),
+        sysroot = sysroot.display(),
+    );
+    env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
+    env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+
+    // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
+    // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
+    // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc
+    let path = std::env::var("PATH").unwrap_or_default();
+    env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path));
+
+    let mut rustc_command = vec!["rustc".to_string()];
+    rustc_command.extend_from_slice(&rustflags);
+    rustc_command.extend_from_slice(&[
+        "-L".to_string(),
+        "crate=target/out".to_string(),
+        "--out-dir".to_string(),
+        "target/out".to_string(),
+    ]);
+    Ok(ConfigInfo {
+        target_triple: target_triple.to_string(),
+        rustc_command,
+        run_wrapper,
+    })
+}

build_system/src/main.rs

@@ -2,6 +2,7 @@ use std::env;
 use std::process;
 
 mod build;
+mod config;
 mod prepare;
 mod rustc_info;
 mod utils;
@@ -16,12 +17,14 @@ macro_rules! arg_error {
 }
 
 fn usage() {
-    println!("\
+    println!(
+        "\
 Available commands for build_system:
 
     prepare  : Run prepare command
     build    : Run build command
-    --help   : Show this message");
+    --help   : Show this message"
+    );
 }
 
 pub enum Command {

build_system/src/prepare.rs

@@ -7,7 +7,7 @@ use std::path::Path;
 fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
     let rustc_path = match get_rustc_path() {
         Some(path) => path,
-        None => return Err("`rustc` path not found".to_owned()),
+        None => return Err("`rustc` path not found".to_string()),
     };
 
     let parent = match rustc_path.parent() {
@@ -15,30 +15,38 @@ fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
         None => return Err(format!("No parent for `{}`", rustc_path.display())),
     };
 
-    let rustlib_dir =
+    let rustlib_dir = parent
-        parent
+        .join("../lib/rustlib/src/rust")
-            .join("../lib/rustlib/src/rust")
+        .canonicalize()
-            .canonicalize()
+        .map_err(|error| format!("Failed to canonicalize path: {:?}", error))?;
-            .map_err(|e| format!("Failed to canonicalize path: {e:?}"))?;
     if !rustlib_dir.is_dir() {
-        return Err("Please install `rust-src` component".to_owned());
+        return Err("Please install `rust-src` component".to_string());
     }
 
     let sysroot_dir = sysroot_path.join("sysroot_src");
     if sysroot_dir.is_dir() {
-        if let Err(e) = fs::remove_dir_all(&sysroot_dir) {
+        if let Err(error) = fs::remove_dir_all(&sysroot_dir) {
-            return Err(format!("Failed to remove `{}`: {:?}", sysroot_dir.display(), e));
+            return Err(format!(
+                "Failed to remove `{}`: {:?}",
+                sysroot_dir.display(),
+                error,
+            ));
         }
     }
 
     let sysroot_library_dir = sysroot_dir.join("library");
-    fs::create_dir_all(&sysroot_library_dir)
+    fs::create_dir_all(&sysroot_library_dir).map_err(|error| {
-        .map_err(|e| format!(
+        format!(
-            "Failed to create folder `{}`: {e:?}",
+            "Failed to create folder `{}`: {:?}",
             sysroot_library_dir.display(),
-        ))?;
+            error,
+        )
+    })?;
 
-    run_command(&[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir], None)?;
+    run_command(
+        &[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir],
+        None,
+    )?;
 
     println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
     run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
@@ -49,17 +57,36 @@ fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
     // This is needed on systems where nothing is configured.
     // git really needs something here, or it will fail.
     // Even using --author is not enough.
-    run_command(&[&"git", &"config", &"user.email", &"none@example.com"], Some(&sysroot_dir))?;
+    run_command(
-    run_command(&[&"git", &"config", &"user.name", &"None"], Some(&sysroot_dir))?;
+        &[&"git", &"config", &"user.email", &"none@example.com"],
-    run_command(&[&"git", &"config", &"core.autocrlf", &"false"], Some(&sysroot_dir))?;
+        Some(&sysroot_dir),
-    run_command(&[&"git", &"config", &"commit.gpgSign", &"false"], Some(&sysroot_dir))?;
+    )?;
-    run_command(&[&"git", &"commit", &"-m", &"Initial commit", &"-q"], Some(&sysroot_dir))?;
+    run_command(
+        &[&"git", &"config", &"user.name", &"None"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"core.autocrlf", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"commit.gpgSign", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"commit", &"-m", &"Initial commit", &"-q"],
+        Some(&sysroot_dir),
+    )?;
 
     let mut patches = Vec::new();
-    walk_dir("patches", |_| Ok(()), |file_path: &Path| {
+    walk_dir(
-        patches.push(file_path.to_path_buf());
+        "patches",
-        Ok(())
+        |_| Ok(()),
-    })?;
+        |file_path: &Path| {
+            patches.push(file_path.to_path_buf());
+            Ok(())
+        },
+    )?;
     patches.sort();
     for file_path in patches {
         println!("[GIT] apply `{}`", file_path.display());
@@ -67,7 +94,13 @@ fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
         run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
         run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
         run_command_with_output(
-            &[&"git", &"commit", &"--no-gpg-sign", &"-m", &format!("Patch {}", path.display())],
+            &[
+                &"git",
+                &"commit",
+                &"--no-gpg-sign",
+                &"-m",
+                &format!("Patch {}", path.display()),
+            ],
             Some(&sysroot_dir),
         )?;
     }
@@ -83,7 +116,10 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
         std::fs::remove_file(&mv_target)
             .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?;
     }
-    run_command(&[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], Some(repo_dir))?;
+    run_command(
+        &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"],
+        Some(repo_dir),
+    )?;
     Ok(())
 }
 
@@ -99,16 +135,20 @@ where
     run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?;
     run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?;
     let filter = format!("-{}-", clone_result.repo_name);
-    walk_dir("crate_patches", |_| Ok(()), |file_path| {
+    walk_dir(
-        let s = file_path.as_os_str().to_str().unwrap();
+        "crate_patches",
-        if s.contains(&filter) && s.ends_with(".patch") {
+        |_| Ok(()),
-            run_command_with_output(
+        |file_path| {
-                &[&"git", &"am", &file_path.canonicalize().unwrap()],
+            let patch = file_path.as_os_str().to_str().unwrap();
-                Some(&repo_path),
+            if patch.contains(&filter) && patch.ends_with(".patch") {
-            )?;
+                run_command_with_output(
-        }
+                    &[&"git", &"am", &file_path.canonicalize().unwrap()],
-        Ok(())
+                    Some(&repo_path),
-    })?;
+                )?;
+            }
+            Ok(())
+        },
+    )?;
     if let Some(extra) = extra {
         extra(&repo_path)?;
     }
@@ -128,23 +168,23 @@ impl PrepareArg {
                 "--only-libcore" => only_libcore = true,
                 "--help" => {
                     Self::usage();
-                    return Ok(None)
+                    return Ok(None);
                 }
                 a => return Err(format!("Unknown argument `{a}`")),
             }
         }
-        Ok(Some(Self {
+        Ok(Some(Self { only_libcore }))
-            only_libcore,
-        }))
     }
 
     fn usage() {
-        println!(r#"
+        println!(
+            r#"
 `prepare` command help:
 
     --only-libcore  : Only setup libcore and don't clone other repositories
     --help          : Show this help
-"#)
+"#
+        )
     }
 }
 
@@ -160,9 +200,21 @@ pub fn run() -> Result<(), String> {
         cargo_install("hyperfine")?;
 
         let to_clone = &[
-            ("https://github.com/rust-random/rand.git", "0f933f9c7176e53b2a3c7952ded484e1783f0bf1", None),
+            (
-            ("https://github.com/rust-lang/regex.git", "341f207c1071f7290e3f228c710817c280c8dca1", None),
+                "https://github.com/rust-random/rand.git",
-            ("https://github.com/ebobby/simple-raytracer", "804a7a21b9e673a482797aa289a18ed480e4d813", Some(build_raytracer)),
+                "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
+                None,
+            ),
+            (
+                "https://github.com/rust-lang/regex.git",
+                "341f207c1071f7290e3f228c710817c280c8dca1",
+                None,
+            ),
+            (
+                "https://github.com/ebobby/simple-raytracer",
+                "804a7a21b9e673a482797aa289a18ed480e4d813",
+                Some(build_raytracer),
+            ),
         ];
 
         for (repo_url, checkout_commit, cb) in to_clone {

build_system/src/rustc_info.rs

@@ -8,5 +8,5 @@ pub fn get_rustc_path() -> Option<PathBuf> {
     }
     run_command(&[&"rustup", &"which", &"rustc"], None)
         .ok()
-        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_owned())
+        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_path_buf())
 }

build_system/src/utils.rs

@@ -1,10 +1,15 @@
+use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::fmt::Debug;
 use std::fs;
 use std::path::Path;
 use std::process::{Command, ExitStatus, Output};
 
-fn get_command_inner(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Command {
+fn get_command_inner(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Command {
     let (cmd, args) = match input {
         [] => panic!("empty command"),
         [cmd, args @ ..] => (cmd, args),
@@ -14,6 +19,9 @@ fn get_command_inner(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Command
     if let Some(cwd) = cwd {
         command.current_dir(cwd);
     }
+    if let Some(env) = env {
+        command.envs(env.iter().map(|(k, v)| (k.as_str(), v.as_str())));
+    }
     command
 }
 
@@ -27,7 +35,8 @@ fn check_exit_status(
     } else {
         Err(format!(
             "Command `{}`{} exited with status {:?}",
-            input.iter()
+            input
+                .iter()
                 .map(|s| s.as_ref().to_str().unwrap())
                 .collect::<Vec<_>>()
                 .join(" "),
@@ -41,21 +50,27 @@ fn check_exit_status(
 fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String {
     format!(
         "Command `{}`{} failed to run: {error:?}",
-        input.iter()
+        input
+            .iter()
             .map(|s| s.as_ref().to_str().unwrap())
             .collect::<Vec<_>>()
             .join(" "),
         cwd.as_ref()
-            .map(|cwd| format!(
+            .map(|cwd| format!(" (running in folder `{}`)", cwd.display(),))
-                " (running in folder `{}`)",
-                cwd.display(),
-            ))
             .unwrap_or_default(),
     )
 }
 
 pub fn run_command(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Result<Output, String> {
-    let output = get_command_inner(input, cwd)
+    run_command_with_env(input, cwd, None)
+}
+
+pub fn run_command_with_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<Output, String> {
+    let output = get_command_inner(input, cwd, env)
         .output()
         .map_err(|e| command_error(input, &cwd, e))?;
     check_exit_status(input, cwd, output.status)?;
@@ -66,7 +81,22 @@ pub fn run_command_with_output(
     input: &[&dyn AsRef<OsStr>],
     cwd: Option<&Path>,
 ) -> Result<(), String> {
-    let exit_status = get_command_inner(input, cwd).spawn()
+    let exit_status = get_command_inner(input, cwd, None)
+        .spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn run_command_with_output_and_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd, env)
+        .spawn()
         .map_err(|e| command_error(input, &cwd, e))?
         .wait()
         .map_err(|e| command_error(input, &cwd, e))?;
@@ -100,6 +130,64 @@ pub fn cargo_install(to_install: &str) -> Result<(), String> {
     Ok(())
 }
 
+pub fn get_os_name() -> Result<String, String> {
+    let output = run_command(&[&"uname"], None)?;
+    let name = std::str::from_utf8(&output.stdout)
+        .unwrap_or("")
+        .trim()
+        .to_string();
+    if !name.is_empty() {
+        Ok(name)
+    } else {
+        Err("Failed to retrieve the OS name".to_string())
+    }
+}
+
+pub fn get_rustc_host_triple() -> Result<String, String> {
+    let output = run_command(&[&"rustc", &"-vV"], None)?;
+    let content = std::str::from_utf8(&output.stdout).unwrap_or("");
+
+    for line in content.split('\n').map(|line| line.trim()) {
+        if !line.starts_with("host:") {
+            continue;
+        }
+        return Ok(line.split(':').nth(1).unwrap().trim().to_string());
+    }
+    Err("Cannot find host triple".to_string())
+}
+
+pub fn get_gcc_path() -> Result<String, String> {
+    let content = match fs::read_to_string("gcc_path") {
+        Ok(content) => content,
+        Err(_) => {
+            return Err(
+                "Please put the path to your custom build of libgccjit in the file \
+                   `gcc_path`, see Readme.md for details"
+                    .into(),
+            )
+        }
+    };
+    match content
+        .split('\n')
+        .map(|line| line.trim())
+        .filter(|line| !line.is_empty())
+        .next()
+    {
+        Some(gcc_path) => {
+            let path = Path::new(gcc_path);
+            if !path.exists() {
+                Err(format!(
+                    "Path `{}` contained in the `gcc_path` file doesn't exist",
+                    gcc_path,
+                ))
+            } else {
+                Ok(gcc_path.into())
+            }
+        }
+        None => Err("No path found in `gcc_path` file".into()),
+    }
+}
+
 pub struct CloneResult {
     pub ran_clone: bool,
     pub repo_name: String,
@@ -108,19 +196,25 @@ pub struct CloneResult {
 pub fn git_clone(to_clone: &str, dest: Option<&Path>) -> Result<CloneResult, String> {
     let repo_name = to_clone.split('/').last().unwrap();
     let repo_name = match repo_name.strip_suffix(".git") {
-        Some(n) => n.to_owned(),
+        Some(n) => n.to_string(),
-        None => repo_name.to_owned(),
+        None => repo_name.to_string(),
     };
 
     let dest = dest
         .map(|dest| dest.join(&repo_name))
         .unwrap_or_else(|| Path::new(&repo_name).into());
     if dest.is_dir() {
-        return Ok(CloneResult { ran_clone: false, repo_name });
+        return Ok(CloneResult {
+            ran_clone: false,
+            repo_name,
+        });
     }
 
     run_command_with_output(&[&"git", &"clone", &to_clone, &dest], None)?;
-    Ok(CloneResult { ran_clone: true, repo_name })
+    Ok(CloneResult {
+        ran_clone: true,
+        repo_name,
+    })
 }
 
 pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String>
@@ -130,8 +224,11 @@ where
     F: FnMut(&Path) -> Result<(), String>,
 {
     let dir = dir.as_ref();
-    for entry in fs::read_dir(dir).map_err(|e| format!("Failed to read dir `{}`: {e:?}", dir.display()))? {
+    for entry in fs::read_dir(dir)
-        let entry = entry.map_err(|e| format!("Failed to read entry in `{}`: {e:?}", dir.display()))?;
+        .map_err(|error| format!("Failed to read dir `{}`: {:?}", dir.display(), error))?
+    {
+        let entry = entry
+            .map_err(|error| format!("Failed to read entry in `{}`: {:?}", dir.display(), error))?;
         let entry_path = entry.path();
         if entry_path.is_dir() {
             dir_cb(&entry_path)?;

config.sh

@@ -48,7 +48,7 @@ fi
 export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
 
 # FIXME(antoyo): remove once the atomic shim is gone
-if [[ `uname` == 'Darwin' ]]; then
+if [[ unamestr == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi